From 962bf2627984f18195f4681e6cebb3a08a9661e3 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Tue, 5 Sep 2023 07:07:16 +0000 Subject: [PATCH 01/44] tv-detection first commit --- cifar100-resnet50/credentials.isc | 2 + credentials copy.isc | 2 + credentials.isc | 2 + fashion_mnist/credentials.isc | 2 + tv-detection/README.md | 88 ++++ tv-detection/coco_eval.py | 192 ++++++++ tv-detection/coco_utils.py | 234 ++++++++++ tv-detection/credentials.isc | 2 + tv-detection/engine.py | 137 ++++++ tv-detection/group_by_aspect_ratio.py | 196 ++++++++ tv-detection/maskrcnn_resnet50_fpn.isc | 6 + tv-detection/prep.py | 6 + tv-detection/presets.py | 114 +++++ tv-detection/train_cycling.py | 282 ++++++++++++ tv-detection/transforms.py | 601 +++++++++++++++++++++++++ tv-detection/utils.py | 282 ++++++++++++ 16 files changed, 2148 insertions(+) create mode 100644 cifar100-resnet50/credentials.isc create mode 100644 credentials copy.isc create mode 100644 credentials.isc create mode 100644 fashion_mnist/credentials.isc create mode 100644 tv-detection/README.md create mode 100644 tv-detection/coco_eval.py create mode 100644 tv-detection/coco_utils.py create mode 100644 tv-detection/credentials.isc create mode 100644 tv-detection/engine.py create mode 100644 tv-detection/group_by_aspect_ratio.py create mode 100644 tv-detection/maskrcnn_resnet50_fpn.isc create mode 100644 tv-detection/prep.py create mode 100644 tv-detection/presets.py create mode 100644 tv-detection/train_cycling.py create mode 100644 tv-detection/transforms.py create mode 100644 tv-detection/utils.py diff --git a/cifar100-resnet50/credentials.isc b/cifar100-resnet50/credentials.isc new file mode 100644 index 00000000..5ec12167 --- /dev/null +++ b/cifar100-resnet50/credentials.isc @@ -0,0 +1,2 @@ +username="usyd04_adam" +api_key="PePtdNJeEYstskgHxezUE2UmVoJgjU" \ No newline at end of file diff --git a/credentials copy.isc b/credentials copy.isc new file mode 100644 index 00000000..5ec12167 --- /dev/null +++ b/credentials copy.isc @@ -0,0 +1,2 @@ +username="usyd04_adam" +api_key="PePtdNJeEYstskgHxezUE2UmVoJgjU" \ No newline at end of file diff --git a/credentials.isc b/credentials.isc new file mode 100644 index 00000000..5ec12167 --- /dev/null +++ b/credentials.isc @@ -0,0 +1,2 @@ +username="usyd04_adam" +api_key="PePtdNJeEYstskgHxezUE2UmVoJgjU" \ No newline at end of file diff --git a/fashion_mnist/credentials.isc b/fashion_mnist/credentials.isc new file mode 100644 index 00000000..5ec12167 --- /dev/null +++ b/fashion_mnist/credentials.isc @@ -0,0 +1,2 @@ +username="usyd04_adam" +api_key="PePtdNJeEYstskgHxezUE2UmVoJgjU" \ No newline at end of file diff --git a/tv-detection/README.md b/tv-detection/README.md new file mode 100644 index 00000000..d9af2652 --- /dev/null +++ b/tv-detection/README.md @@ -0,0 +1,88 @@ +# Object detection reference training scripts + +This folder contains reference training scripts for object detection. +They serve as a log of how to train specific models, to provide baseline +training and evaluation scripts to quickly bootstrap research. + +To execute the example commands below you must install the following: + +``` +cython +pycocotools +matplotlib +``` + +You must modify the following flags: + +`--data-path=/path/to/coco/dataset` + +`--nproc_per_node=` + +Except otherwise noted, all models have been trained on 8x V100 GPUs. + +### Faster R-CNN ResNet-50 FPN +``` +torchrun --nproc_per_node=8 train.py\ + --dataset coco --model fasterrcnn_resnet50_fpn --epochs 26\ + --lr-steps 16 22 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 +``` + +### Faster R-CNN MobileNetV3-Large FPN +``` +torchrun --nproc_per_node=8 train.py\ + --dataset coco --model fasterrcnn_mobilenet_v3_large_fpn --epochs 26\ + --lr-steps 16 22 --aspect-ratio-group-factor 3 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 +``` + +### Faster R-CNN MobileNetV3-Large 320 FPN +``` +torchrun --nproc_per_node=8 train.py\ + --dataset coco --model fasterrcnn_mobilenet_v3_large_320_fpn --epochs 26\ + --lr-steps 16 22 --aspect-ratio-group-factor 3 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 +``` + +### FCOS ResNet-50 FPN +``` +torchrun --nproc_per_node=8 train.py\ + --dataset coco --model fcos_resnet50_fpn --epochs 26\ + --lr-steps 16 22 --aspect-ratio-group-factor 3 --lr 0.01 --amp --weights-backbone ResNet50_Weights.IMAGENET1K_V1 +``` + +### RetinaNet +``` +torchrun --nproc_per_node=8 train.py\ + --dataset coco --model retinanet_resnet50_fpn --epochs 26\ + --lr-steps 16 22 --aspect-ratio-group-factor 3 --lr 0.01 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 +``` + +### SSD300 VGG16 +``` +torchrun --nproc_per_node=8 train.py\ + --dataset coco --model ssd300_vgg16 --epochs 120\ + --lr-steps 80 110 --aspect-ratio-group-factor 3 --lr 0.002 --batch-size 4\ + --weight-decay 0.0005 --data-augmentation ssd --weights-backbone VGG16_Weights.IMAGENET1K_FEATURES +``` + +### SSDlite320 MobileNetV3-Large +``` +torchrun --nproc_per_node=8 train.py\ + --dataset coco --model ssdlite320_mobilenet_v3_large --epochs 660\ + --aspect-ratio-group-factor 3 --lr-scheduler cosineannealinglr --lr 0.15 --batch-size 24\ + --weight-decay 0.00004 --data-augmentation ssdlite +``` + + +### Mask R-CNN +``` +torchrun --nproc_per_node=8 train.py\ + --dataset coco --model maskrcnn_resnet50_fpn --epochs 26\ + --lr-steps 16 22 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 +``` + + +### Keypoint R-CNN +``` +torchrun --nproc_per_node=8 train.py\ + --dataset coco_kp --model keypointrcnn_resnet50_fpn --epochs 46\ + --lr-steps 36 43 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 +``` diff --git a/tv-detection/coco_eval.py b/tv-detection/coco_eval.py new file mode 100644 index 00000000..ba1359f8 --- /dev/null +++ b/tv-detection/coco_eval.py @@ -0,0 +1,192 @@ +import copy +import io +from contextlib import redirect_stdout + +import numpy as np +import pycocotools.mask as mask_util +import torch +import utils +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval + + +class CocoEvaluator: + def __init__(self, coco_gt, iou_types): + if not isinstance(iou_types, (list, tuple)): + raise TypeError(f"This constructor expects iou_types of type list or tuple, instead got {type(iou_types)}") + coco_gt = copy.deepcopy(coco_gt) + self.coco_gt = coco_gt + + self.iou_types = iou_types + self.coco_eval = {} + for iou_type in iou_types: + self.coco_eval[iou_type] = COCOeval(coco_gt, iouType=iou_type) + + self.img_ids = [] + self.eval_imgs = {k: [] for k in iou_types} + + def update(self, predictions): + img_ids = list(np.unique(list(predictions.keys()))) + self.img_ids.extend(img_ids) + + for iou_type in self.iou_types: + results = self.prepare(predictions, iou_type) + with redirect_stdout(io.StringIO()): + coco_dt = COCO.loadRes(self.coco_gt, results) if results else COCO() + coco_eval = self.coco_eval[iou_type] + + coco_eval.cocoDt = coco_dt + coco_eval.params.imgIds = list(img_ids) + img_ids, eval_imgs = evaluate(coco_eval) + + self.eval_imgs[iou_type].append(eval_imgs) + + def synchronize_between_processes(self): + for iou_type in self.iou_types: + self.eval_imgs[iou_type] = np.concatenate(self.eval_imgs[iou_type], 2) + create_common_coco_eval(self.coco_eval[iou_type], self.img_ids, self.eval_imgs[iou_type]) + + def accumulate(self): + for coco_eval in self.coco_eval.values(): + coco_eval.accumulate() + + def summarize(self): + for iou_type, coco_eval in self.coco_eval.items(): + print(f"IoU metric: {iou_type}") + coco_eval.summarize() + + def prepare(self, predictions, iou_type): + if iou_type == "bbox": + return self.prepare_for_coco_detection(predictions) + if iou_type == "segm": + return self.prepare_for_coco_segmentation(predictions) + if iou_type == "keypoints": + return self.prepare_for_coco_keypoint(predictions) + raise ValueError(f"Unknown iou type {iou_type}") + + def prepare_for_coco_detection(self, predictions): + coco_results = [] + for original_id, prediction in predictions.items(): + if len(prediction) == 0: + continue + + boxes = prediction["boxes"] + boxes = convert_to_xywh(boxes).tolist() + scores = prediction["scores"].tolist() + labels = prediction["labels"].tolist() + + coco_results.extend( + [ + { + "image_id": original_id, + "category_id": labels[k], + "bbox": box, + "score": scores[k], + } + for k, box in enumerate(boxes) + ] + ) + return coco_results + + def prepare_for_coco_segmentation(self, predictions): + coco_results = [] + for original_id, prediction in predictions.items(): + if len(prediction) == 0: + continue + + scores = prediction["scores"] + labels = prediction["labels"] + masks = prediction["masks"] + + masks = masks > 0.5 + + scores = prediction["scores"].tolist() + labels = prediction["labels"].tolist() + + rles = [ + mask_util.encode(np.array(mask[0, :, :, np.newaxis], dtype=np.uint8, order="F"))[0] for mask in masks + ] + for rle in rles: + rle["counts"] = rle["counts"].decode("utf-8") + + coco_results.extend( + [ + { + "image_id": original_id, + "category_id": labels[k], + "segmentation": rle, + "score": scores[k], + } + for k, rle in enumerate(rles) + ] + ) + return coco_results + + def prepare_for_coco_keypoint(self, predictions): + coco_results = [] + for original_id, prediction in predictions.items(): + if len(prediction) == 0: + continue + + boxes = prediction["boxes"] + boxes = convert_to_xywh(boxes).tolist() + scores = prediction["scores"].tolist() + labels = prediction["labels"].tolist() + keypoints = prediction["keypoints"] + keypoints = keypoints.flatten(start_dim=1).tolist() + + coco_results.extend( + [ + { + "image_id": original_id, + "category_id": labels[k], + "keypoints": keypoint, + "score": scores[k], + } + for k, keypoint in enumerate(keypoints) + ] + ) + return coco_results + + +def convert_to_xywh(boxes): + xmin, ymin, xmax, ymax = boxes.unbind(1) + return torch.stack((xmin, ymin, xmax - xmin, ymax - ymin), dim=1) + + +def merge(img_ids, eval_imgs): + all_img_ids = utils.all_gather(img_ids) + all_eval_imgs = utils.all_gather(eval_imgs) + + merged_img_ids = [] + for p in all_img_ids: + merged_img_ids.extend(p) + + merged_eval_imgs = [] + for p in all_eval_imgs: + merged_eval_imgs.append(p) + + merged_img_ids = np.array(merged_img_ids) + merged_eval_imgs = np.concatenate(merged_eval_imgs, 2) + + # keep only unique (and in sorted order) images + merged_img_ids, idx = np.unique(merged_img_ids, return_index=True) + merged_eval_imgs = merged_eval_imgs[..., idx] + + return merged_img_ids, merged_eval_imgs + + +def create_common_coco_eval(coco_eval, img_ids, eval_imgs): + img_ids, eval_imgs = merge(img_ids, eval_imgs) + img_ids = list(img_ids) + eval_imgs = list(eval_imgs.flatten()) + + coco_eval.evalImgs = eval_imgs + coco_eval.params.imgIds = img_ids + coco_eval._paramsEval = copy.deepcopy(coco_eval.params) + + +def evaluate(imgs): + with redirect_stdout(io.StringIO()): + imgs.evaluate() + return imgs.params.imgIds, np.asarray(imgs.evalImgs).reshape(-1, len(imgs.params.areaRng), len(imgs.params.imgIds)) diff --git a/tv-detection/coco_utils.py b/tv-detection/coco_utils.py new file mode 100644 index 00000000..f40dcdff --- /dev/null +++ b/tv-detection/coco_utils.py @@ -0,0 +1,234 @@ +import os + +import torch +import torch.utils.data +import torchvision +import transforms as T +from pycocotools import mask as coco_mask +from pycocotools.coco import COCO + + +def convert_coco_poly_to_mask(segmentations, height, width): + masks = [] + for polygons in segmentations: + rles = coco_mask.frPyObjects(polygons, height, width) + mask = coco_mask.decode(rles) + if len(mask.shape) < 3: + mask = mask[..., None] + mask = torch.as_tensor(mask, dtype=torch.uint8) + mask = mask.any(dim=2) + masks.append(mask) + if masks: + masks = torch.stack(masks, dim=0) + else: + masks = torch.zeros((0, height, width), dtype=torch.uint8) + return masks + + +class ConvertCocoPolysToMask: + def __call__(self, image, target): + w, h = image.size + + image_id = target["image_id"] + + anno = target["annotations"] + + anno = [obj for obj in anno if obj["iscrowd"] == 0] + + boxes = [obj["bbox"] for obj in anno] + # guard against no boxes via resizing + boxes = torch.as_tensor(boxes, dtype=torch.float32).reshape(-1, 4) + boxes[:, 2:] += boxes[:, :2] + boxes[:, 0::2].clamp_(min=0, max=w) + boxes[:, 1::2].clamp_(min=0, max=h) + + classes = [obj["category_id"] for obj in anno] + classes = torch.tensor(classes, dtype=torch.int64) + + segmentations = [obj["segmentation"] for obj in anno] + masks = convert_coco_poly_to_mask(segmentations, h, w) + + keypoints = None + if anno and "keypoints" in anno[0]: + keypoints = [obj["keypoints"] for obj in anno] + keypoints = torch.as_tensor(keypoints, dtype=torch.float32) + num_keypoints = keypoints.shape[0] + if num_keypoints: + keypoints = keypoints.view(num_keypoints, -1, 3) + + keep = (boxes[:, 3] > boxes[:, 1]) & (boxes[:, 2] > boxes[:, 0]) + boxes = boxes[keep] + classes = classes[keep] + masks = masks[keep] + if keypoints is not None: + keypoints = keypoints[keep] + + target = {} + target["boxes"] = boxes + target["labels"] = classes + target["masks"] = masks + target["image_id"] = image_id + if keypoints is not None: + target["keypoints"] = keypoints + + # for conversion to coco api + area = torch.tensor([obj["area"] for obj in anno]) + iscrowd = torch.tensor([obj["iscrowd"] for obj in anno]) + target["area"] = area + target["iscrowd"] = iscrowd + + return image, target + + +def _coco_remove_images_without_annotations(dataset, cat_list=None): + def _has_only_empty_bbox(anno): + return all(any(o <= 1 for o in obj["bbox"][2:]) for obj in anno) + + def _count_visible_keypoints(anno): + return sum(sum(1 for v in ann["keypoints"][2::3] if v > 0) for ann in anno) + + min_keypoints_per_image = 10 + + def _has_valid_annotation(anno): + # if it's empty, there is no annotation + if len(anno) == 0: + return False + # if all boxes have close to zero area, there is no annotation + if _has_only_empty_bbox(anno): + return False + # keypoints task have a slight different criteria for considering + # if an annotation is valid + if "keypoints" not in anno[0]: + return True + # for keypoint detection tasks, only consider valid images those + # containing at least min_keypoints_per_image + if _count_visible_keypoints(anno) >= min_keypoints_per_image: + return True + return False + + ids = [] + for ds_idx, img_id in enumerate(dataset.ids): + ann_ids = dataset.coco.getAnnIds(imgIds=img_id, iscrowd=None) + anno = dataset.coco.loadAnns(ann_ids) + if cat_list: + anno = [obj for obj in anno if obj["category_id"] in cat_list] + if _has_valid_annotation(anno): + ids.append(ds_idx) + + dataset = torch.utils.data.Subset(dataset, ids) + return dataset + + +def convert_to_coco_api(ds): + coco_ds = COCO() + # annotation IDs need to start at 1, not 0, see torchvision issue #1530 + ann_id = 1 + dataset = {"images": [], "categories": [], "annotations": []} + categories = set() + for img_idx in range(len(ds)): + # find better way to get target + # targets = ds.get_annotations(img_idx) + img, targets = ds[img_idx] + image_id = targets["image_id"] + img_dict = {} + img_dict["id"] = image_id + img_dict["height"] = img.shape[-2] + img_dict["width"] = img.shape[-1] + dataset["images"].append(img_dict) + bboxes = targets["boxes"].clone() + bboxes[:, 2:] -= bboxes[:, :2] + bboxes = bboxes.tolist() + labels = targets["labels"].tolist() + areas = targets["area"].tolist() + iscrowd = targets["iscrowd"].tolist() + if "masks" in targets: + masks = targets["masks"] + # make masks Fortran contiguous for coco_mask + masks = masks.permute(0, 2, 1).contiguous().permute(0, 2, 1) + if "keypoints" in targets: + keypoints = targets["keypoints"] + keypoints = keypoints.reshape(keypoints.shape[0], -1).tolist() + num_objs = len(bboxes) + for i in range(num_objs): + ann = {} + ann["image_id"] = image_id + ann["bbox"] = bboxes[i] + ann["category_id"] = labels[i] + categories.add(labels[i]) + ann["area"] = areas[i] + ann["iscrowd"] = iscrowd[i] + ann["id"] = ann_id + if "masks" in targets: + ann["segmentation"] = coco_mask.encode(masks[i].numpy()) + if "keypoints" in targets: + ann["keypoints"] = keypoints[i] + ann["num_keypoints"] = sum(k != 0 for k in keypoints[i][2::3]) + dataset["annotations"].append(ann) + ann_id += 1 + dataset["categories"] = [{"id": i} for i in sorted(categories)] + coco_ds.dataset = dataset + coco_ds.createIndex() + return coco_ds + + +def get_coco_api_from_dataset(dataset): + # FIXME: This is... awful? + for _ in range(10): + if isinstance(dataset, torchvision.datasets.CocoDetection): + break + if isinstance(dataset, torch.utils.data.Subset): + dataset = dataset.dataset + if isinstance(dataset, torchvision.datasets.CocoDetection): + return dataset.coco + return convert_to_coco_api(dataset) + + +class CocoDetection(torchvision.datasets.CocoDetection): + def __init__(self, img_folder, ann_file, transforms): + super().__init__(img_folder, ann_file) + self._transforms = transforms + + def __getitem__(self, idx): + img, target = super().__getitem__(idx) + image_id = self.ids[idx] + target = dict(image_id=image_id, annotations=target) + if self._transforms is not None: + img, target = self._transforms(img, target) + return img, target + + +def get_coco(root, image_set, transforms, mode="instances", use_v2=False, with_masks=False): + anno_file_template = "{}_{}2017.json" + PATHS = { + "train": ("train2017", os.path.join("annotations", anno_file_template.format(mode, "train"))), + "val": ("val2017", os.path.join("annotations", anno_file_template.format(mode, "val"))), + # "train": ("val2017", os.path.join("annotations", anno_file_template.format(mode, "val"))) + } + + img_folder, ann_file = PATHS[image_set] + img_folder = os.path.join(root, img_folder) + ann_file = os.path.join(root, ann_file) + + if use_v2: + from torchvision.datasets import wrap_dataset_for_transforms_v2 + + dataset = torchvision.datasets.CocoDetection(img_folder, ann_file, transforms=transforms) + target_keys = ["boxes", "labels", "image_id"] + if with_masks: + target_keys += ["masks"] + dataset = wrap_dataset_for_transforms_v2(dataset, target_keys=target_keys) + else: + # TODO: handle with_masks for V1? + t = [ConvertCocoPolysToMask()] + if transforms is not None: + t.append(transforms) + transforms = T.Compose(t) + + dataset = CocoDetection(img_folder, ann_file, transforms=transforms) + + if image_set == "train": + dataset = _coco_remove_images_without_annotations(dataset) + + # dataset = torch.utils.data.Subset(dataset, [i for i in range(500)]) + + return dataset diff --git a/tv-detection/credentials.isc b/tv-detection/credentials.isc new file mode 100644 index 00000000..5ec12167 --- /dev/null +++ b/tv-detection/credentials.isc @@ -0,0 +1,2 @@ +username="usyd04_adam" +api_key="PePtdNJeEYstskgHxezUE2UmVoJgjU" \ No newline at end of file diff --git a/tv-detection/engine.py b/tv-detection/engine.py new file mode 100644 index 00000000..3496e797 --- /dev/null +++ b/tv-detection/engine.py @@ -0,0 +1,137 @@ +import math +import sys +import time + +import torch +import torchvision.models.detection.mask_rcnn +import utils +from coco_eval import CocoEvaluator +from coco_utils import get_coco_api_from_dataset +from cycling_utils import InterruptableDistributedSampler, atomic_torch_save + + +def train_one_epoch(model, optimizer, data_loader, sampler: InterruptableDistributedSampler, args, device, epoch, print_freq, scaler=None): + model.train() + metric_logger = utils.MetricLogger(delimiter=" ") + metric_logger.add_meter("lr", utils.SmoothedValue(window_size=1, fmt="{value}")) + header = f"Epoch: [{epoch}]" + + lr_scheduler = None + if epoch == 0: + warmup_factor = 1.0 / 1000 + warmup_iters = min(1000, len(data_loader) - 1) + + lr_scheduler = torch.optim.lr_scheduler.LinearLR( + optimizer, start_factor=warmup_factor, total_iters=warmup_iters + ) + + for images, targets in metric_logger.log_every(data_loader, sampler.progress // args.batch_size, print_freq, header): ## EDITED THIS - ARGS.BATCH_SIZE == DATALOADER.BATCH_SIZE? GROUPEDBATCHSAMPLER AT PLAY + images = list(image.to(device) for image in images) + targets = [{k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in t.items()} for t in targets] + with torch.cuda.amp.autocast(enabled=scaler is not None): + loss_dict = model(images, targets) + losses = sum(loss for loss in loss_dict.values()) + + # reduce losses over all GPUs for logging purposes + loss_dict_reduced = utils.reduce_dict(loss_dict) + losses_reduced = sum(loss for loss in loss_dict_reduced.values()) + + loss_value = losses_reduced.item() + + if not math.isfinite(loss_value): + print(f"Loss is {loss_value}, stopping training") + print(loss_dict_reduced) + sys.exit(1) + + optimizer.zero_grad() + if scaler is not None: + scaler.scale(losses).backward() + scaler.step(optimizer) + scaler.update() + else: + losses.backward() + optimizer.step() + + if lr_scheduler is not None: + lr_scheduler.step() + + metric_logger.update(loss=losses_reduced, **loss_dict_reduced) + metric_logger.update(lr=optimizer.param_groups[0]["lr"]) + + # ADDED THE FOLLOWING - INC NECESSARY ARGS TO TRAIN + sampler.advance(len(images)) + step = sampler.progress // args.batch_size + if utils.is_main_process() and step % 5 == 0: + print(f"Saving checkpoint at step {step}") + checkpoint = { + "model": model.module.state_dict(), + "optimizer": optimizer.state_dict(), + # "lr_scheduler": lr_scheduler.state_dict(), # EDITED DUE TO SCHEDULER APPLIED ONLY IN EPOCH 0 + "epoch": epoch, + "args": args, + "sampler": sampler.state_dict(), + } + if epoch == 0: + checkpoint["lr_scheduler"] = lr_scheduler.state_dict() + + if args.amp: + checkpoint["scaler"] = scaler.state_dict() + atomic_torch_save(checkpoint, args.resume) + # END ADD + + return metric_logger + + +def _get_iou_types(model): + model_without_ddp = model + if isinstance(model, torch.nn.parallel.DistributedDataParallel): + model_without_ddp = model.module + iou_types = ["bbox"] + if isinstance(model_without_ddp, torchvision.models.detection.MaskRCNN): + iou_types.append("segm") + if isinstance(model_without_ddp, torchvision.models.detection.KeypointRCNN): + iou_types.append("keypoints") + return iou_types + + +@torch.inference_mode() +def evaluate(model, data_loader, device): + n_threads = torch.get_num_threads() + # FIXME remove this and make paste_masks_in_image run on the GPU + torch.set_num_threads(1) + cpu_device = torch.device("cpu") + model.eval() + metric_logger = utils.MetricLogger(delimiter=" ") + header = "Test:" + + coco = get_coco_api_from_dataset(data_loader.dataset) + iou_types = _get_iou_types(model) + coco_evaluator = CocoEvaluator(coco, iou_types) + + for images, targets in metric_logger.log_every(data_loader, 100, header): + images = list(img.to(device) for img in images) + + if torch.cuda.is_available(): + torch.cuda.synchronize() + model_time = time.time() + outputs = model(images) + + outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] + model_time = time.time() - model_time + + res = {target["image_id"]: output for target, output in zip(targets, outputs)} + evaluator_time = time.time() + coco_evaluator.update(res) + evaluator_time = time.time() - evaluator_time + metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger) + coco_evaluator.synchronize_between_processes() + + # accumulate predictions from all images + coco_evaluator.accumulate() + coco_evaluator.summarize() + torch.set_num_threads(n_threads) + return coco_evaluator diff --git a/tv-detection/group_by_aspect_ratio.py b/tv-detection/group_by_aspect_ratio.py new file mode 100644 index 00000000..d12e14b5 --- /dev/null +++ b/tv-detection/group_by_aspect_ratio.py @@ -0,0 +1,196 @@ +import bisect +import copy +import math +from collections import defaultdict +from itertools import chain, repeat + +import numpy as np +import torch +import torch.utils.data +import torchvision +from PIL import Image +from torch.utils.data.sampler import BatchSampler, Sampler +from torch.utils.model_zoo import tqdm + + +def _repeat_to_at_least(iterable, n): + repeat_times = math.ceil(n / len(iterable)) + repeated = chain.from_iterable(repeat(iterable, repeat_times)) + return list(repeated) + + +class GroupedBatchSampler(BatchSampler): + """ + Wraps another sampler to yield a mini-batch of indices. + It enforces that the batch only contain elements from the same group. + It also tries to provide mini-batches which follows an ordering which is + as close as possible to the ordering from the original sampler. + Args: + sampler (Sampler): Base sampler. + group_ids (list[int]): If the sampler produces indices in range [0, N), + `group_ids` must be a list of `N` ints which contains the group id of each sample. + The group ids must be a continuous set of integers starting from + 0, i.e. they must be in the range [0, num_groups). + batch_size (int): Size of mini-batch. + """ + + def __init__(self, sampler, group_ids, batch_size): + if not isinstance(sampler, Sampler): + raise ValueError(f"sampler should be an instance of torch.utils.data.Sampler, but got sampler={sampler}") + self.sampler = sampler + self.group_ids = group_ids + self.batch_size = batch_size + + def __iter__(self): + buffer_per_group = defaultdict(list) + samples_per_group = defaultdict(list) + + num_batches = 0 + for idx in self.sampler: + group_id = self.group_ids[idx] + buffer_per_group[group_id].append(idx) + samples_per_group[group_id].append(idx) + if len(buffer_per_group[group_id]) == self.batch_size: + yield buffer_per_group[group_id] + num_batches += 1 + del buffer_per_group[group_id] + assert len(buffer_per_group[group_id]) < self.batch_size + + # now we have run out of elements that satisfy + # the group criteria, let's return the remaining + # elements so that the size of the sampler is + # deterministic + expected_num_batches = len(self) + num_remaining = expected_num_batches - num_batches + if num_remaining > 0: + # for the remaining batches, take first the buffers with the largest number + # of elements + for group_id, _ in sorted(buffer_per_group.items(), key=lambda x: len(x[1]), reverse=True): + remaining = self.batch_size - len(buffer_per_group[group_id]) + samples_from_group_id = _repeat_to_at_least(samples_per_group[group_id], remaining) + buffer_per_group[group_id].extend(samples_from_group_id[:remaining]) + assert len(buffer_per_group[group_id]) == self.batch_size + yield buffer_per_group[group_id] + num_remaining -= 1 + if num_remaining == 0: + break + assert num_remaining == 0 + + def __len__(self): + return len(self.sampler) // self.batch_size + + +def _compute_aspect_ratios_slow(dataset, indices=None): + print( + "Your dataset doesn't support the fast path for " + "computing the aspect ratios, so will iterate over " + "the full dataset and load every image instead. " + "This might take some time..." + ) + if indices is None: + indices = range(len(dataset)) + + class SubsetSampler(Sampler): + def __init__(self, indices): + self.indices = indices + + def __iter__(self): + return iter(self.indices) + + def __len__(self): + return len(self.indices) + + sampler = SubsetSampler(indices) + data_loader = torch.utils.data.DataLoader( + dataset, + batch_size=1, + sampler=sampler, + num_workers=14, # you might want to increase it for faster processing + collate_fn=lambda x: x[0], + ) + aspect_ratios = [] + with tqdm(total=len(dataset)) as pbar: + for _i, (img, _) in enumerate(data_loader): + pbar.update(1) + height, width = img.shape[-2:] + aspect_ratio = float(width) / float(height) + aspect_ratios.append(aspect_ratio) + return aspect_ratios + + +def _compute_aspect_ratios_custom_dataset(dataset, indices=None): + if indices is None: + indices = range(len(dataset)) + aspect_ratios = [] + for i in indices: + height, width = dataset.get_height_and_width(i) + aspect_ratio = float(width) / float(height) + aspect_ratios.append(aspect_ratio) + return aspect_ratios + + +def _compute_aspect_ratios_coco_dataset(dataset, indices=None): + if indices is None: + indices = range(len(dataset)) + aspect_ratios = [] + for i in indices: + img_info = dataset.coco.imgs[dataset.ids[i]] + aspect_ratio = float(img_info["width"]) / float(img_info["height"]) + aspect_ratios.append(aspect_ratio) + return aspect_ratios + + +def _compute_aspect_ratios_voc_dataset(dataset, indices=None): + if indices is None: + indices = range(len(dataset)) + aspect_ratios = [] + for i in indices: + # this doesn't load the data into memory, because PIL loads it lazily + width, height = Image.open(dataset.images[i]).size + aspect_ratio = float(width) / float(height) + aspect_ratios.append(aspect_ratio) + return aspect_ratios + + +def _compute_aspect_ratios_subset_dataset(dataset, indices=None): + if indices is None: + indices = range(len(dataset)) + + ds_indices = [dataset.indices[i] for i in indices] + return compute_aspect_ratios(dataset.dataset, ds_indices) + + +def compute_aspect_ratios(dataset, indices=None): + if hasattr(dataset, "get_height_and_width"): + return _compute_aspect_ratios_custom_dataset(dataset, indices) + + if isinstance(dataset, torchvision.datasets.CocoDetection): + return _compute_aspect_ratios_coco_dataset(dataset, indices) + + if isinstance(dataset, torchvision.datasets.VOCDetection): + return _compute_aspect_ratios_voc_dataset(dataset, indices) + + if isinstance(dataset, torch.utils.data.Subset): + return _compute_aspect_ratios_subset_dataset(dataset, indices) + + # slow path + return _compute_aspect_ratios_slow(dataset, indices) + + +def _quantize(x, bins): + bins = copy.deepcopy(bins) + bins = sorted(bins) + quantized = list(map(lambda y: bisect.bisect_right(bins, y), x)) + return quantized + + +def create_aspect_ratio_groups(dataset, k=0): + aspect_ratios = compute_aspect_ratios(dataset) + bins = (2 ** np.linspace(-1, 1, 2 * k + 1)).tolist() if k > 0 else [1.0] + groups = _quantize(aspect_ratios, bins) + # count number of elements per group + counts = np.unique(groups, return_counts=True)[1] + fbins = [0] + bins + [np.inf] + print(f"Using {fbins} as bins for aspect ratio quantization") + print(f"Count of instances per bin: {counts}") + return groups diff --git a/tv-detection/maskrcnn_resnet50_fpn.isc b/tv-detection/maskrcnn_resnet50_fpn.isc new file mode 100644 index 00000000..f640afe0 --- /dev/null +++ b/tv-detection/maskrcnn_resnet50_fpn.isc @@ -0,0 +1,6 @@ +experiment_name="maskrcnn_resnet50_fpn" +gpu_type="24GB VRAM GPU" +nnodes = 9 +venv_path = "~/.venv/bin/activate" +output_path = "~/output_maskrcnn_resnet50_fpn" +command="train_cycling.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/tv-detection/prep.py b/tv-detection/prep.py new file mode 100644 index 00000000..e123c416 --- /dev/null +++ b/tv-detection/prep.py @@ -0,0 +1,6 @@ +import torchvision +from torchvision.models import resnet50, ResNet50_Weights + +# weights = torchvision.models.get_weight('ResNet50_Weights.IMAGENET1K_V1') +_ = resnet50(weights=ResNet50_Weights.IMAGENET1K_V1) +# weights = torchvision.models.get_weight(...) \ No newline at end of file diff --git a/tv-detection/presets.py b/tv-detection/presets.py new file mode 100644 index 00000000..e9b6d56c --- /dev/null +++ b/tv-detection/presets.py @@ -0,0 +1,114 @@ +from collections import defaultdict + +import torch +import transforms as reference_transforms + + +def get_modules(use_v2): + # We need a protected import to avoid the V2 warning in case just V1 is used + if use_v2: + import torchvision.transforms.v2 + import torchvision.tv_tensors + + return torchvision.transforms.v2, torchvision.tv_tensors + else: + return reference_transforms, None + + +class DetectionPresetTrain: + # Note: this transform assumes that the input to forward() are always PIL + # images, regardless of the backend parameter. + def __init__( + self, + *, + data_augmentation, + hflip_prob=0.5, + mean=(123.0, 117.0, 104.0), + backend="pil", + use_v2=False, + ): + + T, tv_tensors = get_modules(use_v2) + + transforms = [] + backend = backend.lower() + if backend == "tv_tensor": + transforms.append(T.ToImage()) + elif backend == "tensor": + transforms.append(T.PILToTensor()) + elif backend != "pil": + raise ValueError(f"backend can be 'tv_tensor', 'tensor' or 'pil', but got {backend}") + + if data_augmentation == "hflip": + transforms += [T.RandomHorizontalFlip(p=hflip_prob)] + elif data_augmentation == "lsj": + transforms += [ + T.ScaleJitter(target_size=(1024, 1024), antialias=True), + # TODO: FixedSizeCrop below doesn't work on tensors! + reference_transforms.FixedSizeCrop(size=(1024, 1024), fill=mean), + T.RandomHorizontalFlip(p=hflip_prob), + ] + elif data_augmentation == "multiscale": + transforms += [ + T.RandomShortestSize(min_size=(480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800), max_size=1333), + T.RandomHorizontalFlip(p=hflip_prob), + ] + elif data_augmentation == "ssd": + fill = defaultdict(lambda: mean, {tv_tensors.Mask: 0}) if use_v2 else list(mean) + transforms += [ + T.RandomPhotometricDistort(), + T.RandomZoomOut(fill=fill), + T.RandomIoUCrop(), + T.RandomHorizontalFlip(p=hflip_prob), + ] + elif data_augmentation == "ssdlite": + transforms += [ + T.RandomIoUCrop(), + T.RandomHorizontalFlip(p=hflip_prob), + ] + else: + raise ValueError(f'Unknown data augmentation policy "{data_augmentation}"') + + if backend == "pil": + # Note: we could just convert to pure tensors even in v2. + transforms += [T.ToImage() if use_v2 else T.PILToTensor()] + + transforms += [T.ToDtype(torch.float, scale=True)] + + if use_v2: + transforms += [ + T.ConvertBoundingBoxFormat(tv_tensors.BoundingBoxFormat.XYXY), + T.SanitizeBoundingBoxes(), + T.ToPureTensor(), + ] + + self.transforms = T.Compose(transforms) + + def __call__(self, img, target): + return self.transforms(img, target) + + +class DetectionPresetEval: + def __init__(self, backend="pil", use_v2=False): + T, _ = get_modules(use_v2) + transforms = [] + backend = backend.lower() + if backend == "pil": + # Note: we could just convert to pure tensors even in v2? + transforms += [T.ToImage() if use_v2 else T.PILToTensor()] + elif backend == "tensor": + transforms += [T.PILToTensor()] + elif backend == "tv_tensor": + transforms += [T.ToImage()] + else: + raise ValueError(f"backend can be 'tv_tensor', 'tensor' or 'pil', but got {backend}") + + transforms += [T.ToDtype(torch.float, scale=True)] + + if use_v2: + transforms += [T.ToPureTensor()] + + self.transforms = T.Compose(transforms) + + def __call__(self, img, target): + return self.transforms(img, target) diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py new file mode 100644 index 00000000..5ea17f46 --- /dev/null +++ b/tv-detection/train_cycling.py @@ -0,0 +1,282 @@ +r"""PyTorch Detection Training. + +To run in a multi-gpu environment, use the distributed launcher:: + + python -m torch.distributed.launch --nproc_per_node=$NGPU --use_env \ + train.py ... --world-size $NGPU + +The default hyperparameters are tuned for training on 8 gpus and 2 images per gpu. + --lr 0.02 --batch-size 2 --world-size 8 +If you use different number of gpus, the learning rate should be changed to 0.02/8*$NGPU. + +On top of that, for training Faster/Mask R-CNN, the default hyperparameters are + --epochs 26 --lr-steps 16 22 --aspect-ratio-group-factor 3 + +Also, if you train Keypoint R-CNN, the default hyperparameters are + --epochs 46 --lr-steps 36 43 --aspect-ratio-group-factor 3 +Because the number of images is smaller in the person keypoint subset of COCO, +the number of epochs should be adapted so that we have the same number of iterations. +""" +import datetime +import os +import time +# import warnings + +from pathlib import Path +import presets +import torch +import torch.utils.data +import torchvision +import utils +from coco_utils import get_coco + +import torchvision.models.detection +import torchvision.models.detection.mask_rcnn +from engine import evaluate, train_one_epoch +from group_by_aspect_ratio import create_aspect_ratio_groups, GroupedBatchSampler +from torchvision.transforms import InterpolationMode +from transforms import SimpleCopyPaste + +from cycling_utils import InterruptableDistributedSampler, atomic_torch_save + +def copypaste_collate_fn(batch): + copypaste = SimpleCopyPaste(blending=True, resize_interpolation=InterpolationMode.BILINEAR) + return copypaste(*utils.collate_fn(batch)) + +def get_dataset(is_train, args): + image_set = "train" if is_train else "val" + num_classes, mode = {"coco": (91, "instances"), "coco_kp": (2, "person_keypoints")}[args.dataset] + with_masks = "mask" in args.model + ds = get_coco( + root=args.data_path, + image_set=image_set, + transforms=get_transform(is_train, args), + mode=mode, + use_v2=args.use_v2, + with_masks=with_masks, + ) + return ds, num_classes + +def get_transform(is_train, args): + if is_train: + return presets.DetectionPresetTrain( + data_augmentation=args.data_augmentation, backend=args.backend, use_v2=args.use_v2 + ) + elif args.weights and args.test_only: + weights = torchvision.models.get_weight(args.weights) + trans = weights.transforms() + return lambda img, target: (trans(img), target) + else: + return presets.DetectionPresetEval(backend=args.backend, use_v2=args.use_v2) + +def main(args): + if args.backend.lower() == "tv_tensor" and not args.use_v2: + raise ValueError("Use --use-v2 if you want to use the tv_tensor backend.") + if args.dataset not in ("coco", "coco_kp"): + raise ValueError(f"Dataset should be coco or coco_kp, got {args.dataset}") + if "keypoint" in args.model and args.dataset != "coco_kp": + raise ValueError("Oops, if you want Keypoint detection, set --dataset coco_kp") + if args.dataset == "coco_kp" and args.use_v2: + raise ValueError("KeyPoint detection doesn't support V2 transforms yet") + + if args.output_dir: + utils.mkdir(args.output_dir) + + utils.init_distributed_mode(args) + print(args) + + device = torch.device(args.device) + + if args.use_deterministic_algorithms: + torch.use_deterministic_algorithms(True) + + # Data loading code + print("Loading data") + dataset, num_classes = get_dataset(is_train=True, args=args) + dataset_test, _ = get_dataset(is_train=False, args=args) + + print("Creating data loaders") + # if args.distributed: + # train_sampler = torch.utils.data.distributed.DistributedSampler(dataset) + # test_sampler = torch.utils.data.distributed.DistributedSampler(dataset_test, shuffle=False) + # else: + # train_sampler = torch.utils.data.RandomSampler(dataset) + # test_sampler = torch.utils.data.SequentialSampler(dataset_test) + train_sampler = InterruptableDistributedSampler(dataset) + test_sampler = torch.utils.data.distributed.DistributedSampler(dataset_test, shuffle=False) + + if args.aspect_ratio_group_factor >= 0: + group_ids = create_aspect_ratio_groups(dataset, k=args.aspect_ratio_group_factor) + train_batch_sampler = GroupedBatchSampler(train_sampler, group_ids, args.batch_size) + else: + train_batch_sampler = torch.utils.data.BatchSampler(train_sampler, args.batch_size, drop_last=True) + + train_collate_fn = utils.collate_fn + if args.use_copypaste: + if args.data_augmentation != "lsj": + raise RuntimeError("SimpleCopyPaste algorithm currently only supports the 'lsj' data augmentation policies") + train_collate_fn = copypaste_collate_fn + + data_loader = torch.utils.data.DataLoader( + dataset, batch_sampler=train_batch_sampler, num_workers=args.workers, collate_fn=train_collate_fn + ) + data_loader_test = torch.utils.data.DataLoader( + dataset_test, batch_size=1, sampler=test_sampler, num_workers=args.workers, collate_fn=utils.collate_fn + ) + + print("Creating model") + kwargs = {"trainable_backbone_layers": args.trainable_backbone_layers} + if args.data_augmentation in ["multiscale", "lsj"]: + kwargs["_skip_resize"] = True + if "rcnn" in args.model: + if args.rpn_score_thresh is not None: + kwargs["rpn_score_thresh"] = args.rpn_score_thresh + + model = torchvision.models.get_model( + args.model, weights=args.weights, weights_backbone=args.weights_backbone, num_classes=num_classes, **kwargs + ) + + model.to(device) + if args.distributed and args.sync_bn: + model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model) + + model_without_ddp = model + if args.distributed: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + model_without_ddp = model.module + + if args.norm_weight_decay is None: + parameters = [p for p in model.parameters() if p.requires_grad] + else: + param_groups = torchvision.ops._utils.split_normalization_params(model) + wd_groups = [args.norm_weight_decay, args.weight_decay] + parameters = [{"params": p, "weight_decay": w} for p, w in zip(param_groups, wd_groups) if p] + + opt_name = args.opt.lower() + if opt_name.startswith("sgd"): + optimizer = torch.optim.SGD( + parameters, + lr=args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay, + nesterov="nesterov" in opt_name, + ) + elif opt_name == "adamw": + optimizer = torch.optim.AdamW(parameters, lr=args.lr, weight_decay=args.weight_decay) + else: + raise RuntimeError(f"Invalid optimizer {args.opt}. Only SGD and AdamW are supported.") + + scaler = torch.cuda.amp.GradScaler() if args.amp else None + + args.lr_scheduler = args.lr_scheduler.lower() + if args.lr_scheduler == "multisteplr": + lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=args.lr_steps, gamma=args.lr_gamma) + elif args.lr_scheduler == "cosineannealinglr": + lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=args.epochs) + else: + raise RuntimeError( + f"Invalid lr scheduler '{args.lr_scheduler}'. Only MultiStepLR and CosineAnnealingLR are supported." + ) + + Path(args.resume).parent.mkdir(parents=True, exist_ok=True) ### ADDED THIS + if args.resume and os.path.isfile(args.resume): ## EDITED THIS + checkpoint = torch.load(args.resume, map_location="cpu") + model_without_ddp.load_state_dict(checkpoint["model"]) + optimizer.load_state_dict(checkpoint["optimizer"]) + if "lr_scheduler" in checkpoint: ## EDITED THIS + lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) ## EDITED THIS + args.start_epoch = checkpoint["epoch"] # + 1 + if args.amp: + scaler.load_state_dict(checkpoint["scaler"]) + train_sampler.load_state_dict(checkpoint["sampler"]) + + if args.test_only: + torch.backends.cudnn.deterministic = True + evaluate(model, data_loader_test, device=device) + return + + print("Start training") + start_time = time.time() + for epoch in range(args.start_epoch, args.epochs): + # if args.distributed: + # train_sampler.set_epoch(epoch) + with train_sampler.in_epoch(epoch): + train_one_epoch(model, optimizer, data_loader, train_sampler, args, device, epoch, args.print_freq, scaler) + lr_scheduler.step() + if args.output_dir: + checkpoint = { + "model": model_without_ddp.state_dict(), + "optimizer": optimizer.state_dict(), + "lr_scheduler": lr_scheduler.state_dict(), + "args": args, + "epoch": epoch, + } + if args.amp: + checkpoint["scaler"] = scaler.state_dict() + # utils.save_on_master(checkpoint, os.path.join(args.output_dir, f"model_{epoch}.pth")) + # utils.save_on_master(checkpoint, os.path.join(args.output_dir, "checkpoint.pth")) + atomic_torch_save(checkpoint, args.resume) + + # # KILL THIS FOR NOW + # # evaluate after every epoch + # evaluate(model, data_loader_test, device=device) + + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print(f"Training time {total_time_str}") + + +def get_args_parser(add_help=True): + import argparse + + parser = argparse.ArgumentParser(description="PyTorch Detection Training", add_help=add_help) + + parser.add_argument("--data-path", default="/datasets01/COCO/022719/", type=str, help="dataset path") + parser.add_argument("--dataset",default="coco",type=str,help="dataset name. Use coco for object detection and instance segmentation and coco_kp for Keypoint detection",) + parser.add_argument("--model", default="maskrcnn_resnet50_fpn", type=str, help="model name") + parser.add_argument("--device", default="cuda", type=str, help="device (Use cuda or cpu Default: cuda)") + parser.add_argument("-b", "--batch-size", default=2, type=int, help="images per gpu, the total batch size is $NGPU x batch_size") + parser.add_argument("--epochs", default=26, type=int, metavar="N", help="number of total epochs to run") + parser.add_argument("-j", "--workers", default=4, type=int, metavar="N", help="number of data loading workers (default: 4)") + parser.add_argument("--opt", default="sgd", type=str, help="optimizer") + parser.add_argument("--lr",default=0.02,type=float,help="initial learning rate, 0.02 is the default value for training on 8 gpus and 2 images_per_gpu") + parser.add_argument("--momentum", default=0.9, type=float, metavar="M", help="momentum") + parser.add_argument("--wd","--weight-decay",default=1e-4,type=float,metavar="W",help="weight decay (default: 1e-4)",dest="weight_decay",) + parser.add_argument("--norm-weight-decay",default=None,type=float,help="weight decay for Normalization layers (default: None, same value as --wd)") + parser.add_argument("--lr-scheduler", default="multisteplr", type=str, help="name of lr scheduler (default: multisteplr)") + parser.add_argument("--lr-step-size", default=8, type=int, help="decrease lr every step-size epochs (multisteplr scheduler only)") + parser.add_argument("--lr-steps",default=[16, 22],nargs="+",type=int,help="decrease lr every step-size epochs (multisteplr scheduler only)") + parser.add_argument("--lr-gamma", default=0.1, type=float, help="decrease lr by a factor of lr-gamma (multisteplr scheduler only)") + parser.add_argument("--print-freq", default=20, type=int, help="print frequency") + parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") + parser.add_argument("--resume", default="", type=str, help="path of checkpoint") + parser.add_argument("--start_epoch", default=0, type=int, help="start epoch") + parser.add_argument("--aspect-ratio-group-factor", default=3, type=int) + parser.add_argument("--rpn-score-thresh", default=None, type=float, help="rpn score threshold for faster-rcnn") + parser.add_argument("--trainable-backbone-layers", default=None, type=int, help="number of trainable layers of backbone") + parser.add_argument("--data-augmentation", default="hflip", type=str, help="data augmentation policy (default: hflip)") + parser.add_argument("--sync-bn",dest="sync_bn",help="Use sync batch norm",action="store_true") + parser.add_argument("--test-only",dest="test_only",help="Only test the model",action="store_true") + parser.add_argument("--use-deterministic-algorithms", action="store_true", help="Forces the use of deterministic algorithms only.") + + # distributed training parameters + parser.add_argument("--world-size", default=1, type=int, help="number of distributed processes") + parser.add_argument("--dist-url", default="env://", type=str, help="url used to set up distributed training") + parser.add_argument("--weights", default=None, type=str, help="the weights enum name to load") + parser.add_argument("--weights-backbone", default=None, type=str, help="the backbone weights enum name to load") + + # Mixed precision training parameters + parser.add_argument("--amp", action="store_true", help="Use torch.cuda.amp for mixed precision training") + + # Use CopyPaste augmentation training parameter + parser.add_argument("--use-copypaste",action="store_true",help="Use CopyPaste data augmentation. Works only with data-augmentation='lsj'.",) + + parser.add_argument("--backend", default="PIL", type=str.lower, help="PIL or tensor - case insensitive") + parser.add_argument("--use-v2", action="store_true", help="Use V2 transforms") + + return parser + + +if __name__ == "__main__": + args = get_args_parser().parse_args() + main(args) diff --git a/tv-detection/transforms.py b/tv-detection/transforms.py new file mode 100644 index 00000000..e07ccfc9 --- /dev/null +++ b/tv-detection/transforms.py @@ -0,0 +1,601 @@ +from typing import Dict, List, Optional, Tuple, Union + +import torch +import torchvision +from torch import nn, Tensor +from torchvision import ops +from torchvision.transforms import functional as F, InterpolationMode, transforms as T + + +def _flip_coco_person_keypoints(kps, width): + flip_inds = [0, 2, 1, 4, 3, 6, 5, 8, 7, 10, 9, 12, 11, 14, 13, 16, 15] + flipped_data = kps[:, flip_inds] + flipped_data[..., 0] = width - flipped_data[..., 0] + # Maintain COCO convention that if visibility == 0, then x, y = 0 + inds = flipped_data[..., 2] == 0 + flipped_data[inds] = 0 + return flipped_data + + +class Compose: + def __init__(self, transforms): + self.transforms = transforms + + def __call__(self, image, target): + for t in self.transforms: + image, target = t(image, target) + return image, target + + +class RandomHorizontalFlip(T.RandomHorizontalFlip): + def forward( + self, image: Tensor, target: Optional[Dict[str, Tensor]] = None + ) -> Tuple[Tensor, Optional[Dict[str, Tensor]]]: + if torch.rand(1) < self.p: + image = F.hflip(image) + if target is not None: + _, _, width = F.get_dimensions(image) + target["boxes"][:, [0, 2]] = width - target["boxes"][:, [2, 0]] + if "masks" in target: + target["masks"] = target["masks"].flip(-1) + if "keypoints" in target: + keypoints = target["keypoints"] + keypoints = _flip_coco_person_keypoints(keypoints, width) + target["keypoints"] = keypoints + return image, target + + +class PILToTensor(nn.Module): + def forward( + self, image: Tensor, target: Optional[Dict[str, Tensor]] = None + ) -> Tuple[Tensor, Optional[Dict[str, Tensor]]]: + image = F.pil_to_tensor(image) + return image, target + + +class ToDtype(nn.Module): + def __init__(self, dtype: torch.dtype, scale: bool = False) -> None: + super().__init__() + self.dtype = dtype + self.scale = scale + + def forward( + self, image: Tensor, target: Optional[Dict[str, Tensor]] = None + ) -> Tuple[Tensor, Optional[Dict[str, Tensor]]]: + if not self.scale: + return image.to(dtype=self.dtype), target + image = F.convert_image_dtype(image, self.dtype) + return image, target + + +class RandomIoUCrop(nn.Module): + def __init__( + self, + min_scale: float = 0.3, + max_scale: float = 1.0, + min_aspect_ratio: float = 0.5, + max_aspect_ratio: float = 2.0, + sampler_options: Optional[List[float]] = None, + trials: int = 40, + ): + super().__init__() + # Configuration similar to https://github.com/weiliu89/caffe/blob/ssd/examples/ssd/ssd_coco.py#L89-L174 + self.min_scale = min_scale + self.max_scale = max_scale + self.min_aspect_ratio = min_aspect_ratio + self.max_aspect_ratio = max_aspect_ratio + if sampler_options is None: + sampler_options = [0.0, 0.1, 0.3, 0.5, 0.7, 0.9, 1.0] + self.options = sampler_options + self.trials = trials + + def forward( + self, image: Tensor, target: Optional[Dict[str, Tensor]] = None + ) -> Tuple[Tensor, Optional[Dict[str, Tensor]]]: + if target is None: + raise ValueError("The targets can't be None for this transform.") + + if isinstance(image, torch.Tensor): + if image.ndimension() not in {2, 3}: + raise ValueError(f"image should be 2/3 dimensional. Got {image.ndimension()} dimensions.") + elif image.ndimension() == 2: + image = image.unsqueeze(0) + + _, orig_h, orig_w = F.get_dimensions(image) + + while True: + # sample an option + idx = int(torch.randint(low=0, high=len(self.options), size=(1,))) + min_jaccard_overlap = self.options[idx] + if min_jaccard_overlap >= 1.0: # a value larger than 1 encodes the leave as-is option + return image, target + + for _ in range(self.trials): + # check the aspect ratio limitations + r = self.min_scale + (self.max_scale - self.min_scale) * torch.rand(2) + new_w = int(orig_w * r[0]) + new_h = int(orig_h * r[1]) + aspect_ratio = new_w / new_h + if not (self.min_aspect_ratio <= aspect_ratio <= self.max_aspect_ratio): + continue + + # check for 0 area crops + r = torch.rand(2) + left = int((orig_w - new_w) * r[0]) + top = int((orig_h - new_h) * r[1]) + right = left + new_w + bottom = top + new_h + if left == right or top == bottom: + continue + + # check for any valid boxes with centers within the crop area + cx = 0.5 * (target["boxes"][:, 0] + target["boxes"][:, 2]) + cy = 0.5 * (target["boxes"][:, 1] + target["boxes"][:, 3]) + is_within_crop_area = (left < cx) & (cx < right) & (top < cy) & (cy < bottom) + if not is_within_crop_area.any(): + continue + + # check at least 1 box with jaccard limitations + boxes = target["boxes"][is_within_crop_area] + ious = torchvision.ops.boxes.box_iou( + boxes, torch.tensor([[left, top, right, bottom]], dtype=boxes.dtype, device=boxes.device) + ) + if ious.max() < min_jaccard_overlap: + continue + + # keep only valid boxes and perform cropping + target["boxes"] = boxes + target["labels"] = target["labels"][is_within_crop_area] + target["boxes"][:, 0::2] -= left + target["boxes"][:, 1::2] -= top + target["boxes"][:, 0::2].clamp_(min=0, max=new_w) + target["boxes"][:, 1::2].clamp_(min=0, max=new_h) + image = F.crop(image, top, left, new_h, new_w) + + return image, target + + +class RandomZoomOut(nn.Module): + def __init__( + self, fill: Optional[List[float]] = None, side_range: Tuple[float, float] = (1.0, 4.0), p: float = 0.5 + ): + super().__init__() + if fill is None: + fill = [0.0, 0.0, 0.0] + self.fill = fill + self.side_range = side_range + if side_range[0] < 1.0 or side_range[0] > side_range[1]: + raise ValueError(f"Invalid canvas side range provided {side_range}.") + self.p = p + + @torch.jit.unused + def _get_fill_value(self, is_pil): + # type: (bool) -> int + # We fake the type to make it work on JIT + return tuple(int(x) for x in self.fill) if is_pil else 0 + + def forward( + self, image: Tensor, target: Optional[Dict[str, Tensor]] = None + ) -> Tuple[Tensor, Optional[Dict[str, Tensor]]]: + if isinstance(image, torch.Tensor): + if image.ndimension() not in {2, 3}: + raise ValueError(f"image should be 2/3 dimensional. Got {image.ndimension()} dimensions.") + elif image.ndimension() == 2: + image = image.unsqueeze(0) + + if torch.rand(1) >= self.p: + return image, target + + _, orig_h, orig_w = F.get_dimensions(image) + + r = self.side_range[0] + torch.rand(1) * (self.side_range[1] - self.side_range[0]) + canvas_width = int(orig_w * r) + canvas_height = int(orig_h * r) + + r = torch.rand(2) + left = int((canvas_width - orig_w) * r[0]) + top = int((canvas_height - orig_h) * r[1]) + right = canvas_width - (left + orig_w) + bottom = canvas_height - (top + orig_h) + + if torch.jit.is_scripting(): + fill = 0 + else: + fill = self._get_fill_value(F._is_pil_image(image)) + + image = F.pad(image, [left, top, right, bottom], fill=fill) + if isinstance(image, torch.Tensor): + # PyTorch's pad supports only integers on fill. So we need to overwrite the colour + v = torch.tensor(self.fill, device=image.device, dtype=image.dtype).view(-1, 1, 1) + image[..., :top, :] = image[..., :, :left] = image[..., (top + orig_h) :, :] = image[ + ..., :, (left + orig_w) : + ] = v + + if target is not None: + target["boxes"][:, 0::2] += left + target["boxes"][:, 1::2] += top + + return image, target + + +class RandomPhotometricDistort(nn.Module): + def __init__( + self, + contrast: Tuple[float, float] = (0.5, 1.5), + saturation: Tuple[float, float] = (0.5, 1.5), + hue: Tuple[float, float] = (-0.05, 0.05), + brightness: Tuple[float, float] = (0.875, 1.125), + p: float = 0.5, + ): + super().__init__() + self._brightness = T.ColorJitter(brightness=brightness) + self._contrast = T.ColorJitter(contrast=contrast) + self._hue = T.ColorJitter(hue=hue) + self._saturation = T.ColorJitter(saturation=saturation) + self.p = p + + def forward( + self, image: Tensor, target: Optional[Dict[str, Tensor]] = None + ) -> Tuple[Tensor, Optional[Dict[str, Tensor]]]: + if isinstance(image, torch.Tensor): + if image.ndimension() not in {2, 3}: + raise ValueError(f"image should be 2/3 dimensional. Got {image.ndimension()} dimensions.") + elif image.ndimension() == 2: + image = image.unsqueeze(0) + + r = torch.rand(7) + + if r[0] < self.p: + image = self._brightness(image) + + contrast_before = r[1] < 0.5 + if contrast_before: + if r[2] < self.p: + image = self._contrast(image) + + if r[3] < self.p: + image = self._saturation(image) + + if r[4] < self.p: + image = self._hue(image) + + if not contrast_before: + if r[5] < self.p: + image = self._contrast(image) + + if r[6] < self.p: + channels, _, _ = F.get_dimensions(image) + permutation = torch.randperm(channels) + + is_pil = F._is_pil_image(image) + if is_pil: + image = F.pil_to_tensor(image) + image = F.convert_image_dtype(image) + image = image[..., permutation, :, :] + if is_pil: + image = F.to_pil_image(image) + + return image, target + + +class ScaleJitter(nn.Module): + """Randomly resizes the image and its bounding boxes within the specified scale range. + The class implements the Scale Jitter augmentation as described in the paper + `"Simple Copy-Paste is a Strong Data Augmentation Method for Instance Segmentation" `_. + + Args: + target_size (tuple of ints): The target size for the transform provided in (height, weight) format. + scale_range (tuple of ints): scaling factor interval, e.g (a, b), then scale is randomly sampled from the + range a <= scale <= b. + interpolation (InterpolationMode): Desired interpolation enum defined by + :class:`torchvision.transforms.InterpolationMode`. Default is ``InterpolationMode.BILINEAR``. + """ + + def __init__( + self, + target_size: Tuple[int, int], + scale_range: Tuple[float, float] = (0.1, 2.0), + interpolation: InterpolationMode = InterpolationMode.BILINEAR, + antialias=True, + ): + super().__init__() + self.target_size = target_size + self.scale_range = scale_range + self.interpolation = interpolation + self.antialias = antialias + + def forward( + self, image: Tensor, target: Optional[Dict[str, Tensor]] = None + ) -> Tuple[Tensor, Optional[Dict[str, Tensor]]]: + if isinstance(image, torch.Tensor): + if image.ndimension() not in {2, 3}: + raise ValueError(f"image should be 2/3 dimensional. Got {image.ndimension()} dimensions.") + elif image.ndimension() == 2: + image = image.unsqueeze(0) + + _, orig_height, orig_width = F.get_dimensions(image) + + scale = self.scale_range[0] + torch.rand(1) * (self.scale_range[1] - self.scale_range[0]) + r = min(self.target_size[1] / orig_height, self.target_size[0] / orig_width) * scale + new_width = int(orig_width * r) + new_height = int(orig_height * r) + + image = F.resize(image, [new_height, new_width], interpolation=self.interpolation, antialias=self.antialias) + + if target is not None: + target["boxes"][:, 0::2] *= new_width / orig_width + target["boxes"][:, 1::2] *= new_height / orig_height + if "masks" in target: + target["masks"] = F.resize( + target["masks"], + [new_height, new_width], + interpolation=InterpolationMode.NEAREST, + antialias=self.antialias, + ) + + return image, target + + +class FixedSizeCrop(nn.Module): + def __init__(self, size, fill=0, padding_mode="constant"): + super().__init__() + size = tuple(T._setup_size(size, error_msg="Please provide only two dimensions (h, w) for size.")) + self.crop_height = size[0] + self.crop_width = size[1] + self.fill = fill # TODO: Fill is currently respected only on PIL. Apply tensor patch. + self.padding_mode = padding_mode + + def _pad(self, img, target, padding): + # Taken from the functional_tensor.py pad + if isinstance(padding, int): + pad_left = pad_right = pad_top = pad_bottom = padding + elif len(padding) == 1: + pad_left = pad_right = pad_top = pad_bottom = padding[0] + elif len(padding) == 2: + pad_left = pad_right = padding[0] + pad_top = pad_bottom = padding[1] + else: + pad_left = padding[0] + pad_top = padding[1] + pad_right = padding[2] + pad_bottom = padding[3] + + padding = [pad_left, pad_top, pad_right, pad_bottom] + img = F.pad(img, padding, self.fill, self.padding_mode) + if target is not None: + target["boxes"][:, 0::2] += pad_left + target["boxes"][:, 1::2] += pad_top + if "masks" in target: + target["masks"] = F.pad(target["masks"], padding, 0, "constant") + + return img, target + + def _crop(self, img, target, top, left, height, width): + img = F.crop(img, top, left, height, width) + if target is not None: + boxes = target["boxes"] + boxes[:, 0::2] -= left + boxes[:, 1::2] -= top + boxes[:, 0::2].clamp_(min=0, max=width) + boxes[:, 1::2].clamp_(min=0, max=height) + + is_valid = (boxes[:, 0] < boxes[:, 2]) & (boxes[:, 1] < boxes[:, 3]) + + target["boxes"] = boxes[is_valid] + target["labels"] = target["labels"][is_valid] + if "masks" in target: + target["masks"] = F.crop(target["masks"][is_valid], top, left, height, width) + + return img, target + + def forward(self, img, target=None): + _, height, width = F.get_dimensions(img) + new_height = min(height, self.crop_height) + new_width = min(width, self.crop_width) + + if new_height != height or new_width != width: + offset_height = max(height - self.crop_height, 0) + offset_width = max(width - self.crop_width, 0) + + r = torch.rand(1) + top = int(offset_height * r) + left = int(offset_width * r) + + img, target = self._crop(img, target, top, left, new_height, new_width) + + pad_bottom = max(self.crop_height - new_height, 0) + pad_right = max(self.crop_width - new_width, 0) + if pad_bottom != 0 or pad_right != 0: + img, target = self._pad(img, target, [0, 0, pad_right, pad_bottom]) + + return img, target + + +class RandomShortestSize(nn.Module): + def __init__( + self, + min_size: Union[List[int], Tuple[int], int], + max_size: int, + interpolation: InterpolationMode = InterpolationMode.BILINEAR, + ): + super().__init__() + self.min_size = [min_size] if isinstance(min_size, int) else list(min_size) + self.max_size = max_size + self.interpolation = interpolation + + def forward( + self, image: Tensor, target: Optional[Dict[str, Tensor]] = None + ) -> Tuple[Tensor, Optional[Dict[str, Tensor]]]: + _, orig_height, orig_width = F.get_dimensions(image) + + min_size = self.min_size[torch.randint(len(self.min_size), (1,)).item()] + r = min(min_size / min(orig_height, orig_width), self.max_size / max(orig_height, orig_width)) + + new_width = int(orig_width * r) + new_height = int(orig_height * r) + + image = F.resize(image, [new_height, new_width], interpolation=self.interpolation) + + if target is not None: + target["boxes"][:, 0::2] *= new_width / orig_width + target["boxes"][:, 1::2] *= new_height / orig_height + if "masks" in target: + target["masks"] = F.resize( + target["masks"], [new_height, new_width], interpolation=InterpolationMode.NEAREST + ) + + return image, target + + +def _copy_paste( + image: torch.Tensor, + target: Dict[str, Tensor], + paste_image: torch.Tensor, + paste_target: Dict[str, Tensor], + blending: bool = True, + resize_interpolation: F.InterpolationMode = F.InterpolationMode.BILINEAR, +) -> Tuple[torch.Tensor, Dict[str, Tensor]]: + + # Random paste targets selection: + num_masks = len(paste_target["masks"]) + + if num_masks < 1: + # Such degerante case with num_masks=0 can happen with LSJ + # Let's just return (image, target) + return image, target + + # We have to please torch script by explicitly specifying dtype as torch.long + random_selection = torch.randint(0, num_masks, (num_masks,), device=paste_image.device) + random_selection = torch.unique(random_selection).to(torch.long) + + paste_masks = paste_target["masks"][random_selection] + paste_boxes = paste_target["boxes"][random_selection] + paste_labels = paste_target["labels"][random_selection] + + masks = target["masks"] + + # We resize source and paste data if they have different sizes + # This is something we introduced here as originally the algorithm works + # on equal-sized data (for example, coming from LSJ data augmentations) + size1 = image.shape[-2:] + size2 = paste_image.shape[-2:] + if size1 != size2: + paste_image = F.resize(paste_image, size1, interpolation=resize_interpolation) + paste_masks = F.resize(paste_masks, size1, interpolation=F.InterpolationMode.NEAREST) + # resize bboxes: + ratios = torch.tensor((size1[1] / size2[1], size1[0] / size2[0]), device=paste_boxes.device) + paste_boxes = paste_boxes.view(-1, 2, 2).mul(ratios).view(paste_boxes.shape) + + paste_alpha_mask = paste_masks.sum(dim=0) > 0 + + if blending: + paste_alpha_mask = F.gaussian_blur( + paste_alpha_mask.unsqueeze(0), + kernel_size=(5, 5), + sigma=[ + 2.0, + ], + ) + + # Copy-paste images: + image = (image * (~paste_alpha_mask)) + (paste_image * paste_alpha_mask) + + # Copy-paste masks: + masks = masks * (~paste_alpha_mask) + non_all_zero_masks = masks.sum((-1, -2)) > 0 + masks = masks[non_all_zero_masks] + + # Do a shallow copy of the target dict + out_target = {k: v for k, v in target.items()} + + out_target["masks"] = torch.cat([masks, paste_masks]) + + # Copy-paste boxes and labels + boxes = ops.masks_to_boxes(masks) + out_target["boxes"] = torch.cat([boxes, paste_boxes]) + + labels = target["labels"][non_all_zero_masks] + out_target["labels"] = torch.cat([labels, paste_labels]) + + # Update additional optional keys: area and iscrowd if exist + if "area" in target: + out_target["area"] = out_target["masks"].sum((-1, -2)).to(torch.float32) + + if "iscrowd" in target and "iscrowd" in paste_target: + # target['iscrowd'] size can be differ from mask size (non_all_zero_masks) + # For example, if previous transforms geometrically modifies masks/boxes/labels but + # does not update "iscrowd" + if len(target["iscrowd"]) == len(non_all_zero_masks): + iscrowd = target["iscrowd"][non_all_zero_masks] + paste_iscrowd = paste_target["iscrowd"][random_selection] + out_target["iscrowd"] = torch.cat([iscrowd, paste_iscrowd]) + + # Check for degenerated boxes and remove them + boxes = out_target["boxes"] + degenerate_boxes = boxes[:, 2:] <= boxes[:, :2] + if degenerate_boxes.any(): + valid_targets = ~degenerate_boxes.any(dim=1) + + out_target["boxes"] = boxes[valid_targets] + out_target["masks"] = out_target["masks"][valid_targets] + out_target["labels"] = out_target["labels"][valid_targets] + + if "area" in out_target: + out_target["area"] = out_target["area"][valid_targets] + if "iscrowd" in out_target and len(out_target["iscrowd"]) == len(valid_targets): + out_target["iscrowd"] = out_target["iscrowd"][valid_targets] + + return image, out_target + + +class SimpleCopyPaste(torch.nn.Module): + def __init__(self, blending=True, resize_interpolation=F.InterpolationMode.BILINEAR): + super().__init__() + self.resize_interpolation = resize_interpolation + self.blending = blending + + def forward( + self, images: List[torch.Tensor], targets: List[Dict[str, Tensor]] + ) -> Tuple[List[torch.Tensor], List[Dict[str, Tensor]]]: + torch._assert( + isinstance(images, (list, tuple)) and all([isinstance(v, torch.Tensor) for v in images]), + "images should be a list of tensors", + ) + torch._assert( + isinstance(targets, (list, tuple)) and len(images) == len(targets), + "targets should be a list of the same size as images", + ) + for target in targets: + # Can not check for instance type dict with inside torch.jit.script + # torch._assert(isinstance(target, dict), "targets item should be a dict") + for k in ["masks", "boxes", "labels"]: + torch._assert(k in target, f"Key {k} should be present in targets") + torch._assert(isinstance(target[k], torch.Tensor), f"Value for the key {k} should be a tensor") + + # images = [t1, t2, ..., tN] + # Let's define paste_images as shifted list of input images + # paste_images = [t2, t3, ..., tN, t1] + # FYI: in TF they mix data on the dataset level + images_rolled = images[-1:] + images[:-1] + targets_rolled = targets[-1:] + targets[:-1] + + output_images: List[torch.Tensor] = [] + output_targets: List[Dict[str, Tensor]] = [] + + for image, target, paste_image, paste_target in zip(images, targets, images_rolled, targets_rolled): + output_image, output_data = _copy_paste( + image, + target, + paste_image, + paste_target, + blending=self.blending, + resize_interpolation=self.resize_interpolation, + ) + output_images.append(output_image) + output_targets.append(output_data) + + return output_images, output_targets + + def __repr__(self) -> str: + s = f"{self.__class__.__name__}(blending={self.blending}, resize_interpolation={self.resize_interpolation})" + return s diff --git a/tv-detection/utils.py b/tv-detection/utils.py new file mode 100644 index 00000000..37386c8a --- /dev/null +++ b/tv-detection/utils.py @@ -0,0 +1,282 @@ +import datetime +import errno +import os +import time +from collections import defaultdict, deque + +import torch +import torch.distributed as dist + + +class SmoothedValue: + """Track a series of values and provide access to smoothed values over a + window or the global series average. + """ + + def __init__(self, window_size=20, fmt=None): + if fmt is None: + fmt = "{median:.4f} ({global_avg:.4f})" + self.deque = deque(maxlen=window_size) + self.total = 0.0 + self.count = 0 + self.fmt = fmt + + def update(self, value, n=1): + self.deque.append(value) + self.count += n + self.total += value * n + + def synchronize_between_processes(self): + """ + Warning: does not synchronize the deque! + """ + if not is_dist_avail_and_initialized(): + return + t = torch.tensor([self.count, self.total], dtype=torch.float64, device="cuda") + dist.barrier() + dist.all_reduce(t) + t = t.tolist() + self.count = int(t[0]) + self.total = t[1] + + @property + def median(self): + d = torch.tensor(list(self.deque)) + return d.median().item() + + @property + def avg(self): + d = torch.tensor(list(self.deque), dtype=torch.float32) + return d.mean().item() + + @property + def global_avg(self): + return self.total / self.count + + @property + def max(self): + return max(self.deque) + + @property + def value(self): + return self.deque[-1] + + def __str__(self): + return self.fmt.format( + median=self.median, avg=self.avg, global_avg=self.global_avg, max=self.max, value=self.value + ) + + +def all_gather(data): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors) + Args: + data: any picklable object + Returns: + list[data]: list of data gathered from each rank + """ + world_size = get_world_size() + if world_size == 1: + return [data] + data_list = [None] * world_size + dist.all_gather_object(data_list, data) + return data_list + + +def reduce_dict(input_dict, average=True): + """ + Args: + input_dict (dict): all the values will be reduced + average (bool): whether to do average or sum + Reduce the values in the dictionary from all processes so that all processes + have the averaged results. Returns a dict with the same fields as + input_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return input_dict + with torch.inference_mode(): + names = [] + values = [] + # sort the keys so that they are consistent across processes + for k in sorted(input_dict.keys()): + names.append(k) + values.append(input_dict[k]) + values = torch.stack(values, dim=0) + dist.all_reduce(values) + if average: + values /= world_size + reduced_dict = {k: v for k, v in zip(names, values)} + return reduced_dict + + +class MetricLogger: + def __init__(self, delimiter="\t"): + self.meters = defaultdict(SmoothedValue) + self.delimiter = delimiter + + def update(self, **kwargs): + for k, v in kwargs.items(): + if isinstance(v, torch.Tensor): + v = v.item() + assert isinstance(v, (float, int)) + self.meters[k].update(v) + + def __getattr__(self, attr): + if attr in self.meters: + return self.meters[attr] + if attr in self.__dict__: + return self.__dict__[attr] + raise AttributeError(f"'{type(self).__name__}' object has no attribute '{attr}'") + + def __str__(self): + loss_str = [] + for name, meter in self.meters.items(): + loss_str.append(f"{name}: {str(meter)}") + return self.delimiter.join(loss_str) + + def synchronize_between_processes(self): + for meter in self.meters.values(): + meter.synchronize_between_processes() + + def add_meter(self, name, meter): + self.meters[name] = meter + + def log_every(self, iterable, i, print_freq, header=None): + # i = 0 + if not header: + header = "" + start_time = time.time() + end = time.time() + iter_time = SmoothedValue(fmt="{avg:.4f}") + data_time = SmoothedValue(fmt="{avg:.4f}") + space_fmt = ":" + str(len(str(len(iterable)))) + "d" + if torch.cuda.is_available(): + log_msg = self.delimiter.join( + [ + header, + "[{0" + space_fmt + "}/{1}]", + "eta: {eta}", + "{meters}", + "time: {time}", + "data: {data}", + "max mem: {memory:.0f}", + ] + ) + else: + log_msg = self.delimiter.join( + [header, "[{0" + space_fmt + "}/{1}]", "eta: {eta}", "{meters}", "time: {time}", "data: {data}"] + ) + MB = 1024.0 * 1024.0 + for obj in iterable: + data_time.update(time.time() - end) + yield obj + iter_time.update(time.time() - end) + if i % print_freq == 0: # or i == len(iterable) - 1: ## EDITED - POTENTIALLY UNNECESSARY + eta_seconds = iter_time.global_avg * (len(iterable) - i) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + if torch.cuda.is_available(): + print( + log_msg.format( + i, + len(iterable), + eta=eta_string, + meters=str(self), + time=str(iter_time), + data=str(data_time), + memory=torch.cuda.max_memory_allocated() / MB, + ) + ) + else: + print( + log_msg.format( + i, len(iterable), eta=eta_string, meters=str(self), time=str(iter_time), data=str(data_time) + ) + ) + i += 1 + end = time.time() + total_time = time.time() - start_time + total_time_str = str(datetime.timedelta(seconds=int(total_time))) + print(f"{header} Total time: {total_time_str}") # ({total_time / len(iterable):.4f} s / it)") ## EDITED + + +def collate_fn(batch): + return tuple(zip(*batch)) + + +def mkdir(path): + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + import builtins as __builtin__ + + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop("force", False) + if is_master or force: + builtin_print(*args, **kwargs) + + __builtin__.print = print + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def is_main_process(): + return get_rank() == 0 + + +def save_on_master(*args, **kwargs): + if is_main_process(): + torch.save(*args, **kwargs) + + +def init_distributed_mode(args): + if "RANK" in os.environ and "WORLD_SIZE" in os.environ: + args.rank = int(os.environ["RANK"]) + args.world_size = int(os.environ["WORLD_SIZE"]) + args.gpu = int(os.environ["LOCAL_RANK"]) + elif "SLURM_PROCID" in os.environ: + args.rank = int(os.environ["SLURM_PROCID"]) + args.gpu = args.rank % torch.cuda.device_count() + else: + print("Not using distributed mode") + args.distributed = False + return + + args.distributed = True + + torch.cuda.set_device(args.gpu) + args.dist_backend = "nccl" + print(f"| distributed init (rank {args.rank}): {args.dist_url}", flush=True) + torch.distributed.init_process_group( + backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank + ) + torch.distributed.barrier() + setup_for_distributed(args.rank == 0) From 41e4d05e64a990a8b90009cc22f37fa3047d91f9 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Wed, 6 Sep 2023 03:14:46 +0000 Subject: [PATCH 02/44] tv-detections working, pending epoch completion --- cifar100-resnet50/credentials.isc | 2 -- credentials copy.isc | 2 -- credentials.isc | 2 -- fashion_mnist/credentials.isc | 2 -- tv-detection/credentials.isc | 2 -- tv-detection/engine.py | 36 ++++++++++++------------- tv-detection/retinanet_resnet50_fpn.isc | 7 +++++ tv-detection/train_cycling.py | 30 +++++++++++++++------ tv-segmentation/fcn_resnet101.isc | 4 +-- tv-segmentation/prep.py | 3 +++ tv-segmentation/train_cycling.py | 5 ++-- 11 files changed, 54 insertions(+), 41 deletions(-) delete mode 100644 cifar100-resnet50/credentials.isc delete mode 100644 credentials copy.isc delete mode 100644 credentials.isc delete mode 100644 fashion_mnist/credentials.isc delete mode 100644 tv-detection/credentials.isc create mode 100644 tv-detection/retinanet_resnet50_fpn.isc diff --git a/cifar100-resnet50/credentials.isc b/cifar100-resnet50/credentials.isc deleted file mode 100644 index 5ec12167..00000000 --- a/cifar100-resnet50/credentials.isc +++ /dev/null @@ -1,2 +0,0 @@ -username="usyd04_adam" -api_key="PePtdNJeEYstskgHxezUE2UmVoJgjU" \ No newline at end of file diff --git a/credentials copy.isc b/credentials copy.isc deleted file mode 100644 index 5ec12167..00000000 --- a/credentials copy.isc +++ /dev/null @@ -1,2 +0,0 @@ -username="usyd04_adam" -api_key="PePtdNJeEYstskgHxezUE2UmVoJgjU" \ No newline at end of file diff --git a/credentials.isc b/credentials.isc deleted file mode 100644 index 5ec12167..00000000 --- a/credentials.isc +++ /dev/null @@ -1,2 +0,0 @@ -username="usyd04_adam" -api_key="PePtdNJeEYstskgHxezUE2UmVoJgjU" \ No newline at end of file diff --git a/fashion_mnist/credentials.isc b/fashion_mnist/credentials.isc deleted file mode 100644 index 5ec12167..00000000 --- a/fashion_mnist/credentials.isc +++ /dev/null @@ -1,2 +0,0 @@ -username="usyd04_adam" -api_key="PePtdNJeEYstskgHxezUE2UmVoJgjU" \ No newline at end of file diff --git a/tv-detection/credentials.isc b/tv-detection/credentials.isc deleted file mode 100644 index 5ec12167..00000000 --- a/tv-detection/credentials.isc +++ /dev/null @@ -1,2 +0,0 @@ -username="usyd04_adam" -api_key="PePtdNJeEYstskgHxezUE2UmVoJgjU" \ No newline at end of file diff --git a/tv-detection/engine.py b/tv-detection/engine.py index 3496e797..ee6da38e 100644 --- a/tv-detection/engine.py +++ b/tv-detection/engine.py @@ -9,23 +9,22 @@ from coco_utils import get_coco_api_from_dataset from cycling_utils import InterruptableDistributedSampler, atomic_torch_save - -def train_one_epoch(model, optimizer, data_loader, sampler: InterruptableDistributedSampler, args, device, epoch, print_freq, scaler=None): +def train_one_epoch(model, optimizer, data_loader, train_batch_sampler, lr_scheduler, warmup_lr_scheduler, args, device, epoch, print_freq, scaler=None): model.train() metric_logger = utils.MetricLogger(delimiter=" ") metric_logger.add_meter("lr", utils.SmoothedValue(window_size=1, fmt="{value}")) header = f"Epoch: [{epoch}]" - lr_scheduler = None - if epoch == 0: - warmup_factor = 1.0 / 1000 - warmup_iters = min(1000, len(data_loader) - 1) - - lr_scheduler = torch.optim.lr_scheduler.LinearLR( - optimizer, start_factor=warmup_factor, total_iters=warmup_iters - ) + # # warmup_lr_scheduler will either be None if (starting from scratch or after epoch 0) or not None, in which case run with that. + # warmup_lr_scheduler = warmup_lr_scheduler if warmup_lr_scheduler else None + # if warmup_lr_scheduler is None and epoch == 0: + # warmup_factor = 1.0 / 1000 + # warmup_iters = min(1000, len(data_loader) - 1) + # warmup_lr_scheduler = torch.optim.lr_scheduler.LinearLR( + # optimizer, start_factor=warmup_factor, total_iters=warmup_iters + # ) - for images, targets in metric_logger.log_every(data_loader, sampler.progress // args.batch_size, print_freq, header): ## EDITED THIS - ARGS.BATCH_SIZE == DATALOADER.BATCH_SIZE? GROUPEDBATCHSAMPLER AT PLAY + for images, targets in metric_logger.log_every(data_loader, train_batch_sampler.sampler.progress // args.batch_size, print_freq, header): ## EDITED THIS - ARGS.BATCH_SIZE == DATALOADER.BATCH_SIZE? GROUPEDBATCHSAMPLER AT PLAY images = list(image.to(device) for image in images) targets = [{k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in t.items()} for t in targets] with torch.cuda.amp.autocast(enabled=scaler is not None): @@ -52,27 +51,26 @@ def train_one_epoch(model, optimizer, data_loader, sampler: InterruptableDistrib losses.backward() optimizer.step() - if lr_scheduler is not None: - lr_scheduler.step() + ## Always update warmup_lr_scheduler - once progressed past epoch 0, this will make no difference. + warmup_lr_scheduler.step() metric_logger.update(loss=losses_reduced, **loss_dict_reduced) metric_logger.update(lr=optimizer.param_groups[0]["lr"]) # ADDED THE FOLLOWING - INC NECESSARY ARGS TO TRAIN - sampler.advance(len(images)) - step = sampler.progress // args.batch_size + train_batch_sampler.sampler.advance(len(images)) + step = train_batch_sampler.sampler.progress // args.batch_size if utils.is_main_process() and step % 5 == 0: print(f"Saving checkpoint at step {step}") checkpoint = { "model": model.module.state_dict(), "optimizer": optimizer.state_dict(), - # "lr_scheduler": lr_scheduler.state_dict(), # EDITED DUE TO SCHEDULER APPLIED ONLY IN EPOCH 0 + "lr_scheduler": lr_scheduler.state_dict(), + "warmup_lr_scheduler": warmup_lr_scheduler.state_dict(), "epoch": epoch, "args": args, - "sampler": sampler.state_dict(), + "sampler": train_batch_sampler.sampler.state_dict(), } - if epoch == 0: - checkpoint["lr_scheduler"] = lr_scheduler.state_dict() if args.amp: checkpoint["scaler"] = scaler.state_dict() diff --git a/tv-detection/retinanet_resnet50_fpn.isc b/tv-detection/retinanet_resnet50_fpn.isc new file mode 100644 index 00000000..fe195144 --- /dev/null +++ b/tv-detection/retinanet_resnet50_fpn.isc @@ -0,0 +1,7 @@ +experiment_name="retinanet_resnet50_fpn" +gpu_type="24GB VRAM GPU" +nnodes = 9 +venv_path = "~/.venv/bin/activate" +output_path = "~/output_retinanet_resnet50_fpn" +command="train_cycling.py --dataset coco --model retinanet_resnet50_fpn --epochs 26 --lr-steps 16 22 --aspect-ratio-group-factor 3 --lr 0.01 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" + diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index 5ea17f46..7d68ac96 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -17,6 +17,7 @@ Because the number of images is smaller in the person keypoint subset of COCO, the number of epochs should be adapted so that we have the same number of iterations. """ + import datetime import os import time @@ -105,7 +106,7 @@ def main(args): train_sampler = InterruptableDistributedSampler(dataset) test_sampler = torch.utils.data.distributed.DistributedSampler(dataset_test, shuffle=False) - if args.aspect_ratio_group_factor >= 0: + if args.aspect_ratio_group_factor >= 0: # default == 3 group_ids = create_aspect_ratio_groups(dataset, k=args.aspect_ratio_group_factor) train_batch_sampler = GroupedBatchSampler(train_sampler, group_ids, args.batch_size) else: @@ -115,6 +116,7 @@ def main(args): if args.use_copypaste: if args.data_augmentation != "lsj": raise RuntimeError("SimpleCopyPaste algorithm currently only supports the 'lsj' data augmentation policies") + print("Using copypaste_collate_fn for train_collate_fn") train_collate_fn = copypaste_collate_fn data_loader = torch.utils.data.DataLoader( @@ -135,8 +137,8 @@ def main(args): model = torchvision.models.get_model( args.model, weights=args.weights, weights_backbone=args.weights_backbone, num_classes=num_classes, **kwargs ) - model.to(device) + if args.distributed and args.sync_bn: model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model) @@ -168,6 +170,7 @@ def main(args): scaler = torch.cuda.amp.GradScaler() if args.amp else None + ## OUTER LR_SCHEDULER args.lr_scheduler = args.lr_scheduler.lower() if args.lr_scheduler == "multisteplr": lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=args.lr_steps, gamma=args.lr_gamma) @@ -177,20 +180,29 @@ def main(args): raise RuntimeError( f"Invalid lr scheduler '{args.lr_scheduler}'. Only MultiStepLR and CosineAnnealingLR are supported." ) + + ## WARMUP LR_SCHEDULER + warmup_factor = 1.0 / 1000 + warmup_iters = min(1000, len(data_loader) - 1) + warmup_lr_scheduler = torch.optim.lr_scheduler.LinearLR( + optimizer, start_factor=warmup_factor, total_iters=warmup_iters + ) Path(args.resume).parent.mkdir(parents=True, exist_ok=True) ### ADDED THIS if args.resume and os.path.isfile(args.resume): ## EDITED THIS checkpoint = torch.load(args.resume, map_location="cpu") model_without_ddp.load_state_dict(checkpoint["model"]) optimizer.load_state_dict(checkpoint["optimizer"]) - if "lr_scheduler" in checkpoint: ## EDITED THIS - lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) ## EDITED THIS + lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) ## EDITED THIS + warmup_lr_scheduler.load_state_dict(checkpoint["warmup_lr_scheduler"]) ## ADDED THIS args.start_epoch = checkpoint["epoch"] # + 1 if args.amp: scaler.load_state_dict(checkpoint["scaler"]) - train_sampler.load_state_dict(checkpoint["sampler"]) + train_batch_sampler.sampler.load_state_dict(checkpoint["sampler"]) # INTERRUPTABLE SAMPLER IS MEMBER OF GROUPED BATCH SAMPLER if args.test_only: + # We disable the cudnn benchmarking because it can noticeably affect the accuracy + torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True evaluate(model, data_loader_test, device=device) return @@ -200,16 +212,18 @@ def main(args): for epoch in range(args.start_epoch, args.epochs): # if args.distributed: # train_sampler.set_epoch(epoch) - with train_sampler.in_epoch(epoch): - train_one_epoch(model, optimizer, data_loader, train_sampler, args, device, epoch, args.print_freq, scaler) - lr_scheduler.step() + with train_batch_sampler.sampler.in_epoch(epoch): + train_one_epoch(model, optimizer, data_loader, train_batch_sampler, lr_scheduler, warmup_lr_scheduler, args, device, epoch, args.print_freq, scaler) + lr_scheduler.step() # OUTER LR_SCHEDULER STEP EACH EPOCH if args.output_dir: checkpoint = { "model": model_without_ddp.state_dict(), "optimizer": optimizer.state_dict(), "lr_scheduler": lr_scheduler.state_dict(), + "warmup_lr_scheduler": warmup_lr_scheduler.state_dict(), "args": args, "epoch": epoch, + "sampler": train_batch_sampler.sampler.state_dict(), } if args.amp: checkpoint["scaler"] = scaler.state_dict() diff --git a/tv-segmentation/fcn_resnet101.isc b/tv-segmentation/fcn_resnet101.isc index 9c91f9cb..4d8a1afd 100644 --- a/tv-segmentation/fcn_resnet101.isc +++ b/tv-segmentation/fcn_resnet101.isc @@ -1,7 +1,7 @@ -experiment_name="seg-fcn_resnet50" +experiment_name="fcn_resnet101" gpu_type="24GB VRAM GPU" nnodes = 9 venv_path = "~/.venv/bin/activate" -output_path = "~/output_tv" +output_path = "~/output_fcn_resnet101" command="train_cycling.py --lr 0.0002 --dataset coco -b 8 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --epochs=1 --resume $OUTPUT_PATH/checkpoint.isc" diff --git a/tv-segmentation/prep.py b/tv-segmentation/prep.py index de0974a8..443d2005 100644 --- a/tv-segmentation/prep.py +++ b/tv-segmentation/prep.py @@ -1,5 +1,8 @@ import torchvision +from torchvision.models import resnet101, ResNet101_Weights weights = torchvision.models.get_weight('MobileNet_V3_Large_Weights.IMAGENET1K_V1') weights = torchvision.models.get_weight('ResNet101_Weights.IMAGENET1K_V1') +_ = resnet101(weights=ResNet101_Weights.IMAGENET1K_V1) + diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index 013a88d7..de68c6d1 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -123,7 +123,7 @@ def train_one_epoch(model, criterion, optimizer, data_loader, sampler: Interrupt sampler.advance(len(image)) step = sampler.progress // data_loader.batch_size - if utils.is_main_process() and step % 5 == 0: + if utils.is_main_process() and step % 1 == 0: print(f"Saving checkpoint at step {step}") checkpoint = { "model": model.module.state_dict(), @@ -251,6 +251,7 @@ def main(args): print(confmat) return + print("Starting training...") start_time = time.time() for epoch in range(args.start_epoch, args.epochs): # if args.distributed: @@ -287,7 +288,7 @@ def get_args_parser(add_help=True): parser.add_argument("--aux-loss", action="store_true", help="auxiliary loss") parser.add_argument("--device", default="cuda", type=str, help="device (Use cuda or cpu Default: cuda)") parser.add_argument( - "-b", "--batch-size", default=8, type=int, help="images per gpu, the total batch size is $NGPU x batch_size" + "-b", "--batch-size", default=2, type=int, help="images per gpu, the total batch size is $NGPU x batch_size" ) parser.add_argument("--epochs", default=30, type=int, metavar="N", help="number of total epochs to run") From a09c24b8ee42249be9575d16d2bcc27629a085b1 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Wed, 6 Sep 2023 04:26:13 +0000 Subject: [PATCH 03/44] all detection and segmentation working pending full epoch --- tv-segmentation/deeplabv3_mobilenet_v3_large.isc | 4 ++-- tv-segmentation/fcn_resnet101.isc | 2 +- tv-segmentation/prep.py | 3 ++- tv-segmentation/train_cycling.py | 15 +++++---------- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/tv-segmentation/deeplabv3_mobilenet_v3_large.isc b/tv-segmentation/deeplabv3_mobilenet_v3_large.isc index 5dfccfb1..44a53289 100644 --- a/tv-segmentation/deeplabv3_mobilenet_v3_large.isc +++ b/tv-segmentation/deeplabv3_mobilenet_v3_large.isc @@ -3,5 +3,5 @@ experiment_name="deeplabv3_mobilenet_v3_large" gpu_type="24GB VRAM GPU" nnodes = 9 venv_path = "~/.venv/bin/activate" -output_path = "~/output_tv" -command="train_cycling.py --dataset coco -b 4 --model deeplabv3_mobilenet_v3_large --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" +output_path = "~/output_deeplabv3_mobilenet_v3_large" +command="train_cycling.py --dataset coco -b 1 --model deeplabv3_mobilenet_v3_large --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" diff --git a/tv-segmentation/fcn_resnet101.isc b/tv-segmentation/fcn_resnet101.isc index 4d8a1afd..320f9257 100644 --- a/tv-segmentation/fcn_resnet101.isc +++ b/tv-segmentation/fcn_resnet101.isc @@ -4,4 +4,4 @@ gpu_type="24GB VRAM GPU" nnodes = 9 venv_path = "~/.venv/bin/activate" output_path = "~/output_fcn_resnet101" -command="train_cycling.py --lr 0.0002 --dataset coco -b 8 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --epochs=1 --resume $OUTPUT_PATH/checkpoint.isc" +command="train_cycling.py --lr 0.0002 --dataset coco -b 1 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --epochs=1 --resume $OUTPUT_PATH/checkpoint.isc" diff --git a/tv-segmentation/prep.py b/tv-segmentation/prep.py index 443d2005..6a39b7fc 100644 --- a/tv-segmentation/prep.py +++ b/tv-segmentation/prep.py @@ -1,8 +1,9 @@ import torchvision -from torchvision.models import resnet101, ResNet101_Weights +from torchvision.models import resnet101, ResNet101_Weights, mobilenet_v3_large, MobileNet_V3_Large_Weights weights = torchvision.models.get_weight('MobileNet_V3_Large_Weights.IMAGENET1K_V1') weights = torchvision.models.get_weight('ResNet101_Weights.IMAGENET1K_V1') _ = resnet101(weights=ResNet101_Weights.IMAGENET1K_V1) +_ = mobilenet_v3_large(weights=MobileNet_V3_Large_Weights.IMAGENET1K_V1) diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index de68c6d1..9c87358d 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -123,7 +123,7 @@ def train_one_epoch(model, criterion, optimizer, data_loader, sampler: Interrupt sampler.advance(len(image)) step = sampler.progress // data_loader.batch_size - if utils.is_main_process() and step % 1 == 0: + if utils.is_main_process() and step % 5 == 0: print(f"Saving checkpoint at step {step}") checkpoint = { "model": model.module.state_dict(), @@ -251,7 +251,7 @@ def main(args): print(confmat) return - print("Starting training...") + print("Starting training") start_time = time.time() for epoch in range(args.start_epoch, args.epochs): # if args.distributed: @@ -287,14 +287,9 @@ def get_args_parser(add_help=True): parser.add_argument("--model", default="fcn_resnet101", type=str, help="model name") parser.add_argument("--aux-loss", action="store_true", help="auxiliary loss") parser.add_argument("--device", default="cuda", type=str, help="device (Use cuda or cpu Default: cuda)") - parser.add_argument( - "-b", "--batch-size", default=2, type=int, help="images per gpu, the total batch size is $NGPU x batch_size" - ) + parser.add_argument("-b", "--batch-size", default=8, type=int, help="images per gpu, the total batch size is $NGPU x batch_size", dest="batch_size") parser.add_argument("--epochs", default=30, type=int, metavar="N", help="number of total epochs to run") - - parser.add_argument( - "-j", "--workers", default=16, type=int, metavar="N", help="number of data loading workers (default: 16)" - ) + parser.add_argument("-j", "--workers", default=16, type=int, metavar="N", help="number of data loading workers (default: 16)") parser.add_argument("--lr", default=0.01, type=float, help="initial learning rate") parser.add_argument("--momentum", default=0.9, type=float, metavar="M", help="momentum") parser.add_argument( @@ -323,7 +318,7 @@ def get_args_parser(add_help=True): "--use-deterministic-algorithms", action="store_true", help="Forces the use of deterministic algorithms only." ) # distributed training parameters - parser.add_argument("--world-size", default=1, type=int, help="number of distributed processes") + parser.add_argument("--world-size", default=9, type=int, help="number of distributed processes") parser.add_argument("--dist-url", default="env://", type=str, help="url used to set up distributed training") parser.add_argument("--weights", default=None, type=str, help="the weights enum name to load") From 7a3cb0cc318c4e806606e6a3184e5f877e47e023 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Wed, 6 Sep 2023 05:49:15 +0000 Subject: [PATCH 04/44] removed WIP from isc-demos README --- README.md | 8 ++++---- llama2-qlora/requirements.txt | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 llama2-qlora/requirements.txt diff --git a/README.md b/README.md index 3315710d..033d4096 100644 --- a/README.md +++ b/README.md @@ -64,12 +64,12 @@ isc clusters # view the status of the clusters (from https://github.com/pytorch/vision/tree/main/references/segmentation) -- WIP [fcn_resnet101.isc](./tv-segmentation/fcn_resnet101.isc) -- WIP [deeplabv3_mobilenet_v3_large.isc](./tv-segmentation/deeplabv3_mobilenet_v3_large.isc) +- [fcn_resnet101.isc](./tv-segmentation/fcn_resnet101.isc) +- [deeplabv3_mobilenet_v3_large.isc](./tv-segmentation/deeplabv3_mobilenet_v3_large.isc) ### tv-detection (from https://github.com/pytorch/vision/tree/main/references/detection) -- WIP [maskrcnn_resnet50_fpn.isc](./tv-detection/fasterrcnn_resnet50_fpn.isc) -- WIP [retinanet_resnet50_fpn.isc](./tv-detection/retinanet_resnet50_fpn.isc) +- [maskrcnn_resnet50_fpn.isc](./tv-detection/fasterrcnn_resnet50_fpn.isc) +- [retinanet_resnet50_fpn.isc](./tv-detection/retinanet_resnet50_fpn.isc) diff --git a/llama2-qlora/requirements.txt b/llama2-qlora/requirements.txt new file mode 100644 index 00000000..9deb35bf --- /dev/null +++ b/llama2-qlora/requirements.txt @@ -0,0 +1,8 @@ +torch +accelerate @ git+https://github.com/huggingface/accelerate.git +bitsandbytes +datasets==2.13.1 +transformers @ git+https://github.com/huggingface/transformers.git +peft @ git+https://github.com/huggingface/peft.git +trl @ git+https://github.com/lvwerra/trl.git +scipy \ No newline at end of file From ccbc0f69d414727057eb8b3b0f1dbb89992599c8 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Wed, 6 Sep 2023 06:06:19 +0000 Subject: [PATCH 05/44] Updated tv-detection and tv-segmentation to checkpoint (and report) after every batch --- tv-detection/engine.py | 2 +- tv-segmentation/train_cycling.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tv-detection/engine.py b/tv-detection/engine.py index ee6da38e..a94a7e90 100644 --- a/tv-detection/engine.py +++ b/tv-detection/engine.py @@ -60,7 +60,7 @@ def train_one_epoch(model, optimizer, data_loader, train_batch_sampler, lr_sched # ADDED THE FOLLOWING - INC NECESSARY ARGS TO TRAIN train_batch_sampler.sampler.advance(len(images)) step = train_batch_sampler.sampler.progress // args.batch_size - if utils.is_main_process() and step % 5 == 0: + if utils.is_main_process() and step % 1 == 0: print(f"Saving checkpoint at step {step}") checkpoint = { "model": model.module.state_dict(), diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index 9c87358d..041bd39b 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -123,7 +123,7 @@ def train_one_epoch(model, criterion, optimizer, data_loader, sampler: Interrupt sampler.advance(len(image)) step = sampler.progress // data_loader.batch_size - if utils.is_main_process() and step % 5 == 0: + if utils.is_main_process() and step % 1 == 0: print(f"Saving checkpoint at step {step}") checkpoint = { "model": model.module.state_dict(), From 98969742762e68e61186b9a5b60a943910aad04a Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Thu, 7 Sep 2023 04:04:35 +0000 Subject: [PATCH 06/44] Updated tv-detection to fix epoch roll-over error and added sophisticated timer function --- cycling_utils/cycling_utils/saving.py | 11 ++- tv-detection/engine.py | 72 +++++++++++++----- tv-detection/group_by_aspect_ratio.py | 4 +- tv-detection/train_cycling.py | 105 ++++++++++++++++++-------- 4 files changed, 136 insertions(+), 56 deletions(-) diff --git a/cycling_utils/cycling_utils/saving.py b/cycling_utils/cycling_utils/saving.py index a71fef99..0ca69743 100644 --- a/cycling_utils/cycling_utils/saving.py +++ b/cycling_utils/cycling_utils/saving.py @@ -2,8 +2,15 @@ import os import torch -def atomic_torch_save(obj, f: str | Path, **kwargs): +def atomic_torch_save(obj, f: str | Path, timer=None, **kwargs): f = str(f) temp_f = f + ".temp" torch.save(obj, temp_f, **kwargs) - os.replace(temp_f, f) \ No newline at end of file + if timer is not None: + timer.report(f'saving temp checkpoint') + os.replace(temp_f, f) + if timer is not None: + timer.report(f'replacing temp checkpoint with checkpoint') + return timer + else: + return \ No newline at end of file diff --git a/tv-detection/engine.py b/tv-detection/engine.py index a94a7e90..c31a26da 100644 --- a/tv-detection/engine.py +++ b/tv-detection/engine.py @@ -9,34 +9,38 @@ from coco_utils import get_coco_api_from_dataset from cycling_utils import InterruptableDistributedSampler, atomic_torch_save -def train_one_epoch(model, optimizer, data_loader, train_batch_sampler, lr_scheduler, warmup_lr_scheduler, args, device, epoch, print_freq, scaler=None): +def train_one_epoch(model, optimizer, data_loader, train_batch_sampler, lr_scheduler, warmup_lr_scheduler, args, device, epoch, print_freq, scaler=None, timer=None): + model.train() metric_logger = utils.MetricLogger(delimiter=" ") metric_logger.add_meter("lr", utils.SmoothedValue(window_size=1, fmt="{value}")) header = f"Epoch: [{epoch}]" - # # warmup_lr_scheduler will either be None if (starting from scratch or after epoch 0) or not None, in which case run with that. - # warmup_lr_scheduler = warmup_lr_scheduler if warmup_lr_scheduler else None - # if warmup_lr_scheduler is None and epoch == 0: - # warmup_factor = 1.0 / 1000 - # warmup_iters = min(1000, len(data_loader) - 1) - # warmup_lr_scheduler = torch.optim.lr_scheduler.LinearLR( - # optimizer, start_factor=warmup_factor, total_iters=warmup_iters - # ) + timer.report('training preliminaries') + + # Running this before starting the training loop assists reporting on progress after resuming - step == batch count + step = train_batch_sampler.sampler.progress // args.batch_size for images, targets in metric_logger.log_every(data_loader, train_batch_sampler.sampler.progress // args.batch_size, print_freq, header): ## EDITED THIS - ARGS.BATCH_SIZE == DATALOADER.BATCH_SIZE? GROUPEDBATCHSAMPLER AT PLAY + images = list(image.to(device) for image in images) targets = [{k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in t.items()} for t in targets] + + timer.report(f'Epoch: {epoch} Step {step}: moving batch data to device') + with torch.cuda.amp.autocast(enabled=scaler is not None): loss_dict = model(images, targets) losses = sum(loss for loss in loss_dict.values()) + timer.report(f'Epoch: {epoch} Step {step}: forward pass') + # reduce losses over all GPUs for logging purposes loss_dict_reduced = utils.reduce_dict(loss_dict) losses_reduced = sum(loss for loss in loss_dict_reduced.values()) - loss_value = losses_reduced.item() + timer.report(f'Epoch: {epoch} Step {step}: computing loss') + if not math.isfinite(loss_value): print(f"Loss is {loss_value}, stopping training") print(loss_dict_reduced) @@ -51,17 +55,23 @@ def train_one_epoch(model, optimizer, data_loader, train_batch_sampler, lr_sched losses.backward() optimizer.step() + timer.report(f'Epoch: {epoch} Step {step}: backward pass') + ## Always update warmup_lr_scheduler - once progressed past epoch 0, this will make no difference. warmup_lr_scheduler.step() metric_logger.update(loss=losses_reduced, **loss_dict_reduced) metric_logger.update(lr=optimizer.param_groups[0]["lr"]) + timer.report(f'Epoch: {epoch} Step {step}: updating metric logger') + # ADDED THE FOLLOWING - INC NECESSARY ARGS TO TRAIN train_batch_sampler.sampler.advance(len(images)) step = train_batch_sampler.sampler.progress // args.batch_size - if utils.is_main_process() and step % 1 == 0: - print(f"Saving checkpoint at step {step}") + + timer.report(f'Epoch: {epoch} Step {step}: advancing sampler and computing step') + + if utils.is_main_process() and step % 5 == 0: checkpoint = { "model": model.module.state_dict(), "optimizer": optimizer.state_dict(), @@ -71,13 +81,18 @@ def train_one_epoch(model, optimizer, data_loader, train_batch_sampler, lr_sched "args": args, "sampler": train_batch_sampler.sampler.state_dict(), } - if args.amp: checkpoint["scaler"] = scaler.state_dict() - atomic_torch_save(checkpoint, args.resume) - # END ADD + timer = atomic_torch_save(checkpoint, args.resume, timer) - return metric_logger + # # Simulating end of epoch + # if step >= 10: + # print("Simulating end of epoch") + # return metric_logger, timer + + # # END ADD + + return metric_logger, timer def _get_iou_types(model): @@ -93,7 +108,10 @@ def _get_iou_types(model): @torch.inference_mode() -def evaluate(model, data_loader, device): +def evaluate(model, data_loader, device, timer): + + timer.report('starting evaluation routine') + n_threads = torch.get_num_threads() # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) @@ -102,26 +120,37 @@ def evaluate(model, data_loader, device): metric_logger = utils.MetricLogger(delimiter=" ") header = "Test:" + timer.report(f'preliminaries') + coco = get_coco_api_from_dataset(data_loader.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) + timer.report(f'preparing coco evaluator') + + eval_batch = 1 for images, targets in metric_logger.log_every(data_loader, 100, header): + images = list(img.to(device) for img in images) + timer.report(f'eval batch: {eval_batch} moving to device') + if torch.cuda.is_available(): torch.cuda.synchronize() - model_time = time.time() outputs = model(images) + timer.report(f'eval batch: {eval_batch} forward through model') + outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] - model_time = time.time() - model_time + + timer.report(f'eval batch: {eval_batch} outputs back to cpu') res = {target["image_id"]: output for target, output in zip(targets, outputs)} evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) + timer.report(f'eval batch: {eval_batch} update evaluator') # gather the stats from all processes metric_logger.synchronize_between_processes() @@ -132,4 +161,7 @@ def evaluate(model, data_loader, device): coco_evaluator.accumulate() coco_evaluator.summarize() torch.set_num_threads(n_threads) - return coco_evaluator + + timer.report(f'evaluator accumulation and summarization') + + return coco_evaluator, timer diff --git a/tv-detection/group_by_aspect_ratio.py b/tv-detection/group_by_aspect_ratio.py index d12e14b5..7ed30b0b 100644 --- a/tv-detection/group_by_aspect_ratio.py +++ b/tv-detection/group_by_aspect_ratio.py @@ -185,9 +185,9 @@ def _quantize(x, bins): def create_aspect_ratio_groups(dataset, k=0): - aspect_ratios = compute_aspect_ratios(dataset) + aspect_ratios = compute_aspect_ratios(dataset) # list of aspect ratios for each image in the dataset bins = (2 ** np.linspace(-1, 1, 2 * k + 1)).tolist() if k > 0 else [1.0] - groups = _quantize(aspect_ratios, bins) + groups = _quantize(aspect_ratios, bins) # list of bin indexes to which each image belongs # count number of elements per group counts = np.unique(groups, return_counts=True)[1] fbins = [0] + bins + [np.inf] diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index 7d68ac96..32c5fb03 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -69,8 +69,22 @@ def get_transform(is_train, args): return lambda img, target: (trans(img), target) else: return presets.DetectionPresetEval(backend=args.backend, use_v2=args.use_v2) + +class Timer: + def __init__(self, start_time=None, running=0): + self.start_time = start_time if start_time is not None else time.time() + self.running = running + def report(self, annot): + now = time.time() + duration = now - self.start_time + self.running += duration + print("Completed {:<70}{:>12} milliseconds, {:>12} seconds total".format(annot, f'{1000*duration:,.3f}', f'{self.running:,.2f}')) + self.start_time = now def main(args): + + timer = Timer() + if args.backend.lower() == "tv_tensor" and not args.use_v2: raise ValueError("Use --use-v2 if you want to use the tv_tensor backend.") if args.dataset not in ("coco", "coco_kp"): @@ -91,12 +105,14 @@ def main(args): if args.use_deterministic_algorithms: torch.use_deterministic_algorithms(True) + timer.report('preliminaries') + # Data loading code - print("Loading data") dataset, num_classes = get_dataset(is_train=True, args=args) dataset_test, _ = get_dataset(is_train=False, args=args) - print("Creating data loaders") + timer.report('loading data') + # if args.distributed: # train_sampler = torch.utils.data.distributed.DistributedSampler(dataset) # test_sampler = torch.utils.data.distributed.DistributedSampler(dataset_test, shuffle=False) @@ -106,12 +122,16 @@ def main(args): train_sampler = InterruptableDistributedSampler(dataset) test_sampler = torch.utils.data.distributed.DistributedSampler(dataset_test, shuffle=False) + timer.report('creating data loaders') + if args.aspect_ratio_group_factor >= 0: # default == 3 group_ids = create_aspect_ratio_groups(dataset, k=args.aspect_ratio_group_factor) train_batch_sampler = GroupedBatchSampler(train_sampler, group_ids, args.batch_size) else: train_batch_sampler = torch.utils.data.BatchSampler(train_sampler, args.batch_size, drop_last=True) + timer.report('GroupedBatchSampler (data loaders 1)') + train_collate_fn = utils.collate_fn if args.use_copypaste: if args.data_augmentation != "lsj": @@ -126,7 +146,8 @@ def main(args): dataset_test, batch_size=1, sampler=test_sampler, num_workers=args.workers, collate_fn=utils.collate_fn ) - print("Creating model") + timer.report('data_loader, data_loader_test (data loaders 2)') + kwargs = {"trainable_backbone_layers": args.trainable_backbone_layers} if args.data_augmentation in ["multiscale", "lsj"]: kwargs["_skip_resize"] = True @@ -139,6 +160,8 @@ def main(args): ) model.to(device) + timer.report('creating model and .to(device)') + if args.distributed and args.sync_bn: model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model) @@ -154,6 +177,8 @@ def main(args): wd_groups = [args.norm_weight_decay, args.weight_decay] parameters = [{"params": p, "weight_decay": w} for p, w in zip(param_groups, wd_groups) if p] + timer.report('preparing model for distributed training') + opt_name = args.opt.lower() if opt_name.startswith("sgd"): optimizer = torch.optim.SGD( @@ -170,6 +195,8 @@ def main(args): scaler = torch.cuda.amp.GradScaler() if args.amp else None + timer.report('optimizer and scaler') + ## OUTER LR_SCHEDULER args.lr_scheduler = args.lr_scheduler.lower() if args.lr_scheduler == "multisteplr": @@ -188,6 +215,8 @@ def main(args): optimizer, start_factor=warmup_factor, total_iters=warmup_iters ) + timer.report('learning rate schedulers') + Path(args.resume).parent.mkdir(parents=True, exist_ok=True) ### ADDED THIS if args.resume and os.path.isfile(args.resume): ## EDITED THIS checkpoint = torch.load(args.resume, map_location="cpu") @@ -200,45 +229,57 @@ def main(args): scaler.load_state_dict(checkpoint["scaler"]) train_batch_sampler.sampler.load_state_dict(checkpoint["sampler"]) # INTERRUPTABLE SAMPLER IS MEMBER OF GROUPED BATCH SAMPLER + timer.report('retrieving checkpoint') + + # KILL THIS FOR NOW if args.test_only: # We disable the cudnn benchmarking because it can noticeably affect the accuracy torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True - evaluate(model, data_loader_test, device=device) + coco_evaluator, timer = evaluate(model, data_loader_test, device, timer) return - print("Start training") - start_time = time.time() for epoch in range(args.start_epoch, args.epochs): # if args.distributed: # train_sampler.set_epoch(epoch) with train_batch_sampler.sampler.in_epoch(epoch): - train_one_epoch(model, optimizer, data_loader, train_batch_sampler, lr_scheduler, warmup_lr_scheduler, args, device, epoch, args.print_freq, scaler) - lr_scheduler.step() # OUTER LR_SCHEDULER STEP EACH EPOCH - if args.output_dir: - checkpoint = { - "model": model_without_ddp.state_dict(), - "optimizer": optimizer.state_dict(), - "lr_scheduler": lr_scheduler.state_dict(), - "warmup_lr_scheduler": warmup_lr_scheduler.state_dict(), - "args": args, - "epoch": epoch, - "sampler": train_batch_sampler.sampler.state_dict(), - } - if args.amp: - checkpoint["scaler"] = scaler.state_dict() - # utils.save_on_master(checkpoint, os.path.join(args.output_dir, f"model_{epoch}.pth")) - # utils.save_on_master(checkpoint, os.path.join(args.output_dir, "checkpoint.pth")) - atomic_torch_save(checkpoint, args.resume) - - # # KILL THIS FOR NOW - # # evaluate after every epoch - # evaluate(model, data_loader_test, device=device) - - total_time = time.time() - start_time - total_time_str = str(datetime.timedelta(seconds=int(total_time))) - print(f"Training time {total_time_str}") + print('\n') + timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch + timer.report(f'launching epoch {epoch}') + metric_logger, timer = train_one_epoch(model, optimizer, data_loader, train_batch_sampler, lr_scheduler, warmup_lr_scheduler, args, device, epoch, args.print_freq, scaler, timer) + + timer.report(f'incrementing sampler epoch to {train_batch_sampler.sampler.epoch}') + lr_scheduler.step() # OUTER LR_SCHEDULER STEP EACH EPOCH + + timer.report(f'training for epoch {epoch}') + + if utils.is_main_process(): + checkpoint = { + "model": model_without_ddp.state_dict(), + "optimizer": optimizer.state_dict(), + "lr_scheduler": lr_scheduler.state_dict(), + "warmup_lr_scheduler": warmup_lr_scheduler.state_dict(), + "args": args, + "epoch": epoch, + "sampler": train_batch_sampler.sampler.state_dict(), + } + + if args.amp: + checkpoint["scaler"] = scaler.state_dict() + + timer.report(f'defining epoch {epoch} checkpoint') + + # utils.save_on_master(checkpoint, os.path.join(args.output_dir, f"model_{epoch}.pth")) + # utils.save_on_master(checkpoint, os.path.join(args.output_dir, "checkpoint.pth")) + timer = atomic_torch_save(checkpoint, args.resume, timer) + + # KILL THIS FOR NOW + # evaluate after every epoch + coco_evaluator, timer = evaluate(model, data_loader_test, device, timer) + + # Restart the timer + timer = Timer() def get_args_parser(add_help=True): import argparse @@ -261,7 +302,7 @@ def get_args_parser(add_help=True): parser.add_argument("--lr-step-size", default=8, type=int, help="decrease lr every step-size epochs (multisteplr scheduler only)") parser.add_argument("--lr-steps",default=[16, 22],nargs="+",type=int,help="decrease lr every step-size epochs (multisteplr scheduler only)") parser.add_argument("--lr-gamma", default=0.1, type=float, help="decrease lr by a factor of lr-gamma (multisteplr scheduler only)") - parser.add_argument("--print-freq", default=20, type=int, help="print frequency") + parser.add_argument("--print-freq", default=1, type=int, help="print frequency") parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") parser.add_argument("--resume", default="", type=str, help="path of checkpoint") parser.add_argument("--start_epoch", default=0, type=int, help="start epoch") From cea7e8b102c823a6ef1dc5f2ebd1245147f63e56 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Thu, 7 Sep 2023 04:31:19 +0000 Subject: [PATCH 07/44] Updates --- tv-detection/engine.py | 2 + tv-detection/train_cycling.py | 17 ++--- tv-segmentation/train_cycling.py | 104 ++++++++++++++++++++++++++----- 3 files changed, 95 insertions(+), 28 deletions(-) diff --git a/tv-detection/engine.py b/tv-detection/engine.py index c31a26da..319b57c4 100644 --- a/tv-detection/engine.py +++ b/tv-detection/engine.py @@ -152,6 +152,8 @@ def evaluate(model, data_loader, device, timer): metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) timer.report(f'eval batch: {eval_batch} update evaluator') + eval_batch += 1 + # gather the stats from all processes metric_logger.synchronize_between_processes() print("Averaged stats:", metric_logger) diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index 32c5fb03..6100aafa 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -122,7 +122,7 @@ def main(args): train_sampler = InterruptableDistributedSampler(dataset) test_sampler = torch.utils.data.distributed.DistributedSampler(dataset_test, shuffle=False) - timer.report('creating data loaders') + timer.report('creating data samplers') if args.aspect_ratio_group_factor >= 0: # default == 3 group_ids = create_aspect_ratio_groups(dataset, k=args.aspect_ratio_group_factor) @@ -130,7 +130,7 @@ def main(args): else: train_batch_sampler = torch.utils.data.BatchSampler(train_sampler, args.batch_size, drop_last=True) - timer.report('GroupedBatchSampler (data loaders 1)') + timer.report('creating GroupedBatchSampler') train_collate_fn = utils.collate_fn if args.use_copypaste: @@ -146,7 +146,7 @@ def main(args): dataset_test, batch_size=1, sampler=test_sampler, num_workers=args.workers, collate_fn=utils.collate_fn ) - timer.report('data_loader, data_loader_test (data loaders 2)') + timer.report('creating data loaders') kwargs = {"trainable_backbone_layers": args.trainable_backbone_layers} if args.data_augmentation in ["multiscale", "lsj"]: @@ -248,8 +248,7 @@ def main(args): timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch timer.report(f'launching epoch {epoch}') metric_logger, timer = train_one_epoch(model, optimizer, data_loader, train_batch_sampler, lr_scheduler, warmup_lr_scheduler, args, device, epoch, args.print_freq, scaler, timer) - - timer.report(f'incrementing sampler epoch to {train_batch_sampler.sampler.epoch}') + lr_scheduler.step() # OUTER LR_SCHEDULER STEP EACH EPOCH timer.report(f'training for epoch {epoch}') @@ -274,12 +273,8 @@ def main(args): # utils.save_on_master(checkpoint, os.path.join(args.output_dir, "checkpoint.pth")) timer = atomic_torch_save(checkpoint, args.resume, timer) - # KILL THIS FOR NOW - # evaluate after every epoch - coco_evaluator, timer = evaluate(model, data_loader_test, device, timer) - - # Restart the timer - timer = Timer() + # KILL THIS FOR NOW + coco_evaluator, timer = evaluate(model, data_loader_test, device, timer) def get_args_parser(add_help=True): import argparse diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index 041bd39b..e5e1fab8 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -15,6 +15,16 @@ from torchvision.transforms import functional as F, InterpolationMode from cycling_utils import InterruptableDistributedSampler, atomic_torch_save +class Timer: + def __init__(self, start_time=None, running=0): + self.start_time = start_time if start_time is not None else time.time() + self.running = running + def report(self, annot): + now = time.time() + duration = now - self.start_time + self.running += duration + print("Completed {:<70}{:>12} milliseconds, {:>12} seconds total".format(annot, f'{1000*duration:,.3f}', f'{self.running:,.2f}')) + self.start_time = now def get_dataset(dir_path, name, image_set, transform): def sbd(*args, **kwargs): @@ -60,26 +70,44 @@ def criterion(inputs, target): return losses["out"] + 0.5 * losses["aux"] -def evaluate(model, data_loader, device, num_classes): +def evaluate(model, data_loader, device, num_classes, timer): + + timer.report('starting evaluation routine') + model.eval() confmat = utils.ConfusionMatrix(num_classes) metric_logger = utils.MetricLogger(delimiter=" ") header = "Test:" num_processed_samples = 0 + + timer.report(f'preliminaries') + with torch.inference_mode(): + eval_batch = 1 for image, target in metric_logger.log_every(data_loader, 100, header): image, target = image.to(device), target.to(device) + + timer.report(f'eval batch: {eval_batch} moving to device') + output = model(image) output = output["out"] + timer.report(f'eval batch: {eval_batch} forward through model') + confmat.update(target.flatten(), output.argmax(1).flatten()) # FIXME need to take into account that the datasets # could have been padded in distributed setup num_processed_samples += image.shape[0] + timer.report(f'eval batch: {eval_batch} confmat update') + + eval_batch += 1 + confmat.reduce_from_all_processes() num_processed_samples = utils.reduce_across_processes(num_processed_samples) + + timer.report(f'confmat reduction accross processes') if ( hasattr(data_loader.dataset, "__len__") and len(data_loader.dataset) != num_processed_samples @@ -93,20 +121,29 @@ def evaluate(model, data_loader, device, num_classes): "Setting the world size to 1 is always a safe bet." ) - return confmat + return confmat, timer -def train_one_epoch(model, criterion, optimizer, data_loader, sampler: InterruptableDistributedSampler, lr_scheduler, device, epoch, print_freq, scaler=None): +def train_one_epoch(model, criterion, optimizer, data_loader, sampler: InterruptableDistributedSampler, lr_scheduler, device, epoch, print_freq, scaler=None, timer=None): + model.train() metric_logger = utils.MetricLogger(delimiter=" ") metric_logger.add_meter("lr", utils.SmoothedValue(window_size=1, fmt="{value}")) header = f"Epoch: [{epoch}]" + + timer.report('training preliminaries') + for image, target in metric_logger.log_every(data_loader, sampler.progress // data_loader.batch_size, print_freq, header): image, target = image.to(device), target.to(device) + + timer.report(f'Epoch: {epoch} Step {step}: moving batch data to device') + with torch.cuda.amp.autocast(enabled=scaler is not None): output = model(image) loss = criterion(output, target) + timer.report(f'Epoch: {epoch} Step {step}: forward pass') + optimizer.zero_grad() if scaler is not None: scaler.scale(loss).backward() @@ -116,14 +153,16 @@ def train_one_epoch(model, criterion, optimizer, data_loader, sampler: Interrupt loss.backward() optimizer.step() - lr_scheduler.step() + timer.report(f'Epoch: {epoch} Step {step}: backward pass') + lr_scheduler.step() metric_logger.update(loss=loss.item(), lr=optimizer.param_groups[0]["lr"]) - sampler.advance(len(image)) + timer.report(f'Epoch: {epoch} Step {step}: updating metric logger') + step = sampler.progress // data_loader.batch_size - if utils.is_main_process() and step % 1 == 0: + if utils.is_main_process() and step % 5 == 0: print(f"Saving checkpoint at step {step}") checkpoint = { "model": model.module.state_dict(), @@ -135,10 +174,15 @@ def train_one_epoch(model, criterion, optimizer, data_loader, sampler: Interrupt } if args.amp: checkpoint["scaler"] = scaler.state_dict() - atomic_torch_save(checkpoint, args.resume) + timer = atomic_torch_save(checkpoint, args.resume, timer) + + return metric_logger, timer def main(args): + + timer = Timer() + if args.output_dir: utils.mkdir(args.output_dir) @@ -155,9 +199,13 @@ def main(args): else: torch.backends.cudnn.benchmark = True + timer.report('preliminaries') + dataset, num_classes = get_dataset(args.data_path, args.dataset, "train", get_transform(True, args)) dataset_test, _ = get_dataset(args.data_path, args.dataset, "val", get_transform(False, args)) + timer.report('loading data') + # if args.distributed: train_sampler = InterruptableDistributedSampler(dataset) test_sampler = torch.utils.data.distributed.DistributedSampler(dataset_test, shuffle=False) @@ -165,6 +213,8 @@ def main(args): # train_sampler = torch.utils.data.RandomSampler(dataset) # test_sampler = torch.utils.data.SequentialSampler(dataset_test) + timer.report('creating data samplers') + data_loader = torch.utils.data.DataLoader( dataset, batch_size=args.batch_size, @@ -178,6 +228,8 @@ def main(args): dataset_test, batch_size=1, sampler=test_sampler, num_workers=args.workers, collate_fn=utils.collate_fn ) + timer.report('creating data loaders') + model = torchvision.models.get_model( args.model, weights=args.weights, @@ -186,6 +238,9 @@ def main(args): aux_loss=args.aux_loss, ) model.to(device) + + timer.report('creating model and .to(device)') + if args.distributed: model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model) @@ -198,6 +253,9 @@ def main(args): {"params": [p for p in model_without_ddp.backbone.parameters() if p.requires_grad]}, {"params": [p for p in model_without_ddp.classifier.parameters() if p.requires_grad]}, ] + + timer.report('preparing model for distributed training') + if args.aux_loss: params = [p for p in model_without_ddp.aux_classifier.parameters() if p.requires_grad] params_to_optimize.append({"params": params, "lr": args.lr * 10}) @@ -205,6 +263,8 @@ def main(args): scaler = torch.cuda.amp.GradScaler() if args.amp else None + timer.report('optimizer and scaler') + iters_per_epoch = len(data_loader) main_lr_scheduler = PolynomialLR( optimizer, total_iters=iters_per_epoch * (args.epochs - args.lr_warmup_epochs), power=0.9 @@ -231,6 +291,8 @@ def main(args): else: lr_scheduler = main_lr_scheduler + timer.report('learning rate schedulers') + Path(args.resume).parent.mkdir(parents=True, exist_ok=True) if args.resume and os.path.isfile(args.resume): checkpoint = torch.load(args.resume, map_location="cpu") @@ -243,22 +305,28 @@ def main(args): scaler.load_state_dict(checkpoint["scaler"]) train_sampler.load_state_dict(checkpoint["sampler"]) + timer.report('retrieving checkpoint') + if args.test_only: # We disable the cudnn benchmarking because it can noticeably affect the accuracy torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True - confmat = evaluate(model, data_loader_test, device=device, num_classes=num_classes) + confmat, timer = evaluate(model, data_loader_test, device=device, num_classes=num_classes, timer=timer) print(confmat) return - print("Starting training") - start_time = time.time() for epoch in range(args.start_epoch, args.epochs): # if args.distributed: with train_sampler.in_epoch(epoch): - train_one_epoch(model, criterion, optimizer, data_loader, train_sampler, lr_scheduler, device, epoch, args.print_freq, scaler) - confmat = evaluate(model, data_loader_test, device=device, num_classes=num_classes) - print(confmat) + + print('\n') + timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch + timer.report(f'launching epoch {epoch}') + + metric_logger, timer = train_one_epoch(model, criterion, optimizer, data_loader, train_sampler, lr_scheduler, device, epoch, args.print_freq, scaler, timer) + + timer.report(f'training for epoch {epoch}') + if utils.is_main_process(): checkpoint = { "model": model.module.state_dict(), @@ -270,12 +338,14 @@ def main(args): } if args.amp: checkpoint["scaler"] = scaler.state_dict() - atomic_torch_save(checkpoint, args.resume) - total_time = time.time() - start_time - total_time_str = str(datetime.timedelta(seconds=int(total_time))) - print(f"Training time {total_time_str}") + timer.report(f'defining epoch {epoch} checkpoint') + timer = atomic_torch_save(checkpoint, args.resume, timer) + + confmat, timer = evaluate(model, data_loader_test, device=device, num_classes=num_classes, timer=timer) + print(confmat) + def get_args_parser(add_help=True): import argparse From 7ce8cb4b9d1403b468bf1199217c9390057eb521 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Thu, 7 Sep 2023 04:39:50 +0000 Subject: [PATCH 08/44] updates --- tv-detection/engine.py | 2 +- tv-segmentation/train_cycling.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tv-detection/engine.py b/tv-detection/engine.py index 319b57c4..6b03e991 100644 --- a/tv-detection/engine.py +++ b/tv-detection/engine.py @@ -71,7 +71,7 @@ def train_one_epoch(model, optimizer, data_loader, train_batch_sampler, lr_sched timer.report(f'Epoch: {epoch} Step {step}: advancing sampler and computing step') - if utils.is_main_process() and step % 5 == 0: + if utils.is_main_process() and step % 5 == 0: # Checkpointing every 5 batches? checkpoint = { "model": model.module.state_dict(), "optimizer": optimizer.state_dict(), diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index e5e1fab8..40166760 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -133,7 +133,10 @@ def train_one_epoch(model, criterion, optimizer, data_loader, sampler: Interrupt timer.report('training preliminaries') - for image, target in metric_logger.log_every(data_loader, sampler.progress // data_loader.batch_size, print_freq, header): + # Running this before starting the training loop assists reporting on progress after resuming - step == batch count + step = sampler.progress // data_loader.batch_size + + for image, target in metric_logger.log_every(data_loader, step, print_freq, header): image, target = image.to(device), target.to(device) timer.report(f'Epoch: {epoch} Step {step}: moving batch data to device') @@ -162,7 +165,7 @@ def train_one_epoch(model, criterion, optimizer, data_loader, sampler: Interrupt timer.report(f'Epoch: {epoch} Step {step}: updating metric logger') step = sampler.progress // data_loader.batch_size - if utils.is_main_process() and step % 5 == 0: + if utils.is_main_process() and step % 5 == 0: # Checkpointing every 5 batches? print(f"Saving checkpoint at step {step}") checkpoint = { "model": model.module.state_dict(), @@ -345,7 +348,7 @@ def main(args): confmat, timer = evaluate(model, data_loader_test, device=device, num_classes=num_classes, timer=timer) print(confmat) - + def get_args_parser(add_help=True): import argparse From ba815e8ad131a4498b15906851e46a8072823e52 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Thu, 7 Sep 2023 04:41:56 +0000 Subject: [PATCH 09/44] updates --- tv-segmentation/train_cycling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index 40166760..969724f0 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -377,7 +377,7 @@ def get_args_parser(add_help=True): parser.add_argument("--lr-warmup-epochs", default=0, type=int, help="the number of epochs to warmup (default: 0)") parser.add_argument("--lr-warmup-method", default="linear", type=str, help="the warmup method (default: linear)") parser.add_argument("--lr-warmup-decay", default=0.01, type=float, help="the decay for lr") - parser.add_argument("--print-freq", default=10, type=int, help="print frequency") + parser.add_argument("--print-freq", default=1, type=int, help="print frequency") parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") parser.add_argument("--resume", type=str, help="path of checkpoint", required=True) parser.add_argument("--start-epoch", default=0, type=int, metavar="N", help="start epoch") From 3539a17a04fe5613f636416ed1d989dbe4f5dd92 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Thu, 7 Sep 2023 04:53:06 +0000 Subject: [PATCH 10/44] Added Timer to cycling_utils/saving --- cycling_utils/cycling_utils/__init__.py | 4 ++-- cycling_utils/cycling_utils/saving.py | 14 +++++++++++++- tv-detection/train_cycling.py | 13 +------------ tv-segmentation/train_cycling.py | 13 +------------ 4 files changed, 17 insertions(+), 27 deletions(-) diff --git a/cycling_utils/cycling_utils/__init__.py b/cycling_utils/cycling_utils/__init__.py index bb9303d4..4255b19c 100644 --- a/cycling_utils/cycling_utils/__init__.py +++ b/cycling_utils/cycling_utils/__init__.py @@ -1,4 +1,4 @@ -from .saving import atomic_torch_save +from .saving import atomic_torch_save, Timer from .sampler import InterruptableDistributedSampler -__all__ = ["InterruptableDistributedSampler", "atomic_torch_save"] \ No newline at end of file +__all__ = ["InterruptableDistributedSampler", "atomic_torch_save", "Timer"] \ No newline at end of file diff --git a/cycling_utils/cycling_utils/saving.py b/cycling_utils/cycling_utils/saving.py index 0ca69743..cd1c5f12 100644 --- a/cycling_utils/cycling_utils/saving.py +++ b/cycling_utils/cycling_utils/saving.py @@ -1,6 +1,7 @@ from pathlib import Path import os import torch +import time def atomic_torch_save(obj, f: str | Path, timer=None, **kwargs): f = str(f) @@ -13,4 +14,15 @@ def atomic_torch_save(obj, f: str | Path, timer=None, **kwargs): timer.report(f'replacing temp checkpoint with checkpoint') return timer else: - return \ No newline at end of file + return + +class Timer: + def __init__(self, start_time=None, running=0): + self.start_time = start_time if start_time is not None else time.time() + self.running = running + def report(self, annot): + now = time.time() + duration = now - self.start_time + self.running += duration + print("Completed {:<70}{:>12} milliseconds, {:>12} seconds total".format(annot, f'{1000*duration:,.3f}', f'{self.running:,.2f}')) + self.start_time = now \ No newline at end of file diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index 6100aafa..6d509e06 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -38,7 +38,7 @@ from torchvision.transforms import InterpolationMode from transforms import SimpleCopyPaste -from cycling_utils import InterruptableDistributedSampler, atomic_torch_save +from cycling_utils import InterruptableDistributedSampler, atomic_torch_save, Timer def copypaste_collate_fn(batch): copypaste = SimpleCopyPaste(blending=True, resize_interpolation=InterpolationMode.BILINEAR) @@ -69,17 +69,6 @@ def get_transform(is_train, args): return lambda img, target: (trans(img), target) else: return presets.DetectionPresetEval(backend=args.backend, use_v2=args.use_v2) - -class Timer: - def __init__(self, start_time=None, running=0): - self.start_time = start_time if start_time is not None else time.time() - self.running = running - def report(self, annot): - now = time.time() - duration = now - self.start_time - self.running += duration - print("Completed {:<70}{:>12} milliseconds, {:>12} seconds total".format(annot, f'{1000*duration:,.3f}', f'{self.running:,.2f}')) - self.start_time = now def main(args): diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index 969724f0..6c101155 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -13,18 +13,7 @@ from torch import nn from torch.optim.lr_scheduler import PolynomialLR from torchvision.transforms import functional as F, InterpolationMode -from cycling_utils import InterruptableDistributedSampler, atomic_torch_save - -class Timer: - def __init__(self, start_time=None, running=0): - self.start_time = start_time if start_time is not None else time.time() - self.running = running - def report(self, annot): - now = time.time() - duration = now - self.start_time - self.running += duration - print("Completed {:<70}{:>12} milliseconds, {:>12} seconds total".format(annot, f'{1000*duration:,.3f}', f'{self.running:,.2f}')) - self.start_time = now +from cycling_utils import InterruptableDistributedSampler, atomic_torch_save, Timer def get_dataset(dir_path, name, image_set, transform): def sbd(*args, **kwargs): From b61aec582f5a2228dd9eb845e4adc0b5d2f6bfb9 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Thu, 7 Sep 2023 05:03:13 +0000 Subject: [PATCH 11/44] moved Timer to its own file, updated training scripts to time imports --- cycling_utils/cycling_utils/__init__.py | 3 ++- cycling_utils/cycling_utils/saving.py | 14 +------------- cycling_utils/cycling_utils/timer.py | 12 ++++++++++++ tv-detection/maskrcnn_resnet50_fpn.isc | 2 +- tv-detection/retinanet_resnet50_fpn.isc | 2 +- tv-detection/train_cycling.py | 14 ++++++++++---- tv-segmentation/deeplabv3_mobilenet_v3_large.isc | 2 +- tv-segmentation/fcn_resnet101.isc | 2 +- tv-segmentation/train_cycling.py | 10 ++++++++-- 9 files changed, 37 insertions(+), 24 deletions(-) create mode 100644 cycling_utils/cycling_utils/timer.py diff --git a/cycling_utils/cycling_utils/__init__.py b/cycling_utils/cycling_utils/__init__.py index 4255b19c..ecff98ef 100644 --- a/cycling_utils/cycling_utils/__init__.py +++ b/cycling_utils/cycling_utils/__init__.py @@ -1,4 +1,5 @@ -from .saving import atomic_torch_save, Timer +from .timer import Timer +from .saving import atomic_torch_save from .sampler import InterruptableDistributedSampler __all__ = ["InterruptableDistributedSampler", "atomic_torch_save", "Timer"] \ No newline at end of file diff --git a/cycling_utils/cycling_utils/saving.py b/cycling_utils/cycling_utils/saving.py index cd1c5f12..0ca69743 100644 --- a/cycling_utils/cycling_utils/saving.py +++ b/cycling_utils/cycling_utils/saving.py @@ -1,7 +1,6 @@ from pathlib import Path import os import torch -import time def atomic_torch_save(obj, f: str | Path, timer=None, **kwargs): f = str(f) @@ -14,15 +13,4 @@ def atomic_torch_save(obj, f: str | Path, timer=None, **kwargs): timer.report(f'replacing temp checkpoint with checkpoint') return timer else: - return - -class Timer: - def __init__(self, start_time=None, running=0): - self.start_time = start_time if start_time is not None else time.time() - self.running = running - def report(self, annot): - now = time.time() - duration = now - self.start_time - self.running += duration - print("Completed {:<70}{:>12} milliseconds, {:>12} seconds total".format(annot, f'{1000*duration:,.3f}', f'{self.running:,.2f}')) - self.start_time = now \ No newline at end of file + return \ No newline at end of file diff --git a/cycling_utils/cycling_utils/timer.py b/cycling_utils/cycling_utils/timer.py new file mode 100644 index 00000000..da7a5eb2 --- /dev/null +++ b/cycling_utils/cycling_utils/timer.py @@ -0,0 +1,12 @@ +import time + +class Timer: + def __init__(self, start_time=None, running=0): + self.start_time = start_time if start_time is not None else time.time() + self.running = running + def report(self, annot): + now = time.time() + duration = now - self.start_time + self.running += duration + print("Completed {:<70}{:>12} milliseconds, {:>12} seconds total".format(annot, f'{1000*duration:,.3f}', f'{self.running:,.2f}')) + self.start_time = now \ No newline at end of file diff --git a/tv-detection/maskrcnn_resnet50_fpn.isc b/tv-detection/maskrcnn_resnet50_fpn.isc index f640afe0..3c247e41 100644 --- a/tv-detection/maskrcnn_resnet50_fpn.isc +++ b/tv-detection/maskrcnn_resnet50_fpn.isc @@ -3,4 +3,4 @@ gpu_type="24GB VRAM GPU" nnodes = 9 venv_path = "~/.venv/bin/activate" output_path = "~/output_maskrcnn_resnet50_fpn" -command="train_cycling.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file +command="train_cycling.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 --b 4 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/tv-detection/retinanet_resnet50_fpn.isc b/tv-detection/retinanet_resnet50_fpn.isc index fe195144..2d29d1ad 100644 --- a/tv-detection/retinanet_resnet50_fpn.isc +++ b/tv-detection/retinanet_resnet50_fpn.isc @@ -3,5 +3,5 @@ gpu_type="24GB VRAM GPU" nnodes = 9 venv_path = "~/.venv/bin/activate" output_path = "~/output_retinanet_resnet50_fpn" -command="train_cycling.py --dataset coco --model retinanet_resnet50_fpn --epochs 26 --lr-steps 16 22 --aspect-ratio-group-factor 3 --lr 0.01 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" +command="train_cycling.py --dataset coco --model retinanet_resnet50_fpn --epochs 26 --lr-steps 16 22 --b 4 --aspect-ratio-group-factor 3 --lr 0.01 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index 6d509e06..7618f09b 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -17,6 +17,10 @@ Because the number of images is smaller in the person keypoint subset of COCO, the number of epochs should be adapted so that we have the same number of iterations. """ +from cycling_utils import Timer + +timer = Timer() +timer.report('importing Timer') import datetime import os @@ -38,7 +42,9 @@ from torchvision.transforms import InterpolationMode from transforms import SimpleCopyPaste -from cycling_utils import InterruptableDistributedSampler, atomic_torch_save, Timer +from cycling_utils import InterruptableDistributedSampler, atomic_torch_save + +timer.report('importing everything else') def copypaste_collate_fn(batch): copypaste = SimpleCopyPaste(blending=True, resize_interpolation=InterpolationMode.BILINEAR) @@ -69,11 +75,11 @@ def get_transform(is_train, args): return lambda img, target: (trans(img), target) else: return presets.DetectionPresetEval(backend=args.backend, use_v2=args.use_v2) + +timer.report('defined other functions') def main(args): - timer = Timer() - if args.backend.lower() == "tv_tensor" and not args.use_v2: raise ValueError("Use --use-v2 if you want to use the tv_tensor backend.") if args.dataset not in ("coco", "coco_kp"): @@ -94,7 +100,7 @@ def main(args): if args.use_deterministic_algorithms: torch.use_deterministic_algorithms(True) - timer.report('preliminaries') + timer.report('main preliminaries') # Data loading code dataset, num_classes = get_dataset(is_train=True, args=args) diff --git a/tv-segmentation/deeplabv3_mobilenet_v3_large.isc b/tv-segmentation/deeplabv3_mobilenet_v3_large.isc index 44a53289..53ea47a0 100644 --- a/tv-segmentation/deeplabv3_mobilenet_v3_large.isc +++ b/tv-segmentation/deeplabv3_mobilenet_v3_large.isc @@ -4,4 +4,4 @@ gpu_type="24GB VRAM GPU" nnodes = 9 venv_path = "~/.venv/bin/activate" output_path = "~/output_deeplabv3_mobilenet_v3_large" -command="train_cycling.py --dataset coco -b 1 --model deeplabv3_mobilenet_v3_large --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" +command="train_cycling.py --dataset coco -b 2 --model deeplabv3_mobilenet_v3_large --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" diff --git a/tv-segmentation/fcn_resnet101.isc b/tv-segmentation/fcn_resnet101.isc index 320f9257..393d46bb 100644 --- a/tv-segmentation/fcn_resnet101.isc +++ b/tv-segmentation/fcn_resnet101.isc @@ -4,4 +4,4 @@ gpu_type="24GB VRAM GPU" nnodes = 9 venv_path = "~/.venv/bin/activate" output_path = "~/output_fcn_resnet101" -command="train_cycling.py --lr 0.0002 --dataset coco -b 1 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --epochs=1 --resume $OUTPUT_PATH/checkpoint.isc" +command="train_cycling.py --lr 0.0002 --dataset coco -b 2 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --epochs=1 --resume $OUTPUT_PATH/checkpoint.isc" diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index 6c101155..3823f25d 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -1,3 +1,8 @@ +from cycling_utils import Timer + +timer = Timer() +timer.report('importing Timer') + import datetime import os import time @@ -15,6 +20,8 @@ from torchvision.transforms import functional as F, InterpolationMode from cycling_utils import InterruptableDistributedSampler, atomic_torch_save, Timer +timer.report('importing everything else') + def get_dataset(dir_path, name, image_set, transform): def sbd(*args, **kwargs): return torchvision.datasets.SBDataset(*args, mode="segmentation", **kwargs) @@ -170,11 +177,10 @@ def train_one_epoch(model, criterion, optimizer, data_loader, sampler: Interrupt return metric_logger, timer +timer.report('defined other functions') def main(args): - timer = Timer() - if args.output_dir: utils.mkdir(args.output_dir) From 228d6e35641c3606758523a9c37eed7363e2fe09 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Thu, 7 Sep 2023 06:31:40 +0000 Subject: [PATCH 12/44] Updated timer to only report if the global rank of the process is == 0 for legible reporting on imports --- cycling_utils/cycling_utils/timer.py | 12 +++++++----- tv-detection/maskrcnn_resnet50_fpn.isc | 2 +- tv-detection/retinanet_resnet50_fpn.isc | 2 +- tv-detection/train_cycling.py | 4 ++-- tv-segmentation/train_cycling.py | 4 ++-- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/cycling_utils/cycling_utils/timer.py b/cycling_utils/cycling_utils/timer.py index da7a5eb2..290d9bd6 100644 --- a/cycling_utils/cycling_utils/timer.py +++ b/cycling_utils/cycling_utils/timer.py @@ -1,12 +1,14 @@ import time +import os class Timer: def __init__(self, start_time=None, running=0): self.start_time = start_time if start_time is not None else time.time() self.running = running def report(self, annot): - now = time.time() - duration = now - self.start_time - self.running += duration - print("Completed {:<70}{:>12} milliseconds, {:>12} seconds total".format(annot, f'{1000*duration:,.3f}', f'{self.running:,.2f}')) - self.start_time = now \ No newline at end of file + if str(os.environ["RANK"]) == "0": + now = time.time() + duration = now - self.start_time + self.running += duration + print("Completed {:<70}{:>12} milliseconds, {:>12} seconds total".format(annot, f'{1000*duration:,.3f}', f'{self.running:,.2f}')) + self.start_time = now \ No newline at end of file diff --git a/tv-detection/maskrcnn_resnet50_fpn.isc b/tv-detection/maskrcnn_resnet50_fpn.isc index 3c247e41..89858aca 100644 --- a/tv-detection/maskrcnn_resnet50_fpn.isc +++ b/tv-detection/maskrcnn_resnet50_fpn.isc @@ -3,4 +3,4 @@ gpu_type="24GB VRAM GPU" nnodes = 9 venv_path = "~/.venv/bin/activate" output_path = "~/output_maskrcnn_resnet50_fpn" -command="train_cycling.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 --b 4 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file +command="train_cycling.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 4 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/tv-detection/retinanet_resnet50_fpn.isc b/tv-detection/retinanet_resnet50_fpn.isc index 2d29d1ad..e70607a2 100644 --- a/tv-detection/retinanet_resnet50_fpn.isc +++ b/tv-detection/retinanet_resnet50_fpn.isc @@ -3,5 +3,5 @@ gpu_type="24GB VRAM GPU" nnodes = 9 venv_path = "~/.venv/bin/activate" output_path = "~/output_retinanet_resnet50_fpn" -command="train_cycling.py --dataset coco --model retinanet_resnet50_fpn --epochs 26 --lr-steps 16 22 --b 4 --aspect-ratio-group-factor 3 --lr 0.01 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" +command="train_cycling.py --dataset coco --model retinanet_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 4 --aspect-ratio-group-factor 3 --lr 0.01 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index 7618f09b..c9771d2a 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -78,7 +78,7 @@ def get_transform(is_train, args): timer.report('defined other functions') -def main(args): +def main(args, timer): if args.backend.lower() == "tv_tensor" and not args.use_v2: raise ValueError("Use --use-v2 if you want to use the tv_tensor backend.") @@ -324,4 +324,4 @@ def get_args_parser(add_help=True): if __name__ == "__main__": args = get_args_parser().parse_args() - main(args) + main(args, timer) diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index 3823f25d..82281d47 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -179,7 +179,7 @@ def train_one_epoch(model, criterion, optimizer, data_loader, sampler: Interrupt timer.report('defined other functions') -def main(args): +def main(args, timer): if args.output_dir: utils.mkdir(args.output_dir) @@ -400,4 +400,4 @@ def get_args_parser(add_help=True): if __name__ == "__main__": args = get_args_parser().parse_args() - main(args) + main(args, timer) From a887392507372758e83a1a598680fa15edb81cd2 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Sun, 10 Sep 2023 03:06:25 +0000 Subject: [PATCH 13/44] Updated tv-detection to use new InterruptableDistributedGroupedBatchSampler class --- cycling_utils/cycling_utils/__init__.py | 4 +- cycling_utils/cycling_utils/sampler.py | 164 ++++++++++++ tv-detection/engine.py | 118 +++++---- tv-detection/maskrcnn_resnet50_fpn.isc | 4 +- tv-detection/retinanet_resnet50_fpn.isc | 4 +- tv-detection/train_cycling.py | 123 +++++---- .../deeplabv3_mobilenet_v3_large.isc | 4 +- tv-segmentation/fcn_resnet101.isc | 4 +- tv-segmentation/train_cycling.py | 240 ++++++++++-------- tv-segmentation/utils.py | 7 +- 10 files changed, 457 insertions(+), 215 deletions(-) diff --git a/cycling_utils/cycling_utils/__init__.py b/cycling_utils/cycling_utils/__init__.py index ecff98ef..3a5de648 100644 --- a/cycling_utils/cycling_utils/__init__.py +++ b/cycling_utils/cycling_utils/__init__.py @@ -1,5 +1,5 @@ from .timer import Timer from .saving import atomic_torch_save -from .sampler import InterruptableDistributedSampler +from .sampler import InterruptableDistributedSampler, InterruptableDistributedGroupedBatchSampler -__all__ = ["InterruptableDistributedSampler", "atomic_torch_save", "Timer"] \ No newline at end of file +__all__ = ["InterruptableDistributedSampler", "InterruptableDistributedGroupedBatchSampler", "atomic_torch_save", "Timer"] \ No newline at end of file diff --git a/cycling_utils/cycling_utils/sampler.py b/cycling_utils/cycling_utils/sampler.py index 86c175ac..6c7325a2 100644 --- a/cycling_utils/cycling_utils/sampler.py +++ b/cycling_utils/cycling_utils/sampler.py @@ -2,6 +2,8 @@ import torch from torch.utils.data import Dataset, DistributedSampler from contextlib import contextmanager +from collections import defaultdict +from itertools import chain, repeat class HasNotResetProgressError(Exception): pass @@ -116,3 +118,165 @@ def in_epoch(self, epoch): self._set_epoch(epoch) yield self._reset_progress() + + + +## FOR tv-detection, require grouped batches + +def _repeat_to_at_least(iterable, n): + repeat_times = math.ceil(n / len(iterable)) + repeated = chain.from_iterable(repeat(iterable, repeat_times)) + return list(repeated) + +class InterruptableDistributedGroupedBatchSampler(DistributedSampler): + def __init__( + self, + dataset: Dataset, + group_ids: list, + batch_size: int, + num_replicas: int | None = None, + rank: int | None = None, + shuffle: bool = True, + seed: int = 0, + drop_last: bool = False, + ) -> None: + """ + This is a DistributedSampler that can be suspended and resumed. + + This works by keeping track of the epoch and progress within the epoch. + The progress is the number of samples that have been returned by the + sampler. The epoch is the number of times the sampler has been iterated + over. + + The epoch is incremented at the start of each epoch. The epoch is set + to 0 at initialization. + + The progress is incremented by the number of samples returned by the + sampler. The progress is reset to 0 at the end of each epoch. + + Suspending and resuming the sampler is done by saving and loading the + state dict. The state dict contains the epoch and progress. This works + because the permutation of the dataset is deterministic given the seed + and epoch. + """ + super().__init__(dataset, num_replicas, rank, shuffle, seed, drop_last) + + # OVERALL STATUS INDICATOR + self.progress = 0 + self._has_reset_progress = True + + # PRE-PROCESS DATASET + if shuffle: + # deterministically shuffle based on seed + g = torch.Generator() + g.manual_seed(seed) + indices = torch.randperm(len(dataset), generator=g).tolist() # type: ignore[arg-type] + else: + indices = list(range(len(dataset))) # type: ignore[arg-type] + + if not self.drop_last: + # add extra samples to make dataset evenly divisible accross ranks + padding_size = self.total_size - len(indices) + if padding_size <= len(indices): + indices += indices[:padding_size] + else: + indices += (indices * math.ceil(padding_size / len(indices)))[:padding_size] + else: + # remove tail of data to make dataset evenly divisible accross ranks + indices = indices[: self.total_size] + assert len(indices) == self.total_size + + # subsample indices to use on this rank + indices = indices[self.rank : self.total_size : self.num_replicas] + # num_samples is the number of samples to be processed each rank + assert len(indices) == self.num_samples + + # PRE-COMPUTE GROUPED BATCHES + + buffer_per_group = defaultdict(list) + samples_per_group = defaultdict(list) + self.num_batches = math.ceil(len(indices)/ batch_size) # why not? + + self.batches = [] # pre-computed so progress refers to batches, not samples. + for idx in indices: + group_id = group_ids[idx] + buffer_per_group[group_id].append(idx) + samples_per_group[group_id].append(idx) + if len(buffer_per_group[group_id]) == batch_size: + self.batches.append(buffer_per_group[group_id]) + del buffer_per_group[group_id] + assert len(buffer_per_group[group_id]) < batch_size + + # now we have run out of elements that satisfy + # the group criteria, let's return the remaining + # elements so that the size of the sampler is + # deterministic + num_remaining = self.num_batches - len(self.batches) + if num_remaining > 0: + # for the remaining batches, take first the buffers with the largest number + # of elements + for group_id, _ in sorted(buffer_per_group.items(), key=lambda x: len(x[1]), reverse=True): + remaining = batch_size - len(buffer_per_group[group_id]) + samples_from_group_id = _repeat_to_at_least(samples_per_group[group_id], remaining) + buffer_per_group[group_id].extend(samples_from_group_id[:remaining]) + assert len(buffer_per_group[group_id]) == batch_size + self.batches.append(buffer_per_group[group_id]) + num_remaining -= 1 + if num_remaining == 0: + break + + assert len(self.batches) == self.num_batches + + + def _reset_progress(self): + self.progress = 0 + self._has_reset_progress = True + + def set_epoch(self, epoch: int) -> None: + raise NotImplementedError("Use `with sampler.in_epoch(epoch)` instead of `sampler.set_epoch(epoch)`") + + def _set_epoch(self, epoch): + if not self._has_reset_progress: + raise HasNotResetProgressError("You must reset progress before setting epoch e.g. `sampler.reset_progress()`\nor use `with sampler.in_epoch(epoch)` instead of `sampler.set_epoch(epoch)`") + self.epoch = epoch + + def state_dict(self): + return {"progress": self.progress, "epoch": self.epoch} + + def load_state_dict(self, state_dict): + self.progress = state_dict["progress"] + if not self.progress <= self.num_batches: + raise AdvancedTooFarError(f"progress should be less than or equal to the number of batches. progress: {self.progress}, num_batches: {self.num_batches}") + self.epoch = state_dict["epoch"] + + def advance(self): + """ + Record that n samples have been consumed. + """ + self.progress += 1 + if self.progress > self.num_batches: + raise AdvancedTooFarError(f"You have advanced too far. You can only advance up to the total number of batches: {self.num_batches}.") + + def __iter__(self): + + # slice from progress to pick up where we left off + for batch in self.batches[self.progress:]: + yield batch + + def __len__(self): + return self.num_batches + + @contextmanager + def in_epoch(self, epoch): + """ + This context manager is used to set the epoch. It is used like this: + ``` + for epoch in range(0, 10): + with sampler.in_epoch(epoch): + for step, (x, ) in enumerate(dataloader): + # work would be done here... + ``` + """ + self._set_epoch(epoch) + yield + self._reset_progress() diff --git a/tv-detection/engine.py b/tv-detection/engine.py index 6b03e991..4621673a 100644 --- a/tv-detection/engine.py +++ b/tv-detection/engine.py @@ -9,7 +9,11 @@ from coco_utils import get_coco_api_from_dataset from cycling_utils import InterruptableDistributedSampler, atomic_torch_save -def train_one_epoch(model, optimizer, data_loader, train_batch_sampler, lr_scheduler, warmup_lr_scheduler, args, device, epoch, print_freq, scaler=None, timer=None): +def train_one_epoch( + model, optimizer, data_loader_train, train_sampler, test_sampler, + lr_scheduler, warmup_lr_scheduler, args, device, coco_evaluator, + epoch, print_freq, scaler=None, timer=None + ): model.train() metric_logger = utils.MetricLogger(delimiter=" ") @@ -19,27 +23,25 @@ def train_one_epoch(model, optimizer, data_loader, train_batch_sampler, lr_sched timer.report('training preliminaries') # Running this before starting the training loop assists reporting on progress after resuming - step == batch count - step = train_batch_sampler.sampler.progress // args.batch_size + # train_step = train_sampler.progress // args.batch_size + print(f'\nTraining / resuming epoch {epoch} from training step {train_sampler.progress}\n') - for images, targets in metric_logger.log_every(data_loader, train_batch_sampler.sampler.progress // args.batch_size, print_freq, header): ## EDITED THIS - ARGS.BATCH_SIZE == DATALOADER.BATCH_SIZE? GROUPEDBATCHSAMPLER AT PLAY + for images, targets in metric_logger.log_every(data_loader_train, train_sampler.progress, print_freq, header): ## EDITED THIS - ARGS.BATCH_SIZE == DATALOADER.BATCH_SIZE? GROUPEDBATCHSAMPLER AT PLAY images = list(image.to(device) for image in images) targets = [{k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in t.items()} for t in targets] - - timer.report(f'Epoch: {epoch} Step {step}: moving batch data to device') + timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: moving batch data to device') with torch.cuda.amp.autocast(enabled=scaler is not None): loss_dict = model(images, targets) losses = sum(loss for loss in loss_dict.values()) - - timer.report(f'Epoch: {epoch} Step {step}: forward pass') + timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: forward pass') # reduce losses over all GPUs for logging purposes loss_dict_reduced = utils.reduce_dict(loss_dict) losses_reduced = sum(loss for loss in loss_dict_reduced.values()) loss_value = losses_reduced.item() - - timer.report(f'Epoch: {epoch} Step {step}: computing loss') + timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: computing loss') if not math.isfinite(loss_value): print(f"Loss is {loss_value}, stopping training") @@ -54,44 +56,40 @@ def train_one_epoch(model, optimizer, data_loader, train_batch_sampler, lr_sched else: losses.backward() optimizer.step() - - timer.report(f'Epoch: {epoch} Step {step}: backward pass') + timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: backward pass') ## Always update warmup_lr_scheduler - once progressed past epoch 0, this will make no difference. warmup_lr_scheduler.step() - metric_logger.update(loss=losses_reduced, **loss_dict_reduced) metric_logger.update(lr=optimizer.param_groups[0]["lr"]) + timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: updating metric logger') - timer.report(f'Epoch: {epoch} Step {step}: updating metric logger') - - # ADDED THE FOLLOWING - INC NECESSARY ARGS TO TRAIN - train_batch_sampler.sampler.advance(len(images)) - step = train_batch_sampler.sampler.progress // args.batch_size - - timer.report(f'Epoch: {epoch} Step {step}: advancing sampler and computing step') - - if utils.is_main_process() and step % 5 == 0: # Checkpointing every 5 batches? + # train_step = train_sampler.progress + train_sampler.advance() # counted in batches, no args to pass + if utils.is_main_process() and train_sampler.progress % 1 == 0: # Checkpointing every batch + print(f"Saving checkpoint at epoch {epoch} train batch {train_sampler.progress}") checkpoint = { + "args": args, + "epoch": epoch, "model": model.module.state_dict(), "optimizer": optimizer.state_dict(), "lr_scheduler": lr_scheduler.state_dict(), "warmup_lr_scheduler": warmup_lr_scheduler.state_dict(), - "epoch": epoch, - "args": args, - "sampler": train_batch_sampler.sampler.state_dict(), + "train_sampler": train_sampler.state_dict(), + "test_sampler": test_sampler.state_dict(), + + # Evaluator state variables + "coco_gt": coco_evaluator.coco_gt, + "iou_types": coco_evaluator.iou_types, + "coco_eval": coco_evaluator.coco_eval, + "img_ids": coco_evaluator.img_ids, + "eval_imgs": coco_evaluator.eval_imgs, } if args.amp: checkpoint["scaler"] = scaler.state_dict() timer = atomic_torch_save(checkpoint, args.resume, timer) - # # Simulating end of epoch - # if step >= 10: - # print("Simulating end of epoch") - # return metric_logger, timer - - # # END ADD - + lr_scheduler.step() # OUTER LR_SCHEDULER STEP EACH EPOCH return metric_logger, timer @@ -108,7 +106,11 @@ def _get_iou_types(model): @torch.inference_mode() -def evaluate(model, data_loader, device, timer): +def evaluate( + model, data_loader_test, epoch, test_sampler, args, coco_evaluator, + optimizer, lr_scheduler, warmup_lr_scheduler, train_sampler, + device, scaler=None, timer=None +): timer.report('starting evaluation routine') @@ -120,39 +122,63 @@ def evaluate(model, data_loader, device, timer): metric_logger = utils.MetricLogger(delimiter=" ") header = "Test:" - timer.report(f'preliminaries') + timer.report(f'evaluation preliminaries') - coco = get_coco_api_from_dataset(data_loader.dataset) - iou_types = _get_iou_types(model) - coco_evaluator = CocoEvaluator(coco, iou_types) + # coco = get_coco_api_from_dataset(data_loader_test.dataset) + # iou_types = _get_iou_types(model) + # coco_evaluator = CocoEvaluator(coco, iou_types) - timer.report(f'preparing coco evaluator') + test_step = test_sampler.progress // data_loader_test.batch_size + print(f'\nEvaluating / resuming epoch {epoch} from eval step {test_step}\n') + timer.report('launch evaluation routine') - eval_batch = 1 for images, targets in metric_logger.log_every(data_loader, 100, header): images = list(img.to(device) for img in images) - - timer.report(f'eval batch: {eval_batch} moving to device') + timer.report(f'Epoch {epoch} batch: {test_step} moving to device') if torch.cuda.is_available(): torch.cuda.synchronize() + model_time = time.time() outputs = model(images) - - timer.report(f'eval batch: {eval_batch} forward through model') + timer.report(f'Epoch {epoch} batch: {test_step} forward through model') outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] - - timer.report(f'eval batch: {eval_batch} outputs back to cpu') + model_time = time.time() - model_time + timer.report(f'Epoch {epoch} batch: {test_step} outputs back to cpu') res = {target["image_id"]: output for target, output in zip(targets, outputs)} evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) - timer.report(f'eval batch: {eval_batch} update evaluator') + timer.report(f'Epoch {epoch} batch: {test_step} update evaluator') - eval_batch += 1 + test_sampler.advance(len(images)) + + test_step = test_sampler.progress // data_loader_test.batch_size + if utils.is_main_process() and test_step % 1 == 0: # Checkpointing every batch + print(f"Saving checkpoint at epoch {epoch} eval batch {test_step}") + checkpoint = { + "args": args, + "epoch": epoch, + "model": model.module.state_dict(), + "optimizer": optimizer.state_dict(), + "lr_scheduler": lr_scheduler.state_dict(), + "warmup_lr_scheduler": warmup_lr_scheduler.state_dict(), + "train_sampler": train_sampler.state_dict(), + "test_sampler": test_sampler.state_dict(), + + # Evaluator state variables + "coco_gt": coco_evaluator.coco_gt, + "iou_types": coco_evaluator.iou_types, + "coco_eval": coco_evaluator.coco_eval, + "img_ids": coco_evaluator.img_ids, + "eval_imgs": coco_evaluator.eval_imgs, + } + if args.amp: + checkpoint["scaler"] = scaler.state_dict() + timer = atomic_torch_save(checkpoint, args.resume, timer) # gather the stats from all processes metric_logger.synchronize_between_processes() diff --git a/tv-detection/maskrcnn_resnet50_fpn.isc b/tv-detection/maskrcnn_resnet50_fpn.isc index 89858aca..44432d9e 100644 --- a/tv-detection/maskrcnn_resnet50_fpn.isc +++ b/tv-detection/maskrcnn_resnet50_fpn.isc @@ -1,6 +1,6 @@ experiment_name="maskrcnn_resnet50_fpn" gpu_type="24GB VRAM GPU" -nnodes = 9 +nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_maskrcnn_resnet50_fpn" -command="train_cycling.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 4 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file +command="train_cycling.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 4 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/Datasets/Open-Datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/tv-detection/retinanet_resnet50_fpn.isc b/tv-detection/retinanet_resnet50_fpn.isc index e70607a2..4c4c9f19 100644 --- a/tv-detection/retinanet_resnet50_fpn.isc +++ b/tv-detection/retinanet_resnet50_fpn.isc @@ -1,7 +1,7 @@ experiment_name="retinanet_resnet50_fpn" gpu_type="24GB VRAM GPU" -nnodes = 9 +nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_retinanet_resnet50_fpn" -command="train_cycling.py --dataset coco --model retinanet_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 4 --aspect-ratio-group-factor 3 --lr 0.01 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" +command="train_cycling.py --dataset coco --model retinanet_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 4 --aspect-ratio-group-factor 3 --lr 0.01 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/Datasets/Open-Datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index c9771d2a..8c75cb8b 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -42,7 +42,7 @@ from torchvision.transforms import InterpolationMode from transforms import SimpleCopyPaste -from cycling_utils import InterruptableDistributedSampler, atomic_torch_save +from cycling_utils import InterruptableDistributedSampler, InterruptableDistributedGroupedBatchSampler, atomic_torch_save timer.report('importing everything else') @@ -76,6 +76,17 @@ def get_transform(is_train, args): else: return presets.DetectionPresetEval(backend=args.backend, use_v2=args.use_v2) +def _get_iou_types(model): + model_without_ddp = model + if isinstance(model, torch.nn.parallel.DistributedDataParallel): + model_without_ddp = model.module + iou_types = ["bbox"] + if isinstance(model_without_ddp, torchvision.models.detection.MaskRCNN): + iou_types.append("segm") + if isinstance(model_without_ddp, torchvision.models.detection.KeypointRCNN): + iou_types.append("keypoints") + return iou_types + timer.report('defined other functions') def main(args, timer): @@ -103,9 +114,13 @@ def main(args, timer): timer.report('main preliminaries') # Data loading code - dataset, num_classes = get_dataset(is_train=True, args=args) + dataset_train, num_classes = get_dataset(is_train=True, args=args) dataset_test, _ = get_dataset(is_train=False, args=args) + ## SUBSET FOR TESTING EPOCH ROLLOVER + dataset_train = torch.utils.data.Subset(dataset_train, torch.arange(450)) + dataset_test = torch.utils.data.Subset(dataset_test, torch.arange(108)) + timer.report('loading data') # if args.distributed: @@ -114,18 +129,19 @@ def main(args, timer): # else: # train_sampler = torch.utils.data.RandomSampler(dataset) # test_sampler = torch.utils.data.SequentialSampler(dataset_test) - train_sampler = InterruptableDistributedSampler(dataset) - test_sampler = torch.utils.data.distributed.DistributedSampler(dataset_test, shuffle=False) - timer.report('creating data samplers') + # if args.aspect_ratio_group_factor >= 0: # default == 3 + # group_ids = create_aspect_ratio_groups(dataset_train, k=args.aspect_ratio_group_factor) + # train_batch_sampler = GroupedBatchSampler(train_sampler, group_ids, args.batch_size) + # else: + # train_batch_sampler = torch.utils.data.BatchSampler(train_sampler, args.batch_size, drop_last=True) - if args.aspect_ratio_group_factor >= 0: # default == 3 - group_ids = create_aspect_ratio_groups(dataset, k=args.aspect_ratio_group_factor) - train_batch_sampler = GroupedBatchSampler(train_sampler, group_ids, args.batch_size) - else: - train_batch_sampler = torch.utils.data.BatchSampler(train_sampler, args.batch_size, drop_last=True) + group_ids = create_aspect_ratio_groups(dataset_train, k=args.aspect_ratio_group_factor) + + train_sampler = InterruptableDistributedGroupedBatchSampler(dataset_train, group_ids, args.batch_size) + test_sampler = InterruptableDistributedSampler(dataset_test) - timer.report('creating GroupedBatchSampler') + timer.report('creating data samplers') train_collate_fn = utils.collate_fn if args.use_copypaste: @@ -134,8 +150,8 @@ def main(args, timer): print("Using copypaste_collate_fn for train_collate_fn") train_collate_fn = copypaste_collate_fn - data_loader = torch.utils.data.DataLoader( - dataset, batch_sampler=train_batch_sampler, num_workers=args.workers, collate_fn=train_collate_fn + data_loader_train = torch.utils.data.DataLoader( + dataset_train, batch_sampler=train_sampler, num_workers=args.workers, collate_fn=train_collate_fn ) data_loader_test = torch.utils.data.DataLoader( dataset_test, batch_size=1, sampler=test_sampler, num_workers=args.workers, collate_fn=utils.collate_fn @@ -165,6 +181,8 @@ def main(args, timer): model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) model_without_ddp = model.module + timer.report('preparing model for distributed training') + if args.norm_weight_decay is None: parameters = [p for p in model.parameters() if p.requires_grad] else: @@ -172,8 +190,6 @@ def main(args, timer): wd_groups = [args.norm_weight_decay, args.weight_decay] parameters = [{"params": p, "weight_decay": w} for p, w in zip(param_groups, wd_groups) if p] - timer.report('preparing model for distributed training') - opt_name = args.opt.lower() if opt_name.startswith("sgd"): optimizer = torch.optim.SGD( @@ -205,27 +221,47 @@ def main(args, timer): ## WARMUP LR_SCHEDULER warmup_factor = 1.0 / 1000 - warmup_iters = min(1000, len(data_loader) - 1) + warmup_iters = min(1000, len(data_loader_train) - 1) warmup_lr_scheduler = torch.optim.lr_scheduler.LinearLR( optimizer, start_factor=warmup_factor, total_iters=warmup_iters ) timer.report('learning rate schedulers') - Path(args.resume).parent.mkdir(parents=True, exist_ok=True) ### ADDED THIS - if args.resume and os.path.isfile(args.resume): ## EDITED THIS + from coco_eval import CocoEvaluator + from coco_utils import get_coco_api_from_dataset + coco = get_coco_api_from_dataset(data_loader_test.dataset) + iou_types = _get_iou_types(model) + coco_evaluator = CocoEvaluator(coco, iou_types) + + timer.report('init coco evaluator') + + Path(args.resume).parent.mkdir(parents=True, exist_ok=True) + if args.resume and os.path.isfile(args.resume): + checkpoint = torch.load(args.resume, map_location="cpu") model_without_ddp.load_state_dict(checkpoint["model"]) + args.start_epoch = checkpoint["epoch"] + optimizer.load_state_dict(checkpoint["optimizer"]) - lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) ## EDITED THIS - warmup_lr_scheduler.load_state_dict(checkpoint["warmup_lr_scheduler"]) ## ADDED THIS - args.start_epoch = checkpoint["epoch"] # + 1 + lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) + warmup_lr_scheduler.load_state_dict(checkpoint["warmup_lr_scheduler"]) + train_sampler.load_state_dict(checkpoint["train_sampler"]) if args.amp: scaler.load_state_dict(checkpoint["scaler"]) - train_batch_sampler.sampler.load_state_dict(checkpoint["sampler"]) # INTERRUPTABLE SAMPLER IS MEMBER OF GROUPED BATCH SAMPLER + + test_sampler.load_state_dict(checkpoint["test_sampler"]) + + # Evaluator state variables + coco_evaluator.coco_gt = checkpoint["coco_gt"] + coco_evaluator.iou_types = checkpoint["iou_types"] + coco_evaluator.coco_eval = checkpoint["coco_eval"] + coco_evaluator.img_ids = checkpoint["img_ids"] + coco_evaluator.eval_imgs = checkpoint["eval_imgs"] timer.report('retrieving checkpoint') + # KILL THIS FOR NOW if args.test_only: # We disable the cudnn benchmarking because it can noticeably affect the accuracy @@ -235,41 +271,18 @@ def main(args, timer): return for epoch in range(args.start_epoch, args.epochs): - # if args.distributed: - # train_sampler.set_epoch(epoch) - with train_batch_sampler.sampler.in_epoch(epoch): + + print('\n') + print(f"EPOCH :: {epoch}") + print('\n') - print('\n') + with train_sampler.in_epoch(epoch): timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch - timer.report(f'launching epoch {epoch}') - metric_logger, timer = train_one_epoch(model, optimizer, data_loader, train_batch_sampler, lr_scheduler, warmup_lr_scheduler, args, device, epoch, args.print_freq, scaler, timer) - - lr_scheduler.step() # OUTER LR_SCHEDULER STEP EACH EPOCH - - timer.report(f'training for epoch {epoch}') - - if utils.is_main_process(): - checkpoint = { - "model": model_without_ddp.state_dict(), - "optimizer": optimizer.state_dict(), - "lr_scheduler": lr_scheduler.state_dict(), - "warmup_lr_scheduler": warmup_lr_scheduler.state_dict(), - "args": args, - "epoch": epoch, - "sampler": train_batch_sampler.sampler.state_dict(), - } - - if args.amp: - checkpoint["scaler"] = scaler.state_dict() - - timer.report(f'defining epoch {epoch} checkpoint') - - # utils.save_on_master(checkpoint, os.path.join(args.output_dir, f"model_{epoch}.pth")) - # utils.save_on_master(checkpoint, os.path.join(args.output_dir, "checkpoint.pth")) - timer = atomic_torch_save(checkpoint, args.resume, timer) - - # KILL THIS FOR NOW - coco_evaluator, timer = evaluate(model, data_loader_test, device, timer) + metric_logger, timer = train_one_epoch(model, optimizer, data_loader_train, train_sampler, test_sampler, lr_scheduler, warmup_lr_scheduler, args, device, coco_evaluator, epoch, args.print_freq, scaler, timer) + + with test_sampler.in_epoch(epoch): + timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch + coco_evaluator, timer = evaluate(model, data_loader_test, epoch, test_sampler, args, coco_evaluator, optimizer, lr_scheduler, warmup_lr_scheduler, train_sampler, device, scaler, timer) def get_args_parser(add_help=True): import argparse diff --git a/tv-segmentation/deeplabv3_mobilenet_v3_large.isc b/tv-segmentation/deeplabv3_mobilenet_v3_large.isc index 53ea47a0..7af6132a 100644 --- a/tv-segmentation/deeplabv3_mobilenet_v3_large.isc +++ b/tv-segmentation/deeplabv3_mobilenet_v3_large.isc @@ -1,7 +1,7 @@ experiment_name="deeplabv3_mobilenet_v3_large" gpu_type="24GB VRAM GPU" -nnodes = 9 +nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_deeplabv3_mobilenet_v3_large" -command="train_cycling.py --dataset coco -b 2 --model deeplabv3_mobilenet_v3_large --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" +command="train_cycling.py --dataset coco -b 2 --model deeplabv3_mobilenet_v3_large --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 --data-path=/mnt/Datasets/Open-Datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" diff --git a/tv-segmentation/fcn_resnet101.isc b/tv-segmentation/fcn_resnet101.isc index 393d46bb..1b8d72d8 100644 --- a/tv-segmentation/fcn_resnet101.isc +++ b/tv-segmentation/fcn_resnet101.isc @@ -1,7 +1,7 @@ experiment_name="fcn_resnet101" gpu_type="24GB VRAM GPU" -nnodes = 9 +nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_fcn_resnet101" -command="train_cycling.py --lr 0.0002 --dataset coco -b 2 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --epochs=1 --resume $OUTPUT_PATH/checkpoint.isc" +command="train_cycling.py --lr 0.0002 --dataset coco -b 2 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/mnt/Datasets/Open-Datasets/coco --epochs=1 --resume $OUTPUT_PATH/checkpoint.isc" diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index 82281d47..b3747571 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -25,102 +25,135 @@ def get_dataset(dir_path, name, image_set, transform): def sbd(*args, **kwargs): return torchvision.datasets.SBDataset(*args, mode="segmentation", **kwargs) - paths = { "voc": (dir_path, torchvision.datasets.VOCSegmentation, 21), "voc_aug": (dir_path, sbd, 21), "coco": (dir_path, get_coco, 21), } p, ds_fn, num_classes = paths[name] - ds = ds_fn(p, image_set=image_set, transforms=transform) return ds, num_classes - def get_transform(train, args): if train: return presets.SegmentationPresetTrain(base_size=520, crop_size=480) elif args.weights and args.test_only: weights = torchvision.models.get_weight(args.weights) trans = weights.transforms() - def preprocessing(img, target): img = trans(img) size = F.get_dimensions(img)[1:] target = F.resize(target, size, interpolation=InterpolationMode.NEAREST) return img, F.pil_to_tensor(target) - return preprocessing else: return presets.SegmentationPresetEval(base_size=520) - def criterion(inputs, target): losses = {} for name, x in inputs.items(): losses[name] = nn.functional.cross_entropy(x, target, ignore_index=255) - if len(losses) == 1: return losses["out"] - return losses["out"] + 0.5 * losses["aux"] - -def evaluate(model, data_loader, device, num_classes, timer): - - timer.report('starting evaluation routine') +def evaluate( + model, data_loader_test, num_classes, confmat, + optimizer, lr_scheduler, + train_sampler: InterruptableDistributedSampler, + test_sampler: InterruptableDistributedSampler, + device, epoch, print_freq, scaler=None, timer=None + ): model.eval() - confmat = utils.ConfusionMatrix(num_classes) metric_logger = utils.MetricLogger(delimiter=" ") header = "Test:" num_processed_samples = 0 + timer.report(f'evaluation preliminaries') - timer.report(f'preliminaries') + test_step = test_sampler.progress // data_loader_test.batch_size + print(f'\nEvaluating / resuming epoch {epoch} from eval step {test_step}\n') + timer.report('launch evaluation routine') with torch.inference_mode(): - eval_batch = 1 - for image, target in metric_logger.log_every(data_loader, 100, header): - image, target = image.to(device), target.to(device) - timer.report(f'eval batch: {eval_batch} moving to device') + for image, target in metric_logger.log_every(data_loader_test, test_step, print_freq, header): + + image, target = image.to(device), target.to(device) + timer.report(f'Epoch {epoch} batch: {test_step} moving to device') output = model(image) output = output["out"] + timer.report(f'Epoch {epoch} batch: {test_step} forward through model') - timer.report(f'eval batch: {eval_batch} forward through model') + confmat_temp = utils.ConfusionMatrix(num_classes) + confmat_temp.update(target.flatten().detach().cpu(), output.argmax(1).flatten().detach().cpu()) + confmat_temp.reduce_from_all_processes() + confmat.mat += confmat_temp.mat.cpu() # Add to the running eval matrix - confmat.update(target.flatten(), output.argmax(1).flatten()) # FIXME need to take into account that the datasets # could have been padded in distributed setup num_processed_samples += image.shape[0] + timer.report(f'Epoch {epoch} batch: {test_step} confmat update') - timer.report(f'eval batch: {eval_batch} confmat update') + test_sampler.advance(len(image)) - eval_batch += 1 + test_step = test_sampler.progress // data_loader_test.batch_size + if utils.is_main_process() and test_step % 1 == 0: # Checkpointing every batch - confmat.reduce_from_all_processes() + print(f"Saving checkpoint at epoch {epoch} eval batch {test_step}") + checkpoint = { + "args": args, + "epoch": epoch, + "model": model.module.state_dict(), + "optimizer": optimizer.state_dict(), + "lr_scheduler": lr_scheduler.state_dict(), + "train_sampler": train_sampler.state_dict(), + "test_sampler": test_sampler.state_dict(), + "confmat": confmat.mat, # For storing eval metric + } + if args.amp: + checkpoint["scaler"] = scaler.state_dict() + timer = atomic_torch_save(checkpoint, args.resume, timer) + print(confmat) num_processed_samples = utils.reduce_across_processes(num_processed_samples) - timer.report(f'confmat reduction accross processes') if ( - hasattr(data_loader.dataset, "__len__") - and len(data_loader.dataset) != num_processed_samples + hasattr(data_loader_test.dataset, "__len__") + and len(data_loader_test.dataset) != num_processed_samples and torch.distributed.get_rank() == 0 ): # See FIXME above warnings.warn( - f"It looks like the dataset has {len(data_loader.dataset)} samples, but {num_processed_samples} " + f"It looks like the dataset has {len(data_loader_test.dataset)} samples, but {num_processed_samples} " "samples were used for the validation, which might bias the results. " "Try adjusting the batch size and / or the world size. " "Setting the world size to 1 is always a safe bet." ) + # # Save final checkpoint once all eval ops have completed - MIGHT BE ABLE TO OMIT THIS ALTOGETHER + # if utils.is_main_process(): + # print(f"Saving checkpoint at end of epoch {epoch} evaluation") + # checkpoint = { + # "mode": "train", + # "model": model.module.state_dict(), + # "epoch": epoch, + # "args": args, + # "test_sampler": test_sampler.state_dict(), + # "conf_mat": confmat, # For storing eval metric + # } + # timer = atomic_torch_save(checkpoint, args.resume, timer) + return confmat, timer -def train_one_epoch(model, criterion, optimizer, data_loader, sampler: InterruptableDistributedSampler, lr_scheduler, device, epoch, print_freq, scaler=None, timer=None): +def train_one_epoch( + model, criterion, optimizer, data_loader_train, + train_sampler: InterruptableDistributedSampler, + test_sampler: InterruptableDistributedSampler, confmat, + lr_scheduler, device, epoch, print_freq, scaler=None, timer=None + ): model.train() metric_logger = utils.MetricLogger(delimiter=" ") @@ -129,19 +162,20 @@ def train_one_epoch(model, criterion, optimizer, data_loader, sampler: Interrupt timer.report('training preliminaries') - # Running this before starting the training loop assists reporting on progress after resuming - step == batch count - step = sampler.progress // data_loader.batch_size + # Running this before starting the training loop assists reporting on progress after resuming - train_step == batch count + # Also means when resuming during evaluation, the training phase is skipped as train_sampler progress == 100%. + train_step = train_sampler.progress // data_loader_train.batch_size + print(f'\nTraining / resuming epoch {epoch} from training step {train_step}\n') - for image, target in metric_logger.log_every(data_loader, step, print_freq, header): - image, target = image.to(device), target.to(device) + for image, target in metric_logger.log_every(data_loader_train, train_step, print_freq, header): - timer.report(f'Epoch: {epoch} Step {step}: moving batch data to device') + image, target = image.to(device), target.to(device) + timer.report(f'Epoch: {epoch} batch {train_step}: moving batch data to device') with torch.cuda.amp.autocast(enabled=scaler is not None): output = model(image) loss = criterion(output, target) - - timer.report(f'Epoch: {epoch} Step {step}: forward pass') + timer.report(f'Epoch: {epoch} batch {train_step}: forward pass') optimizer.zero_grad() if scaler is not None: @@ -151,30 +185,46 @@ def train_one_epoch(model, criterion, optimizer, data_loader, sampler: Interrupt else: loss.backward() optimizer.step() - - timer.report(f'Epoch: {epoch} Step {step}: backward pass') + timer.report(f'Epoch: {epoch} batch {train_step}: backward pass') lr_scheduler.step() metric_logger.update(loss=loss.item(), lr=optimizer.param_groups[0]["lr"]) - sampler.advance(len(image)) - - timer.report(f'Epoch: {epoch} Step {step}: updating metric logger') + train_sampler.advance(len(image)) + timer.report(f'Epoch: {epoch} batch {train_step}: updating metric logger') - step = sampler.progress // data_loader.batch_size - if utils.is_main_process() and step % 5 == 0: # Checkpointing every 5 batches? - print(f"Saving checkpoint at step {step}") + train_step = train_sampler.progress // data_loader_train.batch_size + if utils.is_main_process() and train_step % 1 == 0: # Checkpointing every batch + print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") checkpoint = { + "args": args, + "epoch": epoch, "model": model.module.state_dict(), "optimizer": optimizer.state_dict(), "lr_scheduler": lr_scheduler.state_dict(), - "epoch": epoch, - "args": args, - "sampler": sampler.state_dict(), + "train_sampler": train_sampler.state_dict(), + "test_sampler": test_sampler.state_dict(), + "confmat": confmat, # For storing eval metric } if args.amp: checkpoint["scaler"] = scaler.state_dict() timer = atomic_torch_save(checkpoint, args.resume, timer) + # # Saving checkpoint at the end of training epoch - MIGHT BE ABLE TO OMIT THIS ALTOGETHER + # if utils.is_main_process(): # Checkpointing every batch? + # print(f"Saving checkpoint at end of epoch {epoch} training") + # checkpoint = { + # "mode": "evaluate", + # "model": model.module.state_dict(), + # "optimizer": optimizer.state_dict(), + # "lr_scheduler": lr_scheduler.state_dict(), + # "epoch": epoch, + # "args": args, + # "train_sampler": train_sampler.state_dict(), + # } + # if args.amp: + # checkpoint["scaler"] = scaler.state_dict() + # timer = atomic_torch_save(checkpoint, args.resume, timer) + return metric_logger, timer timer.report('defined other functions') @@ -197,31 +247,30 @@ def main(args, timer): else: torch.backends.cudnn.benchmark = True - timer.report('preliminaries') + timer.report('main preliminaries') - dataset, num_classes = get_dataset(args.data_path, args.dataset, "train", get_transform(True, args)) + dataset_train, num_classes = get_dataset(args.data_path, args.dataset, "train", get_transform(True, args)) dataset_test, _ = get_dataset(args.data_path, args.dataset, "val", get_transform(False, args)) + ## SUBSET FOR TESTING EPOCH ROLLOVER + dataset_train = torch.utils.data.Subset(dataset_train, torch.arange(450)) + dataset_test = torch.utils.data.Subset(dataset_test, torch.arange(108)) + timer.report('loading data') # if args.distributed: - train_sampler = InterruptableDistributedSampler(dataset) - test_sampler = torch.utils.data.distributed.DistributedSampler(dataset_test, shuffle=False) + train_sampler = InterruptableDistributedSampler(dataset_train) + test_sampler = InterruptableDistributedSampler(dataset_test) + # test_sampler = torch.utils.data.distributed.DistributedSampler(dataset_test, shuffle=False) # else: # train_sampler = torch.utils.data.RandomSampler(dataset) # test_sampler = torch.utils.data.SequentialSampler(dataset_test) timer.report('creating data samplers') - data_loader = torch.utils.data.DataLoader( - dataset, - batch_size=args.batch_size, - sampler=train_sampler, - num_workers=args.workers, - collate_fn=utils.collate_fn, - drop_last=True, + data_loader_train = torch.utils.data.DataLoader( + dataset_train, batch_size=args.batch_size, sampler=train_sampler, num_workers=args.workers, collate_fn=utils.collate_fn, drop_last=True, ) - data_loader_test = torch.utils.data.DataLoader( dataset_test, batch_size=1, sampler=test_sampler, num_workers=args.workers, collate_fn=utils.collate_fn ) @@ -229,11 +278,7 @@ def main(args, timer): timer.report('creating data loaders') model = torchvision.models.get_model( - args.model, - weights=args.weights, - weights_backbone=args.weights_backbone, - num_classes=num_classes, - aux_loss=args.aux_loss, + args.model, weights=args.weights, weights_backbone=args.weights_backbone, num_classes=num_classes, aux_loss=args.aux_loss, ) model.to(device) @@ -247,13 +292,13 @@ def main(args, timer): model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) model_without_ddp = model.module + timer.report('preparing model for distributed training') + params_to_optimize = [ {"params": [p for p in model_without_ddp.backbone.parameters() if p.requires_grad]}, {"params": [p for p in model_without_ddp.classifier.parameters() if p.requires_grad]}, ] - timer.report('preparing model for distributed training') - if args.aux_loss: params = [p for p in model_without_ddp.aux_classifier.parameters() if p.requires_grad] params_to_optimize.append({"params": params, "lr": args.lr * 10}) @@ -263,7 +308,7 @@ def main(args, timer): timer.report('optimizer and scaler') - iters_per_epoch = len(data_loader) + iters_per_epoch = len(data_loader_train) main_lr_scheduler = PolynomialLR( optimizer, total_iters=iters_per_epoch * (args.epochs - args.lr_warmup_epochs), power=0.9 ) @@ -291,58 +336,51 @@ def main(args, timer): timer.report('learning rate schedulers') + # Init global confmat for eval - eval accumulator + confmat = utils.ConfusionMatrix(num_classes) + timer.report('init confmat') + Path(args.resume).parent.mkdir(parents=True, exist_ok=True) - if args.resume and os.path.isfile(args.resume): + if args.resume and os.path.isfile(args.resume): # If we're resuming... + checkpoint = torch.load(args.resume, map_location="cpu") model_without_ddp.load_state_dict(checkpoint["model"], strict=not args.test_only) - if not args.test_only: - optimizer.load_state_dict(checkpoint["optimizer"]) - lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) - args.start_epoch = checkpoint["epoch"] #+ 1 - if args.amp: - scaler.load_state_dict(checkpoint["scaler"]) - train_sampler.load_state_dict(checkpoint["sampler"]) - + args.start_epoch = checkpoint["epoch"] + + optimizer.load_state_dict(checkpoint["optimizer"]) + lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) + train_sampler.load_state_dict(checkpoint["train_sampler"]) + if args.amp: # Could align this syntactically... + scaler.load_state_dict(checkpoint["scaler"]) + + test_sampler.load_state_dict(checkpoint["test_sampler"]) + confmat.mat = checkpoint["confmat"] + timer.report('retrieving checkpoint') if args.test_only: # We disable the cudnn benchmarking because it can noticeably affect the accuracy torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True - confmat, timer = evaluate(model, data_loader_test, device=device, num_classes=num_classes, timer=timer) + confmat, timer = evaluate(model, data_loader_test, num_classes, confmat, test_sampler, device, 0, args.print_freq, timer) print(confmat) return for epoch in range(args.start_epoch, args.epochs): - # if args.distributed: - with train_sampler.in_epoch(epoch): - - print('\n') - timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch - timer.report(f'launching epoch {epoch}') - metric_logger, timer = train_one_epoch(model, criterion, optimizer, data_loader, train_sampler, lr_scheduler, device, epoch, args.print_freq, scaler, timer) + print('\n') + print(f"EPOCH :: {epoch}") + print('\n') + with train_sampler.in_epoch(epoch): + timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch + metric_logger, timer = train_one_epoch(model, criterion, optimizer, data_loader_train, train_sampler, test_sampler, confmat, lr_scheduler, device, epoch, args.print_freq, scaler, timer) timer.report(f'training for epoch {epoch}') - - if utils.is_main_process(): - checkpoint = { - "model": model.module.state_dict(), - "optimizer": optimizer.state_dict(), - "lr_scheduler": lr_scheduler.state_dict(), - "epoch": epoch, - "args": args, - "sampler": train_sampler.state_dict(), - } - if args.amp: - checkpoint["scaler"] = scaler.state_dict() - timer.report(f'defining epoch {epoch} checkpoint') - - timer = atomic_torch_save(checkpoint, args.resume, timer) - - confmat, timer = evaluate(model, data_loader_test, device=device, num_classes=num_classes, timer=timer) - print(confmat) + with test_sampler.in_epoch(epoch): + timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch + confmat, timer = evaluate(model, data_loader_test, num_classes, confmat, optimizer, lr_scheduler, train_sampler, test_sampler, device, epoch, args.print_freq, scaler, timer) + timer.report(f'evaluation for epoch {epoch}') def get_args_parser(add_help=True): diff --git a/tv-segmentation/utils.py b/tv-segmentation/utils.py index 73bb882d..31e7bb42 100644 --- a/tv-segmentation/utils.py +++ b/tv-segmentation/utils.py @@ -66,12 +66,13 @@ def __str__(self): class ConfusionMatrix: def __init__(self, num_classes): self.num_classes = num_classes - self.mat = None + # self.mat = None + self.mat = torch.zeros((num_classes, num_classes), dtype=torch.int64, device='cpu') def update(self, a, b): n = self.num_classes - if self.mat is None: - self.mat = torch.zeros((n, n), dtype=torch.int64, device=a.device) + # if self.mat is None: + # self.mat = torch.zeros((n, n), dtype=torch.int64, device=a.device) with torch.inference_mode(): k = (a >= 0) & (a < n) inds = n * a[k].to(torch.int64) + b[k] From 1595277d8ac681c0fa3e0e0dd3fb0bb57dae080d Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Sun, 10 Sep 2023 08:52:11 +0000 Subject: [PATCH 14/44] detection and segmentation working all but detection eval --- tv-detection/engine.py | 20 ++++---- tv-detection/maskrcnn_resnet50_fpn.isc | 2 +- tv-detection/retinanet_resnet50_fpn.isc | 2 +- tv-detection/train_cycling.py | 61 ++++++++++++------------- tv-segmentation/train_cycling.py | 55 ++++++---------------- 5 files changed, 56 insertions(+), 84 deletions(-) diff --git a/tv-detection/engine.py b/tv-detection/engine.py index 4621673a..152def96 100644 --- a/tv-detection/engine.py +++ b/tv-detection/engine.py @@ -11,8 +11,8 @@ def train_one_epoch( model, optimizer, data_loader_train, train_sampler, test_sampler, - lr_scheduler, warmup_lr_scheduler, args, device, coco_evaluator, - epoch, print_freq, scaler=None, timer=None + lr_scheduler, warmup_lr_scheduler, args, device, + epoch, scaler=None, timer=None ): model.train() @@ -26,7 +26,7 @@ def train_one_epoch( # train_step = train_sampler.progress // args.batch_size print(f'\nTraining / resuming epoch {epoch} from training step {train_sampler.progress}\n') - for images, targets in metric_logger.log_every(data_loader_train, train_sampler.progress, print_freq, header): ## EDITED THIS - ARGS.BATCH_SIZE == DATALOADER.BATCH_SIZE? GROUPEDBATCHSAMPLER AT PLAY + for images, targets in metric_logger.log_every(data_loader_train, train_sampler.progress, args.print_freq, header): ## EDITED THIS - ARGS.BATCH_SIZE == DATALOADER.BATCH_SIZE? GROUPEDBATCHSAMPLER AT PLAY images = list(image.to(device) for image in images) targets = [{k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in t.items()} for t in targets] @@ -78,12 +78,12 @@ def train_one_epoch( "train_sampler": train_sampler.state_dict(), "test_sampler": test_sampler.state_dict(), - # Evaluator state variables - "coco_gt": coco_evaluator.coco_gt, - "iou_types": coco_evaluator.iou_types, - "coco_eval": coco_evaluator.coco_eval, - "img_ids": coco_evaluator.img_ids, - "eval_imgs": coco_evaluator.eval_imgs, + # # Evaluator state variables + # "coco_gt": coco_evaluator.coco_gt, + # "iou_types": coco_evaluator.iou_types, + # "coco_eval": coco_evaluator.coco_eval, + # "img_ids": coco_evaluator.img_ids, + # "eval_imgs": coco_evaluator.eval_imgs, } if args.amp: checkpoint["scaler"] = scaler.state_dict() @@ -132,7 +132,7 @@ def evaluate( print(f'\nEvaluating / resuming epoch {epoch} from eval step {test_step}\n') timer.report('launch evaluation routine') - for images, targets in metric_logger.log_every(data_loader, 100, header): + for images, targets in metric_logger.log_every(data_loader_test, test_sampler.progress, args.print_freq, header): images = list(img.to(device) for img in images) timer.report(f'Epoch {epoch} batch: {test_step} moving to device') diff --git a/tv-detection/maskrcnn_resnet50_fpn.isc b/tv-detection/maskrcnn_resnet50_fpn.isc index 44432d9e..11a52700 100644 --- a/tv-detection/maskrcnn_resnet50_fpn.isc +++ b/tv-detection/maskrcnn_resnet50_fpn.isc @@ -3,4 +3,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_maskrcnn_resnet50_fpn" -command="train_cycling.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 4 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/Datasets/Open-Datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file +command="train_cycling.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/Datasets/Open-Datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/tv-detection/retinanet_resnet50_fpn.isc b/tv-detection/retinanet_resnet50_fpn.isc index 4c4c9f19..6fcda85b 100644 --- a/tv-detection/retinanet_resnet50_fpn.isc +++ b/tv-detection/retinanet_resnet50_fpn.isc @@ -3,5 +3,5 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_retinanet_resnet50_fpn" -command="train_cycling.py --dataset coco --model retinanet_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 4 --aspect-ratio-group-factor 3 --lr 0.01 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/Datasets/Open-Datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" +command="train_cycling.py --dataset coco --model retinanet_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --lr 0.01 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/Datasets/Open-Datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index 8c75cb8b..66ed7369 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -76,16 +76,16 @@ def get_transform(is_train, args): else: return presets.DetectionPresetEval(backend=args.backend, use_v2=args.use_v2) -def _get_iou_types(model): - model_without_ddp = model - if isinstance(model, torch.nn.parallel.DistributedDataParallel): - model_without_ddp = model.module - iou_types = ["bbox"] - if isinstance(model_without_ddp, torchvision.models.detection.MaskRCNN): - iou_types.append("segm") - if isinstance(model_without_ddp, torchvision.models.detection.KeypointRCNN): - iou_types.append("keypoints") - return iou_types +# def _get_iou_types(model): +# model_without_ddp = model +# if isinstance(model, torch.nn.parallel.DistributedDataParallel): +# model_without_ddp = model.module +# iou_types = ["bbox"] +# if isinstance(model_without_ddp, torchvision.models.detection.MaskRCNN): +# iou_types.append("segm") +# if isinstance(model_without_ddp, torchvision.models.detection.KeypointRCNN): +# iou_types.append("keypoints") +# return iou_types timer.report('defined other functions') @@ -117,9 +117,9 @@ def main(args, timer): dataset_train, num_classes = get_dataset(is_train=True, args=args) dataset_test, _ = get_dataset(is_train=False, args=args) - ## SUBSET FOR TESTING EPOCH ROLLOVER - dataset_train = torch.utils.data.Subset(dataset_train, torch.arange(450)) - dataset_test = torch.utils.data.Subset(dataset_test, torch.arange(108)) + # ## SUBSET FOR TESTING EPOCH ROLLOVER + # dataset_train = torch.utils.data.Subset(dataset_train, torch.arange(1000)) + # dataset_test = torch.utils.data.Subset(dataset_test, torch.arange(500)) timer.report('loading data') @@ -137,7 +137,6 @@ def main(args, timer): # train_batch_sampler = torch.utils.data.BatchSampler(train_sampler, args.batch_size, drop_last=True) group_ids = create_aspect_ratio_groups(dataset_train, k=args.aspect_ratio_group_factor) - train_sampler = InterruptableDistributedGroupedBatchSampler(dataset_train, group_ids, args.batch_size) test_sampler = InterruptableDistributedSampler(dataset_test) @@ -228,13 +227,13 @@ def main(args, timer): timer.report('learning rate schedulers') - from coco_eval import CocoEvaluator - from coco_utils import get_coco_api_from_dataset - coco = get_coco_api_from_dataset(data_loader_test.dataset) - iou_types = _get_iou_types(model) - coco_evaluator = CocoEvaluator(coco, iou_types) + # from coco_eval import CocoEvaluator + # from coco_utils import get_coco_api_from_dataset + # coco = get_coco_api_from_dataset(data_loader_test.dataset) + # iou_types = _get_iou_types(model) + # coco_evaluator = CocoEvaluator(coco, iou_types) - timer.report('init coco evaluator') + # timer.report('init coco evaluator') Path(args.resume).parent.mkdir(parents=True, exist_ok=True) if args.resume and os.path.isfile(args.resume): @@ -252,16 +251,15 @@ def main(args, timer): test_sampler.load_state_dict(checkpoint["test_sampler"]) - # Evaluator state variables - coco_evaluator.coco_gt = checkpoint["coco_gt"] - coco_evaluator.iou_types = checkpoint["iou_types"] - coco_evaluator.coco_eval = checkpoint["coco_eval"] - coco_evaluator.img_ids = checkpoint["img_ids"] - coco_evaluator.eval_imgs = checkpoint["eval_imgs"] + # # Evaluator state variables + # coco_evaluator.coco_gt = checkpoint["coco_gt"] + # coco_evaluator.iou_types = checkpoint["iou_types"] + # coco_evaluator.coco_eval = checkpoint["coco_eval"] + # coco_evaluator.img_ids = checkpoint["img_ids"] + # coco_evaluator.eval_imgs = checkpoint["eval_imgs"] timer.report('retrieving checkpoint') - # KILL THIS FOR NOW if args.test_only: # We disable the cudnn benchmarking because it can noticeably affect the accuracy @@ -278,11 +276,12 @@ def main(args, timer): with train_sampler.in_epoch(epoch): timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch - metric_logger, timer = train_one_epoch(model, optimizer, data_loader_train, train_sampler, test_sampler, lr_scheduler, warmup_lr_scheduler, args, device, coco_evaluator, epoch, args.print_freq, scaler, timer) + metric_logger, timer = train_one_epoch(model, optimizer, data_loader_train, train_sampler, test_sampler, lr_scheduler, warmup_lr_scheduler, args, device, epoch, scaler, timer) - with test_sampler.in_epoch(epoch): - timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch - coco_evaluator, timer = evaluate(model, data_loader_test, epoch, test_sampler, args, coco_evaluator, optimizer, lr_scheduler, warmup_lr_scheduler, train_sampler, device, scaler, timer) + # # NEST THE TEST SAMPLER IN TRAIN SAMPLER CONTEXT TO AVOID EPOCH RESTART? + # with test_sampler.in_epoch(epoch): + # timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch + # coco_evaluator, timer = evaluate(model, data_loader_test, epoch, test_sampler, args, coco_evaluator, optimizer, lr_scheduler, warmup_lr_scheduler, train_sampler, device, scaler, timer) def get_args_parser(add_help=True): import argparse diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index b3747571..55e92cbc 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -89,7 +89,7 @@ def evaluate( confmat_temp = utils.ConfusionMatrix(num_classes) confmat_temp.update(target.flatten().detach().cpu(), output.argmax(1).flatten().detach().cpu()) confmat_temp.reduce_from_all_processes() - confmat.mat += confmat_temp.mat.cpu() # Add to the running eval matrix + confmat.mat += confmat_temp.mat # Add to the running eval matrix # FIXME need to take into account that the datasets # could have been padded in distributed setup @@ -132,19 +132,6 @@ def evaluate( "Setting the world size to 1 is always a safe bet." ) - # # Save final checkpoint once all eval ops have completed - MIGHT BE ABLE TO OMIT THIS ALTOGETHER - # if utils.is_main_process(): - # print(f"Saving checkpoint at end of epoch {epoch} evaluation") - # checkpoint = { - # "mode": "train", - # "model": model.module.state_dict(), - # "epoch": epoch, - # "args": args, - # "test_sampler": test_sampler.state_dict(), - # "conf_mat": confmat, # For storing eval metric - # } - # timer = atomic_torch_save(checkpoint, args.resume, timer) - return confmat, timer @@ -203,28 +190,12 @@ def train_one_epoch( "lr_scheduler": lr_scheduler.state_dict(), "train_sampler": train_sampler.state_dict(), "test_sampler": test_sampler.state_dict(), - "confmat": confmat, # For storing eval metric + "confmat": confmat.mat, # For storing eval metric } if args.amp: checkpoint["scaler"] = scaler.state_dict() timer = atomic_torch_save(checkpoint, args.resume, timer) - # # Saving checkpoint at the end of training epoch - MIGHT BE ABLE TO OMIT THIS ALTOGETHER - # if utils.is_main_process(): # Checkpointing every batch? - # print(f"Saving checkpoint at end of epoch {epoch} training") - # checkpoint = { - # "mode": "evaluate", - # "model": model.module.state_dict(), - # "optimizer": optimizer.state_dict(), - # "lr_scheduler": lr_scheduler.state_dict(), - # "epoch": epoch, - # "args": args, - # "train_sampler": train_sampler.state_dict(), - # } - # if args.amp: - # checkpoint["scaler"] = scaler.state_dict() - # timer = atomic_torch_save(checkpoint, args.resume, timer) - return metric_logger, timer timer.report('defined other functions') @@ -252,19 +223,19 @@ def main(args, timer): dataset_train, num_classes = get_dataset(args.data_path, args.dataset, "train", get_transform(True, args)) dataset_test, _ = get_dataset(args.data_path, args.dataset, "val", get_transform(False, args)) - ## SUBSET FOR TESTING EPOCH ROLLOVER - dataset_train = torch.utils.data.Subset(dataset_train, torch.arange(450)) - dataset_test = torch.utils.data.Subset(dataset_test, torch.arange(108)) + # ## SUBSET FOR TESTING EPOCH ROLLOVER + # dataset_train = torch.utils.data.Subset(dataset_train, torch.arange(1000)) + # dataset_test = torch.utils.data.Subset(dataset_test, torch.arange(500)) timer.report('loading data') # if args.distributed: - train_sampler = InterruptableDistributedSampler(dataset_train) - test_sampler = InterruptableDistributedSampler(dataset_test) - # test_sampler = torch.utils.data.distributed.DistributedSampler(dataset_test, shuffle=False) + # test_sampler = torch.utils.data.distributed.DistributedSampler(dataset_test, shuffle=False) # else: # train_sampler = torch.utils.data.RandomSampler(dataset) # test_sampler = torch.utils.data.SequentialSampler(dataset_test) + train_sampler = InterruptableDistributedSampler(dataset_train) + test_sampler = InterruptableDistributedSampler(dataset_test) timer.report('creating data samplers') @@ -377,10 +348,12 @@ def main(args, timer): metric_logger, timer = train_one_epoch(model, criterion, optimizer, data_loader_train, train_sampler, test_sampler, confmat, lr_scheduler, device, epoch, args.print_freq, scaler, timer) timer.report(f'training for epoch {epoch}') - with test_sampler.in_epoch(epoch): - timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch - confmat, timer = evaluate(model, data_loader_test, num_classes, confmat, optimizer, lr_scheduler, train_sampler, test_sampler, device, epoch, args.print_freq, scaler, timer) - timer.report(f'evaluation for epoch {epoch}') + # NEST TEST SAMPLER IN TRAIN SAMPLER CONTEXT TO AVOID EPOCH RESTART? + with test_sampler.in_epoch(epoch): + timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch + confmat, timer = evaluate(model, data_loader_test, num_classes, confmat, optimizer, lr_scheduler, train_sampler, test_sampler, device, epoch, args.print_freq, scaler, timer) + confmat = utils.ConfusionMatrix(num_classes) # reset the evaluation + timer.report(f'evaluation for epoch {epoch}') def get_args_parser(add_help=True): From d44cd5bf491b3a67c0b3e14319da1ac9eec77984 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Sun, 10 Sep 2023 21:45:27 +0000 Subject: [PATCH 15/44] minor updates --- tv-detection/engine.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tv-detection/engine.py b/tv-detection/engine.py index 152def96..172156b2 100644 --- a/tv-detection/engine.py +++ b/tv-detection/engine.py @@ -148,6 +148,7 @@ def evaluate( timer.report(f'Epoch {epoch} batch: {test_step} outputs back to cpu') res = {target["image_id"]: output for target, output in zip(targets, outputs)} + # res = {img_id: dict['boxes', 'labels', 'scores', 'masks'], ...} evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time From 62df2d47f92b460888615f4f180726a7f4b74ae4 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Sun, 10 Sep 2023 23:19:38 +0000 Subject: [PATCH 16/44] updates thinking about checkpointing evaluation better --- tv-detection/coco_eval.py | 16 ++++++--- tv-detection/engine.py | 39 +++++++++++---------- tv-detection/train_cycling.py | 60 ++++++++++++++++---------------- tv-segmentation/train_cycling.py | 19 +++++----- tv-segmentation/utils.py | 17 ++++++--- 5 files changed, 85 insertions(+), 66 deletions(-) diff --git a/tv-detection/coco_eval.py b/tv-detection/coco_eval.py index ba1359f8..50b6246a 100644 --- a/tv-detection/coco_eval.py +++ b/tv-detection/coco_eval.py @@ -26,11 +26,18 @@ def __init__(self, coco_gt, iou_types): self.eval_imgs = {k: [] for k in iou_types} def update(self, predictions): - img_ids = list(np.unique(list(predictions.keys()))) - self.img_ids.extend(img_ids) - - for iou_type in self.iou_types: + ''' + predictions = { + image_id: {"boxes": tensor(n x 4), "labels": tensor(n,), "scores": tensor(n,), "masks": tensor(n x 1 x H x W)}, # for n < N detections + ... for image_id in batch + } + ''' + img_ids = list(np.unique(list(predictions.keys()))) # images seen this batch + self.img_ids.extend(img_ids) # catalogue of all images seen + + for iou_type in self.iou_types: # either ["bbox"] or ["bbox", "segm"] or ["bbox", "keypoints"] results = self.prepare(predictions, iou_type) + with redirect_stdout(io.StringIO()): coco_dt = COCO.loadRes(self.coco_gt, results) if results else COCO() coco_eval = self.coco_eval[iou_type] @@ -69,7 +76,6 @@ def prepare_for_coco_detection(self, predictions): for original_id, prediction in predictions.items(): if len(prediction) == 0: continue - boxes = prediction["boxes"] boxes = convert_to_xywh(boxes).tolist() scores = prediction["scores"].tolist() diff --git a/tv-detection/engine.py b/tv-detection/engine.py index 172156b2..d4d9508c 100644 --- a/tv-detection/engine.py +++ b/tv-detection/engine.py @@ -11,7 +11,7 @@ def train_one_epoch( model, optimizer, data_loader_train, train_sampler, test_sampler, - lr_scheduler, warmup_lr_scheduler, args, device, + lr_scheduler, warmup_lr_scheduler, args, device, coco_evaluator, epoch, scaler=None, timer=None ): @@ -78,12 +78,13 @@ def train_one_epoch( "train_sampler": train_sampler.state_dict(), "test_sampler": test_sampler.state_dict(), - # # Evaluator state variables - # "coco_gt": coco_evaluator.coco_gt, - # "iou_types": coco_evaluator.iou_types, - # "coco_eval": coco_evaluator.coco_eval, - # "img_ids": coco_evaluator.img_ids, - # "eval_imgs": coco_evaluator.eval_imgs, + # Evaluator state variables + "img_ids": coco_evaluator.img_ids, # catalogue of images seen already + "eval_imgs": coco_evaluator.eval_imgs, # image evaluations + + # "coco_gt": coco_evaluator.coco_gt, # passed in at init + # "iou_types": coco_evaluator.iou_types, # passed in at init + # "coco_eval": coco_evaluator.coco_eval, # generated at init } if args.amp: checkpoint["scaler"] = scaler.state_dict() @@ -124,10 +125,6 @@ def evaluate( timer.report(f'evaluation preliminaries') - # coco = get_coco_api_from_dataset(data_loader_test.dataset) - # iou_types = _get_iou_types(model) - # coco_evaluator = CocoEvaluator(coco, iou_types) - test_step = test_sampler.progress // data_loader_test.batch_size print(f'\nEvaluating / resuming epoch {epoch} from eval step {test_step}\n') timer.report('launch evaluation routine') @@ -148,7 +145,7 @@ def evaluate( timer.report(f'Epoch {epoch} batch: {test_step} outputs back to cpu') res = {target["image_id"]: output for target, output in zip(targets, outputs)} - # res = {img_id: dict['boxes', 'labels', 'scores', 'masks'], ...} + # res = {img_id: {'boxes': T, 'labels': T, 'scores': T, 'masks': T}, ...} evaluator_time = time.time() coco_evaluator.update(res) evaluator_time = time.time() - evaluator_time @@ -171,11 +168,13 @@ def evaluate( "test_sampler": test_sampler.state_dict(), # Evaluator state variables - "coco_gt": coco_evaluator.coco_gt, - "iou_types": coco_evaluator.iou_types, - "coco_eval": coco_evaluator.coco_eval, - "img_ids": coco_evaluator.img_ids, - "eval_imgs": coco_evaluator.eval_imgs, + "img_ids": coco_evaluator.img_ids, # catalogue of images seen already + "eval_imgs": coco_evaluator.eval_imgs, # image evaluations + + # "coco_gt": coco_evaluator.coco_gt, # passed in at init + # "iou_types": coco_evaluator.iou_types, # passed in at init + # "coco_eval": coco_evaluator.coco_eval, # generated at init + } if args.amp: checkpoint["scaler"] = scaler.state_dict() @@ -191,6 +190,10 @@ def evaluate( coco_evaluator.summarize() torch.set_num_threads(n_threads) - timer.report(f'evaluator accumulation and summarization') + coco = get_coco_api_from_dataset(data_loader_test.dataset) + iou_types = _get_iou_types(model) + coco_evaluator = CocoEvaluator(coco, iou_types) + + timer.report(f'evaluator accumulation, summarization, and reset') return coco_evaluator, timer diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index 66ed7369..19abe7cf 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -76,16 +76,16 @@ def get_transform(is_train, args): else: return presets.DetectionPresetEval(backend=args.backend, use_v2=args.use_v2) -# def _get_iou_types(model): -# model_without_ddp = model -# if isinstance(model, torch.nn.parallel.DistributedDataParallel): -# model_without_ddp = model.module -# iou_types = ["bbox"] -# if isinstance(model_without_ddp, torchvision.models.detection.MaskRCNN): -# iou_types.append("segm") -# if isinstance(model_without_ddp, torchvision.models.detection.KeypointRCNN): -# iou_types.append("keypoints") -# return iou_types +def _get_iou_types(model): # intersection over union (iou) types + model_without_ddp = model + if isinstance(model, torch.nn.parallel.DistributedDataParallel): + model_without_ddp = model.module + iou_types = ["bbox"] + if isinstance(model_without_ddp, torchvision.models.detection.MaskRCNN): + iou_types.append("segm") + if isinstance(model_without_ddp, torchvision.models.detection.KeypointRCNN): + iou_types.append("keypoints") + return iou_types timer.report('defined other functions') @@ -117,9 +117,9 @@ def main(args, timer): dataset_train, num_classes = get_dataset(is_train=True, args=args) dataset_test, _ = get_dataset(is_train=False, args=args) - # ## SUBSET FOR TESTING EPOCH ROLLOVER - # dataset_train = torch.utils.data.Subset(dataset_train, torch.arange(1000)) - # dataset_test = torch.utils.data.Subset(dataset_test, torch.arange(500)) + ## SUBSET FOR TESTING EPOCH ROLLOVER + dataset_train = torch.utils.data.Subset(dataset_train, torch.arange(500)) + dataset_test = torch.utils.data.Subset(dataset_test, torch.arange(200)) timer.report('loading data') @@ -227,13 +227,13 @@ def main(args, timer): timer.report('learning rate schedulers') - # from coco_eval import CocoEvaluator - # from coco_utils import get_coco_api_from_dataset - # coco = get_coco_api_from_dataset(data_loader_test.dataset) - # iou_types = _get_iou_types(model) - # coco_evaluator = CocoEvaluator(coco, iou_types) + from coco_eval import CocoEvaluator + from coco_utils import get_coco_api_from_dataset + coco = get_coco_api_from_dataset(data_loader_test.dataset) + iou_types = _get_iou_types(model) + coco_evaluator = CocoEvaluator(coco, iou_types) - # timer.report('init coco evaluator') + timer.report('init coco evaluator') Path(args.resume).parent.mkdir(parents=True, exist_ok=True) if args.resume and os.path.isfile(args.resume): @@ -251,12 +251,12 @@ def main(args, timer): test_sampler.load_state_dict(checkpoint["test_sampler"]) - # # Evaluator state variables - # coco_evaluator.coco_gt = checkpoint["coco_gt"] - # coco_evaluator.iou_types = checkpoint["iou_types"] - # coco_evaluator.coco_eval = checkpoint["coco_eval"] - # coco_evaluator.img_ids = checkpoint["img_ids"] - # coco_evaluator.eval_imgs = checkpoint["eval_imgs"] + # Evaluator state variables + # coco_evaluator.coco_gt = checkpoint["coco_gt"] # passed in at init + # coco_evaluator.iou_types = checkpoint["iou_types"] # passed in at init + # coco_evaluator.coco_eval = checkpoint["coco_eval"] # generated at init + coco_evaluator.img_ids = checkpoint["img_ids"] + coco_evaluator.eval_imgs = checkpoint["eval_imgs"] timer.report('retrieving checkpoint') @@ -276,12 +276,12 @@ def main(args, timer): with train_sampler.in_epoch(epoch): timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch - metric_logger, timer = train_one_epoch(model, optimizer, data_loader_train, train_sampler, test_sampler, lr_scheduler, warmup_lr_scheduler, args, device, epoch, scaler, timer) + metric_logger, timer = train_one_epoch(model, optimizer, data_loader_train, train_sampler, test_sampler, lr_scheduler, warmup_lr_scheduler, args, device, coco_evaluator, epoch, scaler, timer) - # # NEST THE TEST SAMPLER IN TRAIN SAMPLER CONTEXT TO AVOID EPOCH RESTART? - # with test_sampler.in_epoch(epoch): - # timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch - # coco_evaluator, timer = evaluate(model, data_loader_test, epoch, test_sampler, args, coco_evaluator, optimizer, lr_scheduler, warmup_lr_scheduler, train_sampler, device, scaler, timer) + # NEST THE TEST SAMPLER IN TRAIN SAMPLER CONTEXT TO AVOID EPOCH RESTART? + with test_sampler.in_epoch(epoch): + timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch + coco_evaluator, timer = evaluate(model, data_loader_test, epoch, test_sampler, args, coco_evaluator, optimizer, lr_scheduler, warmup_lr_scheduler, train_sampler, device, scaler, timer) def get_args_parser(add_help=True): import argparse diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index 55e92cbc..cbce1a57 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -86,10 +86,8 @@ def evaluate( output = output["out"] timer.report(f'Epoch {epoch} batch: {test_step} forward through model') - confmat_temp = utils.ConfusionMatrix(num_classes) - confmat_temp.update(target.flatten().detach().cpu(), output.argmax(1).flatten().detach().cpu()) - confmat_temp.reduce_from_all_processes() - confmat.mat += confmat_temp.mat # Add to the running eval matrix + confmat.update(target.flatten().detach().cpu(), output.argmax(1).flatten().detach().cpu()) + confmat.reduce_from_all_processes() # FIXME need to take into account that the datasets # could have been padded in distributed setup @@ -110,13 +108,16 @@ def evaluate( "lr_scheduler": lr_scheduler.state_dict(), "train_sampler": train_sampler.state_dict(), "test_sampler": test_sampler.state_dict(), + "confmat": confmat.mat, # For storing eval metric + "confmat_temp": confmat.temp_mat, # For storing eval metric } if args.amp: checkpoint["scaler"] = scaler.state_dict() timer = atomic_torch_save(checkpoint, args.resume, timer) print(confmat) + confmat.reset() num_processed_samples = utils.reduce_across_processes(num_processed_samples) if ( @@ -190,7 +191,9 @@ def train_one_epoch( "lr_scheduler": lr_scheduler.state_dict(), "train_sampler": train_sampler.state_dict(), "test_sampler": test_sampler.state_dict(), + "confmat": confmat.mat, # For storing eval metric + "confmat_temp": confmat.temp_mat, # For storing eval metric } if args.amp: checkpoint["scaler"] = scaler.state_dict() @@ -223,9 +226,9 @@ def main(args, timer): dataset_train, num_classes = get_dataset(args.data_path, args.dataset, "train", get_transform(True, args)) dataset_test, _ = get_dataset(args.data_path, args.dataset, "val", get_transform(False, args)) - # ## SUBSET FOR TESTING EPOCH ROLLOVER - # dataset_train = torch.utils.data.Subset(dataset_train, torch.arange(1000)) - # dataset_test = torch.utils.data.Subset(dataset_test, torch.arange(500)) + ## SUBSET FOR TESTING EPOCH ROLLOVER + dataset_train = torch.utils.data.Subset(dataset_train, torch.arange(500)) + dataset_test = torch.utils.data.Subset(dataset_test, torch.arange(200)) timer.report('loading data') @@ -326,6 +329,7 @@ def main(args, timer): test_sampler.load_state_dict(checkpoint["test_sampler"]) confmat.mat = checkpoint["confmat"] + confmat.temp_mat = checkpoint["confmat_temp"] timer.report('retrieving checkpoint') @@ -352,7 +356,6 @@ def main(args, timer): with test_sampler.in_epoch(epoch): timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch confmat, timer = evaluate(model, data_loader_test, num_classes, confmat, optimizer, lr_scheduler, train_sampler, test_sampler, device, epoch, args.print_freq, scaler, timer) - confmat = utils.ConfusionMatrix(num_classes) # reset the evaluation timer.report(f'evaluation for epoch {epoch}') diff --git a/tv-segmentation/utils.py b/tv-segmentation/utils.py index 31e7bb42..21255602 100644 --- a/tv-segmentation/utils.py +++ b/tv-segmentation/utils.py @@ -66,8 +66,10 @@ def __str__(self): class ConfusionMatrix: def __init__(self, num_classes): self.num_classes = num_classes - # self.mat = None - self.mat = torch.zeros((num_classes, num_classes), dtype=torch.int64, device='cpu') + # temp_mat will accumulate results from images seen on this node + self.temp_mat = torch.zeros((num_classes, num_classes), dtype=torch.int64, device='cpu', requires_grad=False) + # mat will then store the accumulation of all temp_mats, avoiding multiple-counting + self.mat = torch.zeros((num_classes, num_classes), dtype=torch.int64, device='cpu', requires_grad=False) def update(self, a, b): n = self.num_classes @@ -76,9 +78,10 @@ def update(self, a, b): with torch.inference_mode(): k = (a >= 0) & (a < n) inds = n * a[k].to(torch.int64) + b[k] - self.mat += torch.bincount(inds, minlength=n**2).reshape(n, n) + self.temp_mat += torch.bincount(inds, minlength=n**2).reshape(n, n) def reset(self): + self.temp_mat.zero_() self.mat.zero_() def compute(self): @@ -89,7 +92,11 @@ def compute(self): return acc_global, acc, iu def reduce_from_all_processes(self): - reduce_across_processes(self.mat) + reduce_across_processes(self.temp_mat) + # add the accumulated results from all nodes + self.mat += self.temp_mat.clone() + # reset temp_mat + self.temp_mat.zero_() def __str__(self): acc_global, acc, iu = self.compute() @@ -297,5 +304,5 @@ def reduce_across_processes(val): t = torch.tensor(val, device="cuda") dist.barrier() - dist.all_reduce(t) + dist.all_reduce(t) # default all_reduce op is SUM return t From 60db14ff4195e30d9ab2522e6b24e2c4f21077e8 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Sun, 10 Sep 2023 23:41:14 +0000 Subject: [PATCH 17/44] minor updates and full training kickoffs --- tv-detection/train_cycling.py | 10 +++------- tv-segmentation/train_cycling.py | 6 +++--- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index 19abe7cf..a05c1ea2 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -117,9 +117,9 @@ def main(args, timer): dataset_train, num_classes = get_dataset(is_train=True, args=args) dataset_test, _ = get_dataset(is_train=False, args=args) - ## SUBSET FOR TESTING EPOCH ROLLOVER - dataset_train = torch.utils.data.Subset(dataset_train, torch.arange(500)) - dataset_test = torch.utils.data.Subset(dataset_test, torch.arange(200)) + # ## SUBSET FOR TESTING + # dataset_train = torch.utils.data.Subset(dataset_train, torch.arange(500)) + # dataset_test = torch.utils.data.Subset(dataset_test, torch.arange(200)) timer.report('loading data') @@ -252,15 +252,11 @@ def main(args, timer): test_sampler.load_state_dict(checkpoint["test_sampler"]) # Evaluator state variables - # coco_evaluator.coco_gt = checkpoint["coco_gt"] # passed in at init - # coco_evaluator.iou_types = checkpoint["iou_types"] # passed in at init - # coco_evaluator.coco_eval = checkpoint["coco_eval"] # generated at init coco_evaluator.img_ids = checkpoint["img_ids"] coco_evaluator.eval_imgs = checkpoint["eval_imgs"] timer.report('retrieving checkpoint') - # KILL THIS FOR NOW if args.test_only: # We disable the cudnn benchmarking because it can noticeably affect the accuracy torch.backends.cudnn.benchmark = False diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index cbce1a57..1adda6d8 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -226,9 +226,9 @@ def main(args, timer): dataset_train, num_classes = get_dataset(args.data_path, args.dataset, "train", get_transform(True, args)) dataset_test, _ = get_dataset(args.data_path, args.dataset, "val", get_transform(False, args)) - ## SUBSET FOR TESTING EPOCH ROLLOVER - dataset_train = torch.utils.data.Subset(dataset_train, torch.arange(500)) - dataset_test = torch.utils.data.Subset(dataset_test, torch.arange(200)) + # ## SUBSET FOR TESTING + # dataset_train = torch.utils.data.Subset(dataset_train, torch.arange(500)) + # dataset_test = torch.utils.data.Subset(dataset_test, torch.arange(200)) timer.report('loading data') From 0b7c93200c44eb797c99e51af6b2689899bee20c Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Wed, 13 Sep 2023 14:18:15 +1000 Subject: [PATCH 18/44] monai autoencoder traininig interruptably --- cycling_utils/cycling_utils/timer.py | 26 +- monai/brats_mri_2d/backup/LICENSE | 201 +++++++ monai/brats_mri_2d/backup/brats_mri_2d.isc | 10 + .../backup/configs/inference.json | 103 ++++ .../backup/configs/inference_autoencoder.json | 156 ++++++ .../brats_mri_2d/backup/configs/logging.conf | 21 + .../brats_mri_2d/backup/configs/metadata.json | 110 ++++ .../configs/multi_gpu_train_autoencoder.json | 43 ++ .../configs/multi_gpu_train_diffusion.json | 16 + .../backup/configs/train_autoencoder.json | 151 +++++ .../backup/configs/train_diffusion.json | 174 ++++++ monai/brats_mri_2d/backup/docs/README.md | 166 ++++++ .../brats_mri_2d/backup/docs/data_license.txt | 49 ++ monai/brats_mri_2d/backup/scripts/__init__.py | 12 + .../backup/scripts/ldm_sampler.py | 60 ++ .../backup/scripts/ldm_trainer.py | 380 +++++++++++++ monai/brats_mri_2d/backup/scripts/losses.py | 52 ++ monai/brats_mri_2d/backup/scripts/utils.py | 50 ++ monai/brats_mri_2d/brats_mri_2d_diff.isc | 6 + monai/brats_mri_2d/brats_mri_2d_gen.isc | 6 + monai/brats_mri_2d/loops.py | 282 ++++++++++ monai/brats_mri_2d/models_DIS.py | 23 + monai/brats_mri_2d/prep.py | 4 + monai/brats_mri_2d/train_cycling_diff.py | 260 +++++++++ monai/brats_mri_2d/train_cycling_gen.py | 279 ++++++++++ monai/brats_mri_2d/utils.py | 72 +++ tv-detection/SS/COCO_edits/COCO_interface.py | 521 ++++++++++++++++++ tv-detection/SS/COCO_edits/coco_utils_SC.py | 234 ++++++++ .../SS/COCO_edits/group_by_aspect_ratio_SC.py | 198 +++++++ tv-detection/maskrcnn_resnet50_fpn_small.isc | 6 + tv-detection/train_cycling_small.py | 335 +++++++++++ 31 files changed, 4003 insertions(+), 3 deletions(-) create mode 100644 monai/brats_mri_2d/backup/LICENSE create mode 100644 monai/brats_mri_2d/backup/brats_mri_2d.isc create mode 100644 monai/brats_mri_2d/backup/configs/inference.json create mode 100644 monai/brats_mri_2d/backup/configs/inference_autoencoder.json create mode 100644 monai/brats_mri_2d/backup/configs/logging.conf create mode 100644 monai/brats_mri_2d/backup/configs/metadata.json create mode 100644 monai/brats_mri_2d/backup/configs/multi_gpu_train_autoencoder.json create mode 100644 monai/brats_mri_2d/backup/configs/multi_gpu_train_diffusion.json create mode 100644 monai/brats_mri_2d/backup/configs/train_autoencoder.json create mode 100644 monai/brats_mri_2d/backup/configs/train_diffusion.json create mode 100644 monai/brats_mri_2d/backup/docs/README.md create mode 100644 monai/brats_mri_2d/backup/docs/data_license.txt create mode 100644 monai/brats_mri_2d/backup/scripts/__init__.py create mode 100644 monai/brats_mri_2d/backup/scripts/ldm_sampler.py create mode 100644 monai/brats_mri_2d/backup/scripts/ldm_trainer.py create mode 100644 monai/brats_mri_2d/backup/scripts/losses.py create mode 100644 monai/brats_mri_2d/backup/scripts/utils.py create mode 100644 monai/brats_mri_2d/brats_mri_2d_diff.isc create mode 100644 monai/brats_mri_2d/brats_mri_2d_gen.isc create mode 100644 monai/brats_mri_2d/loops.py create mode 100644 monai/brats_mri_2d/models_DIS.py create mode 100644 monai/brats_mri_2d/prep.py create mode 100644 monai/brats_mri_2d/train_cycling_diff.py create mode 100644 monai/brats_mri_2d/train_cycling_gen.py create mode 100644 monai/brats_mri_2d/utils.py create mode 100644 tv-detection/SS/COCO_edits/COCO_interface.py create mode 100644 tv-detection/SS/COCO_edits/coco_utils_SC.py create mode 100644 tv-detection/SS/COCO_edits/group_by_aspect_ratio_SC.py create mode 100644 tv-detection/maskrcnn_resnet50_fpn_small.isc create mode 100644 tv-detection/train_cycling_small.py diff --git a/cycling_utils/cycling_utils/timer.py b/cycling_utils/cycling_utils/timer.py index 290d9bd6..788abd86 100644 --- a/cycling_utils/cycling_utils/timer.py +++ b/cycling_utils/cycling_utils/timer.py @@ -2,13 +2,33 @@ import os class Timer: - def __init__(self, start_time=None, running=0): + def __init__(self, report=None, start_time=None, running=0): self.start_time = start_time if start_time is not None else time.time() self.running = running + if str(os.environ["RANK"]) == "0": + report = report if report else "Start" + print("{:<70}{:>12} ms, {:>12} s total".format(report, f'{0.0:,.3f}', f'{0.0:,.2f}')) def report(self, annot): if str(os.environ["RANK"]) == "0": now = time.time() duration = now - self.start_time self.running += duration - print("Completed {:<70}{:>12} milliseconds, {:>12} seconds total".format(annot, f'{1000*duration:,.3f}', f'{self.running:,.2f}')) - self.start_time = now \ No newline at end of file + print("Completed {:<70}{:>12} ms, {:>12} s total".format(annot, f'{1000*duration:,.3f}', f'{self.running:,.2f}')) + self.start_time = now + +# import os +# from dateime import datetime +# class Timer: +# def __init__(self, report=None, start_time=None, running=0): +# self.start_time = start_time if start_time is not None else datetime.now() +# self.running = running +# if str(os.environ["RANK"]) == "0": +# report = report if report else "Start" +# print("{:<30} {:<70}{:>12} ms, {:>12} s total".format(self.start_time, report, f'{0.0:,.3f}', f'{0.0:,.2f}')) +# def report(self, annot): +# if str(os.environ["RANK"]) == "0": +# now = datetime.now() +# duration = (now - self.start_time).total_seconds() +# self.running += duration +# print("{:<30} Completed {:<70}{:>12} ms, {:>12} s total".format(now, annot, f'{1000*duration:,.3f}', f'{self.running:,.2f}')) +# self.start_time = now \ No newline at end of file diff --git a/monai/brats_mri_2d/backup/LICENSE b/monai/brats_mri_2d/backup/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/monai/brats_mri_2d/backup/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/monai/brats_mri_2d/backup/brats_mri_2d.isc b/monai/brats_mri_2d/backup/brats_mri_2d.isc new file mode 100644 index 00000000..631a6738 --- /dev/null +++ b/monai/brats_mri_2d/backup/brats_mri_2d.isc @@ -0,0 +1,10 @@ +experiment_name="brats_mri_2d" +gpu_type="24GB VRAM GPU" +nnodes = 11 +venv_path = "~/.venv/bin/activate" +output_path = "~/output_brats_mri_2d" + + +torchrun --standalone --nnodes=1 --nproc_per_node=8 -m monai.bundle run --config_file "['configs/train_autoencoder.json','configs/multi_gpu_train_autoencoder.json']" --lr 4e-4 + +command="train_cycling.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/Datasets/Open-Datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/monai/brats_mri_2d/backup/configs/inference.json b/monai/brats_mri_2d/backup/configs/inference.json new file mode 100644 index 00000000..9e9e74b2 --- /dev/null +++ b/monai/brats_mri_2d/backup/configs/inference.json @@ -0,0 +1,103 @@ +{ + "imports": [ + "$import torch", + "$from datetime import datetime", + "$from pathlib import Path", + "$from PIL import Image", + "$from scripts.utils import visualize_2d_image" + ], + "bundle_root": ".", + "model_dir": "$@bundle_root + '/models'", + "output_dir": "$@bundle_root + '/output'", + "create_output_dir": "$Path(@output_dir).mkdir(exist_ok=True)", + "device": "$torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')", + "output_postfix": "$datetime.now().strftime('sample_%Y%m%d_%H%M%S')", + "channel": 0, + "spatial_dims": 2, + "image_channels": 1, + "latent_channels": 1, + "latent_shape": [ + "@latent_channels", + 64, + 64 + ], + "autoencoder_def": { + "_target_": "generative.networks.nets.AutoencoderKL", + "spatial_dims": "@spatial_dims", + "in_channels": "@image_channels", + "out_channels": "@image_channels", + "latent_channels": "@latent_channels", + "num_channels": [ + 64, + 128, + 256 + ], + "num_res_blocks": 2, + "norm_num_groups": 32, + "norm_eps": 1e-06, + "attention_levels": [ + false, + false, + false + ], + "with_encoder_nonlocal_attn": true, + "with_decoder_nonlocal_attn": true + }, + "network_def": { + "_target_": "generative.networks.nets.DiffusionModelUNet", + "spatial_dims": "@spatial_dims", + "in_channels": "@latent_channels", + "out_channels": "@latent_channels", + "num_channels": [ + 32, + 64, + 128, + 256 + ], + "attention_levels": [ + false, + true, + true, + true + ], + "num_head_channels": [ + 0, + 32, + 32, + 32 + ], + "num_res_blocks": 2 + }, + "load_autoencoder_path": "$@bundle_root + '/models/model_autoencoder.pt'", + "load_autoencoder": "$@autoencoder_def.load_state_dict(torch.load(@load_autoencoder_path))", + "autoencoder": "$@autoencoder_def.to(@device)", + "load_diffusion_path": "$@model_dir + '/model.pt'", + "load_diffusion": "$@network_def.load_state_dict(torch.load(@load_diffusion_path))", + "diffusion": "$@network_def.to(@device)", + "noise_scheduler": { + "_target_": "generative.networks.schedulers.DDIMScheduler", + "_requires_": [ + "@load_diffusion", + "@load_autoencoder" + ], + "num_train_timesteps": 1000, + "beta_start": 0.0015, + "beta_end": 0.0195, + "schedule": "scaled_linear_beta", + "clip_sample": false + }, + "noise": "$torch.randn([1]+@latent_shape).to(@device)", + "set_timesteps": "$@noise_scheduler.set_timesteps(num_inference_steps=50)", + "inferer": { + "_target_": "scripts.ldm_sampler.LDMSampler", + "_requires_": "@set_timesteps" + }, + "sample": "$@inferer.sampling_fn(@noise, @autoencoder, @diffusion, @noise_scheduler)", + "generated_image": "$@sample", + "generated_image_np": "$@generated_image[0,0].cpu().numpy().transpose(1, 0)[::-1, ::-1]", + "img_pil": "$Image.fromarray(visualize_2d_image(@generated_image_np), 'RGB')", + "run": [ + "$@create_output_dir", + "$@img_pil.save(@output_dir+'/synimg_'+@output_postfix+'.png')" + ] +} diff --git a/monai/brats_mri_2d/backup/configs/inference_autoencoder.json b/monai/brats_mri_2d/backup/configs/inference_autoencoder.json new file mode 100644 index 00000000..b31a3ea4 --- /dev/null +++ b/monai/brats_mri_2d/backup/configs/inference_autoencoder.json @@ -0,0 +1,156 @@ +{ + "imports": [ + "$import torch", + "$from datetime import datetime", + "$from pathlib import Path", + "$from PIL import Image", + "$from scripts.utils import visualize_2d_image" + ], + "bundle_root": ".", + "model_dir": "$@bundle_root + '/models'", + "dataset_dir": "/workspace/data/medical", + "output_dir": "$@bundle_root + '/output'", + "create_output_dir": "$Path(@output_dir).mkdir(exist_ok=True)", + "device": "$torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')", + "output_postfix": "$datetime.now().strftime('%Y%m%d_%H%M%S')", + "channel": 0, + "spatial_dims": 2, + "image_channels": 1, + "latent_channels": 1, + "infer_patch_size": [ + 240, + 240 + ], + "infer_batch_size_img": 1, + "infer_batch_size_slice": 1, + "autoencoder_def": { + "_target_": "generative.networks.nets.AutoencoderKL", + "spatial_dims": "@spatial_dims", + "in_channels": "@image_channels", + "out_channels": "@image_channels", + "latent_channels": "@latent_channels", + "num_channels": [ + 64, + 128, + 256 + ], + "num_res_blocks": 2, + "norm_num_groups": 32, + "norm_eps": 1e-06, + "attention_levels": [ + false, + false, + false + ], + "with_encoder_nonlocal_attn": true, + "with_decoder_nonlocal_attn": true + }, + "load_autoencoder_path": "$@bundle_root + '/models/model_autoencoder.pt'", + "load_autoencoder": "$@autoencoder_def.load_state_dict(torch.load(@load_autoencoder_path))", + "autoencoder": "$@autoencoder_def.to(@device)", + "preprocessing_transforms": [ + { + "_target_": "LoadImaged", + "keys": "image" + }, + { + "_target_": "EnsureChannelFirstd", + "keys": "image" + }, + { + "_target_": "Lambdad", + "keys": "image", + "func": "$lambda x: x[@channel, :, :, :]" + }, + { + "_target_": "AddChanneld", + "keys": "image" + }, + { + "_target_": "EnsureTyped", + "keys": "image" + }, + { + "_target_": "Orientationd", + "keys": "image", + "axcodes": "RAS" + }, + { + "_target_": "CenterSpatialCropd", + "keys": "image", + "roi_size": "$[@infer_patch_size[0], @infer_patch_size[1], 20]" + }, + { + "_target_": "ScaleIntensityRangePercentilesd", + "keys": "image", + "lower": 0, + "upper": 100, + "b_min": 0, + "b_max": 1 + } + ], + "crop_transforms": [ + { + "_target_": "DivisiblePadd", + "keys": "image", + "k": [ + 4, + 4, + 1 + ] + }, + { + "_target_": "RandSpatialCropSamplesd", + "keys": "image", + "random_size": false, + "roi_size": "$[@infer_patch_size[0], @infer_patch_size[1], 1]", + "num_samples": "@infer_batch_size_slice" + }, + { + "_target_": "SqueezeDimd", + "keys": "image", + "dim": 3 + } + ], + "final_transforms": [ + { + "_target_": "ScaleIntensityRangePercentilesd", + "keys": "image", + "lower": 0, + "upper": 100, + "b_min": 0, + "b_max": 1 + } + ], + "preprocessing": { + "_target_": "Compose", + "transforms": "$@preprocessing_transforms + @crop_transforms + @final_transforms" + }, + "dataset": { + "_target_": "monai.apps.DecathlonDataset", + "root_dir": "@dataset_dir", + "task": "Task01_BrainTumour", + "section": "validation", + "cache_rate": 0.0, + "num_workers": 8, + "download": false, + "transform": "@preprocessing" + }, + "dataloader": { + "_target_": "DataLoader", + "dataset": "@dataset", + "batch_size": 1, + "shuffle": true, + "num_workers": 0 + }, + "recon_img_pil": "$Image.fromarray(visualize_2d_image(@recon_img), 'RGB')", + "orig_img_pil": "$Image.fromarray(visualize_2d_image(@input_img[0,0,...]), 'RGB')", + "input_img": "$monai.utils.first(@dataloader)['image'].to(@device)", + "recon_img": "$@autoencoder(@input_img)[0][0,0,...]", + "run": [ + "$@create_output_dir", + "$@load_autoencoder", + "$@orig_img_pil.save(@output_dir+'/orig_img_'+@output_postfix+'.png')", + "$@recon_img_pil.save(@output_dir+'/recon_img_'+@output_postfix+'.png')" + ] +} diff --git a/monai/brats_mri_2d/backup/configs/logging.conf b/monai/brats_mri_2d/backup/configs/logging.conf new file mode 100644 index 00000000..91c1a21c --- /dev/null +++ b/monai/brats_mri_2d/backup/configs/logging.conf @@ -0,0 +1,21 @@ +[loggers] +keys=root + +[handlers] +keys=consoleHandler + +[formatters] +keys=fullFormatter + +[logger_root] +level=INFO +handlers=consoleHandler + +[handler_consoleHandler] +class=StreamHandler +level=INFO +formatter=fullFormatter +args=(sys.stdout,) + +[formatter_fullFormatter] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s diff --git a/monai/brats_mri_2d/backup/configs/metadata.json b/monai/brats_mri_2d/backup/configs/metadata.json new file mode 100644 index 00000000..19be69af --- /dev/null +++ b/monai/brats_mri_2d/backup/configs/metadata.json @@ -0,0 +1,110 @@ +{ + "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_generator_ldm_20230507.json", + "version": "1.0.6", + "changelog": { + "1.0.6": "update with new lr scheduler api in inference", + "1.0.5": "fix the wrong GPU index issue of multi-node", + "1.0.4": "update with new lr scheduler api", + "1.0.3": "update required packages", + "1.0.2": "remove unused saver in inference", + "1.0.1": "fix inference folder error", + "1.0.0": "Initial release" + }, + "monai_version": "1.2.0", + "pytorch_version": "1.13.1", + "numpy_version": "1.22.2", + "optional_packages_version": { + "nibabel": "5.1.0", + "lpips": "0.1.4", + "monai-generative": "0.2.2" + }, + "name": "BraTS MRI axial slices latent diffusion generation", + "task": "BraTS MRI axial slices synthesis", + "description": "A generative model for creating 2D brain MRI axial slices from Gaussian noise based on BraTS dataset", + "authors": "MONAI team", + "copyright": "Copyright (c) MONAI Consortium", + "data_source": "http://medicaldecathlon.com/", + "data_type": "nibabel", + "image_classes": "Flair brain MRI axial slices with 1x1 mm voxel size", + "eval_metrics": {}, + "intended_use": "This is a research tool/prototype and not to be used clinically", + "references": [], + "autoencoder_data_format": { + "inputs": { + "image": { + "type": "image", + "format": "image", + "num_channels": 1, + "spatial_shape": [ + 240, + 240 + ], + "dtype": "float32", + "value_range": [ + 0, + 1 + ], + "is_patch_data": true + } + }, + "outputs": { + "pred": { + "type": "image", + "format": "image", + "num_channels": 1, + "spatial_shape": [ + 240, + 240 + ], + "dtype": "float32", + "value_range": [ + 0, + 1 + ], + "is_patch_data": true, + "channel_def": { + "0": "image" + } + } + } + }, + "generator_data_format": { + "inputs": { + "latent": { + "type": "noise", + "format": "image", + "num_channels": 1, + "spatial_shape": [ + 64, + 64 + ], + "dtype": "float32", + "value_range": [ + 0, + 1 + ], + "is_patch_data": true + } + }, + "outputs": { + "pred": { + "type": "feature", + "format": "image", + "num_channels": 1, + "spatial_shape": [ + 64, + 64 + ], + "dtype": "float32", + "value_range": [ + 0, + 1 + ], + "is_patch_data": true, + "channel_def": { + "0": "image" + } + } + } + } +} diff --git a/monai/brats_mri_2d/backup/configs/multi_gpu_train_autoencoder.json b/monai/brats_mri_2d/backup/configs/multi_gpu_train_autoencoder.json new file mode 100644 index 00000000..3fe340be --- /dev/null +++ b/monai/brats_mri_2d/backup/configs/multi_gpu_train_autoencoder.json @@ -0,0 +1,43 @@ +{ + "device": "$torch.device('cuda:' + os.environ['LOCAL_RANK'])", + "gnetwork": { + "_target_": "torch.nn.parallel.DistributedDataParallel", + "module": "$@autoencoder_def.to(@device)", + "device_ids": [ + "@device" + ], + "find_unused_parameters": true + }, + "dnetwork": { + "_target_": "torch.nn.parallel.DistributedDataParallel", + "module": "$@discriminator_def.to(@device)", + "device_ids": [ + "@device" + ], + "find_unused_parameters": true + }, + "train#sampler": { + "_target_": "DistributedSampler", + "dataset": "@train#dataset", + "even_divisible": true, + "shuffle": true + }, + "train#dataloader#sampler": "@train#sampler", + "train#dataloader#shuffle": false, + "train#trainer#train_handlers": "$@train#handlers[: -2 if dist.get_rank() > 0 else None]", + "initialize": [ + "$import torch.distributed as dist", + "$import os", + "$dist.is_initialized() or dist.init_process_group(backend='nccl')", + "$torch.cuda.set_device(@device)", + "$monai.utils.set_determinism(seed=123)", + "$import logging", + "$@train#trainer.logger.setLevel(logging.WARNING if dist.get_rank() > 0 else logging.INFO)" + ], + "run": [ + "$@train#trainer.run()" + ], + "finalize": [ + "$dist.is_initialized() and dist.destroy_process_group()" + ] +} diff --git a/monai/brats_mri_2d/backup/configs/multi_gpu_train_diffusion.json b/monai/brats_mri_2d/backup/configs/multi_gpu_train_diffusion.json new file mode 100644 index 00000000..fbf130d2 --- /dev/null +++ b/monai/brats_mri_2d/backup/configs/multi_gpu_train_diffusion.json @@ -0,0 +1,16 @@ +{ + "diffusion": { + "_target_": "torch.nn.parallel.DistributedDataParallel", + "module": "$@network_def.to(@device)", + "device_ids": [ + "@device" + ], + "find_unused_parameters": true + }, + "run": [ + "@load_autoencoder", + "$@autoencoder.eval()", + "$print('scale factor:',@scale_factor)", + "$@train#trainer.run()" + ] +} diff --git a/monai/brats_mri_2d/backup/configs/train_autoencoder.json b/monai/brats_mri_2d/backup/configs/train_autoencoder.json new file mode 100644 index 00000000..e73c6bb4 --- /dev/null +++ b/monai/brats_mri_2d/backup/configs/train_autoencoder.json @@ -0,0 +1,151 @@ +{ + "imports": ["$import functools", "$import glob", "$import scripts"], + "bundle_root": ".", + "device": "$torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')", + "ckpt_dir": "$@bundle_root + '/models'", + "tf_dir": "$@bundle_root + '/eval'", + "dataset_dir": "/workspace/data/medical", + "pretrained": false, + "perceptual_loss_model_weights_path": null, + "train_batch_size_img": 1, + "train_batch_size_slice": 26, + "lr": 5e-05, + "train_patch_size": [240, 240], + "channel": 0, + "spatial_dims": 2, + "image_channels": 1, + "latent_channels": 1, + + "discriminator_def": { + "_target_": "generative.networks.nets.PatchDiscriminator", + "spatial_dims": "@spatial_dims", + "num_layers_d": 3, + "num_channels": 32, + "in_channels": 1, + "out_channels": 1, + "norm": "INSTANCE" + }, + + "autoencoder_def": { + "_target_": "generative.networks.nets.AutoencoderKL", + "spatial_dims": "@spatial_dims", + "in_channels": "@image_channels", + "out_channels": "@image_channels", + "latent_channels": "@latent_channels", + "num_channels": [64, 128, 256], + "num_res_blocks": 2, + "norm_num_groups": 32, + "norm_eps": 1e-06, + "attention_levels": [false, false, false], + "with_encoder_nonlocal_attn": true, + "with_decoder_nonlocal_attn": true + }, + + "perceptual_loss_def": { + "_target_": "generative.losses.PerceptualLoss", + "spatial_dims": "@spatial_dims", + "network_type": "resnet50", + "pretrained": "@pretrained", + "pretrained_path": "@perceptual_loss_model_weights_path", + "pretrained_state_dict_key": "state_dict" + }, + + "dnetwork": "$@discriminator_def.to(@device)", + "gnetwork": "$@autoencoder_def.to(@device)", + "loss_perceptual": "$@perceptual_loss_def.to(@device)", + "doptimizer": {"_target_": "torch.optim.Adam", "params": "$@dnetwork.parameters()", "lr": "@lr"}, + "goptimizer": {"_target_": "torch.optim.Adam", "params": "$@gnetwork.parameters()", "lr": "@lr"}, + + "preprocessing_transforms": [ + {"_target_": "LoadImaged", "keys": "image"}, + {"_target_": "EnsureChannelFirstd", "keys": "image"}, + {"_target_": "Lambdad", "keys": "image", "func": "$lambda x: x[@channel, :, :, :]"}, + {"_target_": "AddChanneld", "keys": "image"}, + {"_target_": "EnsureTyped", "keys": "image"}, + {"_target_": "Orientationd", "keys": "image", "axcodes": "RAS"}, + {"_target_": "CenterSpatialCropd", "keys": "image", "roi_size": "$[@train_patch_size[0], @train_patch_size[1], 100]"}, + {"_target_": "ScaleIntensityRangePercentilesd", "keys": "image", "lower": 0, "upper": 100, "b_min": 0, "b_max": 1} + ], + + "train": { + "crop_transforms": [ + {"_target_": "DivisiblePadd", "keys": "image", "k": [4, 4, 1]}, + + { + "_target_": "RandSpatialCropSamplesd", "keys": "image", "random_size": false, + "roi_size": "$[@train_patch_size[0], @train_patch_size[1], 1]", "num_samples": "@train_batch_size_slice" + }, + + {"_target_": "SqueezeDimd", "keys": "image", "dim": 3}, + {"_target_": "RandFlipd", "keys": ["image"], "prob": 0.5, "spatial_axis": 0}, + {"_target_": "RandFlipd", "keys": ["image"], "prob": 0.5, "spatial_axis": 1} + ], + + "preprocessing": {"_target_": "Compose", "transforms": "$@preprocessing_transforms + @train#crop_transforms"}, + + "dataset": { + "_target_": "monai.apps.DecathlonDataset", + "root_dir": "@dataset_dir", + "task": "Task01_BrainTumour", + "section": "training", + "cache_rate": 1.0, + "num_workers": 8, + "download": false, + "transform": "@train#preprocessing" + }, + "dataloader": { + "_target_": "DataLoader", + "dataset": "@train#dataset", + "batch_size": "@train_batch_size_img", + "shuffle": true, + "num_workers": 0 + }, + "handlers": [ + { + "_target_": "CheckpointSaver", + "save_dir": "@ckpt_dir", + "save_dict": { + "model": "@gnetwork" + }, + "save_interval": 0, + "save_final": true, + "epoch_level": true, + "final_filename": "model_autoencoder.pt" + }, + { + "_target_": "StatsHandler", + "tag_name": "train_loss", + "output_transform": "$lambda x: monai.handlers.from_engine(['g_loss'], first=True)(x)[0]" + }, + { + "_target_": "TensorBoardStatsHandler", + "log_dir": "@tf_dir", + "tag_name": "train_loss", + "output_transform": "$lambda x: monai.handlers.from_engine(['g_loss'], first=True)(x)[0]" + } + ], + "trainer": { + "_target_": "scripts.ldm_trainer.VaeGanTrainer", + "device": "@device", + "max_epochs": 1500, + "train_data_loader": "@train#dataloader", + "g_network": "@gnetwork", + "g_optimizer": "@goptimizer", + "g_loss_function": "$functools.partial(scripts.losses.generator_loss, disc_net=@dnetwork, loss_perceptual=@loss_perceptual)", + "d_network": "@dnetwork", + "d_optimizer": "@doptimizer", + "d_loss_function": "$functools.partial(scripts.losses.discriminator_loss, disc_net=@dnetwork)", + "d_train_steps": 1, + "g_update_latents": true, + "latent_shape": "@latent_channels", + "key_train_metric": "$None", + "train_handlers": "@train#handlers" + } + }, + "initialize": [ + "$monai.utils.set_determinism(seed=0)" + ], + "run": [ + "$@train#trainer.run()" + ] +} diff --git a/monai/brats_mri_2d/backup/configs/train_diffusion.json b/monai/brats_mri_2d/backup/configs/train_diffusion.json new file mode 100644 index 00000000..2aeede23 --- /dev/null +++ b/monai/brats_mri_2d/backup/configs/train_diffusion.json @@ -0,0 +1,174 @@ +{ + "ckpt_dir": "$@bundle_root + '/models'", + "train_batch_size_img": 2, + "train_batch_size_slice": 50, + "lr": 5e-05, + "train_patch_size": [ + 256, + 256 + ], + "latent_shape": [ + "@latent_channels", + 64, + 64 + ], + "load_autoencoder_path": "$@bundle_root + '/models/model_autoencoder.pt'", + "load_autoencoder": "$@autoencoder_def.load_state_dict(torch.load(@load_autoencoder_path))", + "autoencoder": "$@autoencoder_def.to(@device)", + "network_def": { + "_target_": "generative.networks.nets.DiffusionModelUNet", + "spatial_dims": "@spatial_dims", + "in_channels": "@latent_channels", + "out_channels": "@latent_channels", + "num_channels": [ + 32, + 64, + 128, + 256 + ], + "attention_levels": [ + false, + true, + true, + true + ], + "num_head_channels": [ + 0, + 32, + 32, + 32 + ], + "num_res_blocks": 2 + }, + "diffusion": "$@network_def.to(@device)", + "optimizer": { + "_target_": "torch.optim.Adam", + "params": "$@diffusion.parameters()", + "lr": "@lr" + }, + "lr_scheduler": { + "_target_": "torch.optim.lr_scheduler.MultiStepLR", + "optimizer": "@optimizer", + "milestones": [ + 1000 + ], + "gamma": 0.1 + }, + "scale_factor": "$scripts.utils.compute_scale_factor(@autoencoder,@train#dataloader,@device)", + "noise_scheduler": { + "_target_": "generative.networks.schedulers.DDPMScheduler", + "_requires_": [ + "@load_autoencoder" + ], + "schedule": "scaled_linear_beta", + "num_train_timesteps": 1000, + "beta_start": 0.0015, + "beta_end": 0.0195 + }, + "loss": { + "_target_": "torch.nn.MSELoss" + }, + "train": { + "inferer": { + "_target_": "generative.inferers.LatentDiffusionInferer", + "scheduler": "@noise_scheduler", + "scale_factor": "@scale_factor" + }, + "crop_transforms": [ + { + "_target_": "DivisiblePadd", + "keys": "image", + "k": [ + 32, + 32, + 1 + ] + }, + { + "_target_": "RandSpatialCropSamplesd", + "keys": "image", + "random_size": false, + "roi_size": "$[@train_patch_size[0], @train_patch_size[1], 1]", + "num_samples": "@train_batch_size_slice" + }, + { + "_target_": "SqueezeDimd", + "keys": "image", + "dim": 3 + } + ], + "preprocessing": { + "_target_": "Compose", + "transforms": "$@preprocessing_transforms + @train#crop_transforms" + }, + "dataset": { + "_target_": "monai.apps.DecathlonDataset", + "root_dir": "@dataset_dir", + "task": "Task01_BrainTumour", + "section": "training", + "cache_rate": 1.0, + "num_workers": 8, + "download": false, + "transform": "@train#preprocessing" + }, + "dataloader": { + "_target_": "DataLoader", + "dataset": "@train#dataset", + "batch_size": "@train_batch_size_img", + "shuffle": true, + "num_workers": 0 + }, + "handlers": [ + { + "_target_": "LrScheduleHandler", + "lr_scheduler": "@lr_scheduler", + "print_lr": true + }, + { + "_target_": "CheckpointSaver", + "save_dir": "@ckpt_dir", + "save_dict": { + "model": "@diffusion" + }, + "save_interval": 0, + "save_final": true, + "epoch_level": true, + "final_filename": "model.pt" + }, + { + "_target_": "StatsHandler", + "tag_name": "train_diffusion_loss", + "output_transform": "$lambda x: monai.handlers.from_engine(['loss'], first=True)(x)" + }, + { + "_target_": "TensorBoardStatsHandler", + "log_dir": "@tf_dir", + "tag_name": "train_diffusion_loss", + "output_transform": "$lambda x: monai.handlers.from_engine(['loss'], first=True)(x)" + } + ], + "trainer": { + "_target_": "scripts.ldm_trainer.LDMTrainer", + "device": "@device", + "max_epochs": 1000, + "train_data_loader": "@train#dataloader", + "network": "@diffusion", + "autoencoder_model": "@autoencoder", + "optimizer": "@optimizer", + "loss_function": "@loss", + "latent_shape": "@latent_shape", + "inferer": "@train#inferer", + "key_train_metric": "$None", + "train_handlers": "@train#handlers" + } + }, + "initialize": [ + "$monai.utils.set_determinism(seed=0)" + ], + "run": [ + "@load_autoencoder", + "$@autoencoder.eval()", + "$print('scale factor:',@scale_factor)", + "$@train#trainer.run()" + ] +} diff --git a/monai/brats_mri_2d/backup/docs/README.md b/monai/brats_mri_2d/backup/docs/README.md new file mode 100644 index 00000000..b8129fd2 --- /dev/null +++ b/monai/brats_mri_2d/backup/docs/README.md @@ -0,0 +1,166 @@ +# GETTING SET UP +pip install monai +pip install 'monai[all]' + +# Model Overview +A pre-trained model for 2D Latent Diffusion Generative Model on axial slices of BraTS MRI. + +This model is trained on BraTS 2016 and 2017 data from [Medical Decathlon](http://medicaldecathlon.com/), using the Latent diffusion model [1]. + +![model workflow](https://developer.download.nvidia.com/assets/Clara/Images/monai_brain_image_gen_ldm3d_network.png) + +This model is a generator for creating images like the Flair MRIs based on BraTS 2016 and 2017 data. It was trained as a 2d latent diffusion model and accepts Gaussian random noise as inputs to produce an image output. The `train_autoencoder.json` file describes the training process of the variational autoencoder with GAN loss. The `train_diffusion.json` file describes the training process of the 2D latent diffusion model. + +In this bundle, the autoencoder uses perceptual loss, which is based on ResNet50 with pre-trained weights (the network is frozen and will not be trained in the bundle). In default, the `pretrained` parameter is specified as `False` in `train_autoencoder.json`. To ensure correct training, changing the default settings is necessary. There are two ways to utilize pretrained weights: +1. if set `pretrained` to `True`, ImageNet pretrained weights from [torchvision](https://pytorch.org/vision/stable/_modules/torchvision/models/resnet.html#ResNet50_Weights) will be used. However, the weights are for non-commercial use only. +2. if set `pretrained` to `True` and specifies the `perceptual_loss_model_weights_path` parameter, users are able to load weights from a local path. This is the way this bundle used to train, and the pre-trained weights are from some internal data. + +Please note that each user is responsible for checking the data source of the pre-trained models, the applicable licenses, and determining if suitable for the intended use. + +#### Example synthetic image +An example result from inference is shown below: +![Example synthetic image](https://developer.download.nvidia.com/assets/Clara/Images/monai_brain_image_gen_ldm2d_example_generation_v2.png) + +**This is a demonstration network meant to just show the training process for this sort of network with MONAI. To achieve better performance, users need to use larger dataset like [BraTS 2021](https://www.synapse.org/#!Synapse:syn25829067/wiki/610865).** + +## Data +The training data is BraTS 2016 and 2017 from the Medical Segmentation Decathalon. Users can find more details on the dataset (`Task01_BrainTumour`) at http://medicaldecathlon.com/. + +- Target: Image Generation +- Task: Synthesis +- Modality: MRI +- Size: 388 3D MRI volumes (1 channel used) +- Training data size: 38800 2D MRI axial slices (1 channel used) + +## Training Configuration +If you have a GPU with less than 32G of memory, you may need to decrease the batch size when training. To do so, modify the `"train_batch_size_img"` and `"train_batch_size_slice"` parameters in the `configs/train_autoencoder.json` and `configs/train_diffusion.json` configuration files. +- `"train_batch_size_img"` is number of 3D volumes loaded in each batch. +- `"train_batch_size_slice"` is the number of 2D axial slices extracted from each image. The actual batch size is the product of them. + +### Training Configuration of Autoencoder +The autoencoder was trained using the following configuration: + +- GPU: at least 32GB GPU memory +- Actual Model Input: 240 x 240 +- AMP: False +- Optimizer: Adam +- Learning Rate: 5e-5 +- Loss: L1 loss, perceptual loss, KL divergence loss, adversarial loss, GAN BCE loss + +#### Input +1 channel 2D MRI Flair axial patches + +#### Output +- 1 channel 2D MRI reconstructed patches +- 1 channel mean of latent features +- 1 channel standard deviation of latent features + +### Training Configuration of Diffusion Model +The latent diffusion model was trained using the following configuration: + +- GPU: at least 32GB GPU memory +- Actual Model Input: 64 x 64 +- AMP: False +- Optimizer: Adam +- Learning Rate: 5e-5 +- Loss: MSE loss + +#### Training Input +- 1 channel noisy latent features +- a long int that indicates the time step + +#### Training Output +1 channel predicted added noise + +#### Inference Input +1 channel noise + +#### Inference Output +1 channel denoised latent features + +### Memory Consumption Warning + +If you face memory issues with data loading, you can lower the caching rate `cache_rate` in the configurations within range [0, 1] to minimize the System RAM requirements. + +## Performance + +#### Training Loss +![A graph showing the autoencoder training curve](https://developer.download.nvidia.com/assets/Clara/Images/monai_brain_image_gen_ldm2d_train_autoencoder_loss_v3.png) + +![A graph showing the latent diffusion training curve](https://developer.download.nvidia.com/assets/Clara/Images/monai_brain_image_gen_ldm2d_train_diffusion_loss_v3.png) + + +## MONAI Bundle Commands +In addition to the Pythonic APIs, a few command line interfaces (CLI) are provided to interact with the bundle. The CLI supports flexible use cases, such as overriding configs at runtime and predefining arguments in a file. + +For more details usage instructions, visit the [MONAI Bundle Configuration Page](https://docs.monai.io/en/latest/config_syntax.html). + +### Execute Autoencoder Training + +#### Execute Autoencoder Training on single GPU +``` +python -m monai.bundle run --config_file configs/train_autoencoder.json +``` + +Please note that if the default dataset path is not modified with the actual path (it should be the path that contains Task01_BrainTumour) in the bundle config files, you can also override it by using `--dataset_dir`: + +``` +python -m monai.bundle run --config_file configs/train_autoencoder.json --dataset_dir +``` + +#### Override the `train` config to execute multi-GPU training for Autoencoder +To train with multiple GPUs, use the following command, which requires scaling up the learning rate according to the number of GPUs. + +``` +torchrun --standalone --nnodes=1 --nproc_per_node=8 -m monai.bundle run --config_file "['configs/train_autoencoder.json','configs/multi_gpu_train_autoencoder.json']" --lr 4e-4 +``` + +#### Check the Autoencoder Training result +The following code generates a reconstructed image from a random input image. +We can visualize it to see if the autoencoder is trained correctly. +``` +python -m monai.bundle run --config_file configs/inference_autoencoder.json +``` + +An example of reconstructed image from inference is shown below. If the autoencoder is trained correctly, the reconstructed image should look similar to original image. + +![Example reconstructed image](https://developer.download.nvidia.com/assets/Clara/Images/monai_brain_image_gen_ldm2d_recon_example.png) + +### Execute Latent Diffusion Model Training + +#### Execute Latent Diffusion Model Training on single GPU +After training the autoencoder, run the following command to train the latent diffusion model. This command will print out the scale factor of the latent feature space. If your autoencoder is well trained, this value should be close to 1.0. + +``` +python -m monai.bundle run --config_file "['configs/train_autoencoder.json','configs/train_diffusion.json']" +``` + +#### Override the `train` config to execute multi-GPU training for Latent Diffusion Model +To train with multiple GPUs, use the following command, which requires scaling up the learning rate according to the number of GPUs. + +``` +torchrun --standalone --nnodes=1 --nproc_per_node=8 -m monai.bundle run --config_file "['configs/train_autoencoder.json','configs/train_diffusion.json','configs/multi_gpu_train_autoencoder.json','configs/multi_gpu_train_diffusion.json']" --lr 4e-4 +``` +### Execute inference +The following code generates a synthetic image from a random sampled noise. +``` +python -m monai.bundle run --config_file configs/inference.json +``` + +# References +[1] Rombach, Robin, et al. "High-resolution image synthesis with latent diffusion models." Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition. 2022. https://openaccess.thecvf.com/content/CVPR2022/papers/Rombach_High-Resolution_Image_Synthesis_With_Latent_Diffusion_Models_CVPR_2022_paper.pdf + +# License +Copyright (c) MONAI Consortium + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/monai/brats_mri_2d/backup/docs/data_license.txt b/monai/brats_mri_2d/backup/docs/data_license.txt new file mode 100644 index 00000000..d3d7e227 --- /dev/null +++ b/monai/brats_mri_2d/backup/docs/data_license.txt @@ -0,0 +1,49 @@ +Third Party Licenses +----------------------------------------------------------------------- + +/*********************************************************************/ +i. Multimodal Brain Tumor Segmentation Challenge 2018 + https://www.med.upenn.edu/sbia/brats2018/data.html +/*********************************************************************/ + +Data Usage Agreement / Citations + +You are free to use and/or refer to the BraTS datasets in your own +research, provided that you always cite the following two manuscripts: + +[1] Menze BH, Jakab A, Bauer S, Kalpathy-Cramer J, Farahani K, Kirby +[J, Burren Y, Porz N, Slotboom J, Wiest R, Lanczi L, Gerstner E, Weber +[MA, Arbel T, Avants BB, Ayache N, Buendia P, Collins DL, Cordier N, +[Corso JJ, Criminisi A, Das T, Delingette H, Demiralp Γ, Durst CR, +[Dojat M, Doyle S, Festa J, Forbes F, Geremia E, Glocker B, Golland P, +[Guo X, Hamamci A, Iftekharuddin KM, Jena R, John NM, Konukoglu E, +[Lashkari D, Mariz JA, Meier R, Pereira S, Precup D, Price SJ, Raviv +[TR, Reza SM, Ryan M, Sarikaya D, Schwartz L, Shin HC, Shotton J, +[Silva CA, Sousa N, Subbanna NK, Szekely G, Taylor TJ, Thomas OM, +[Tustison NJ, Unal G, Vasseur F, Wintermark M, Ye DH, Zhao L, Zhao B, +[Zikic D, Prastawa M, Reyes M, Van Leemput K. "The Multimodal Brain +[Tumor Image Segmentation Benchmark (BRATS)", IEEE Transactions on +[Medical Imaging 34(10), 1993-2024 (2015) DOI: +[10.1109/TMI.2014.2377694 + +[2] Bakas S, Akbari H, Sotiras A, Bilello M, Rozycki M, Kirby JS, +[Freymann JB, Farahani K, Davatzikos C. "Advancing The Cancer Genome +[Atlas glioma MRI collections with expert segmentation labels and +[radiomic features", Nature Scientific Data, 4:170117 (2017) DOI: +[10.1038/sdata.2017.117 + +In addition, if there are no restrictions imposed from the +journal/conference you submit your paper about citing "Data +Citations", please be specific and also cite the following: + +[3] Bakas S, Akbari H, Sotiras A, Bilello M, Rozycki M, Kirby J, +[Freymann J, Farahani K, Davatzikos C. "Segmentation Labels and +[Radiomic Features for the Pre-operative Scans of the TCGA-GBM +[collection", The Cancer Imaging Archive, 2017. DOI: +[10.7937/K9/TCIA.2017.KLXWJJ1Q + +[4] Bakas S, Akbari H, Sotiras A, Bilello M, Rozycki M, Kirby J, +[Freymann J, Farahani K, Davatzikos C. "Segmentation Labels and +[Radiomic Features for the Pre-operative Scans of the TCGA-LGG +[collection", The Cancer Imaging Archive, 2017. DOI: +[10.7937/K9/TCIA.2017.GJQ7R0EF diff --git a/monai/brats_mri_2d/backup/scripts/__init__.py b/monai/brats_mri_2d/backup/scripts/__init__.py new file mode 100644 index 00000000..2041a809 --- /dev/null +++ b/monai/brats_mri_2d/backup/scripts/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import ldm_sampler, ldm_trainer, losses, utils diff --git a/monai/brats_mri_2d/backup/scripts/ldm_sampler.py b/monai/brats_mri_2d/backup/scripts/ldm_sampler.py new file mode 100644 index 00000000..392d3333 --- /dev/null +++ b/monai/brats_mri_2d/backup/scripts/ldm_sampler.py @@ -0,0 +1,60 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import torch +import torch.nn as nn +from monai.utils import optional_import +from torch.cuda.amp import autocast + +tqdm, has_tqdm = optional_import("tqdm", name="tqdm") + + +class LDMSampler: + def __init__(self) -> None: + super().__init__() + + @torch.no_grad() + def sampling_fn( + self, + input_noise: torch.Tensor, + autoencoder_model: nn.Module, + diffusion_model: nn.Module, + scheduler: nn.Module, + conditioning: torch.Tensor | None = None, + ) -> torch.Tensor: + if has_tqdm: + progress_bar = tqdm(scheduler.timesteps) + else: + progress_bar = iter(scheduler.timesteps) + + image = input_noise + if conditioning is not None: + cond_concat = conditioning.squeeze(1).unsqueeze(-1).unsqueeze(-1).unsqueeze(-1) + cond_concat = cond_concat.expand(list(cond_concat.shape[0:2]) + list(input_noise.shape[2:])) + + for t in progress_bar: + with torch.no_grad(): + if conditioning is not None: + input_t = torch.cat((image, cond_concat), dim=1) + else: + input_t = image + model_output = diffusion_model( + input_t, timesteps=torch.Tensor((t,)).to(input_noise.device).long(), context=conditioning + ) + image, _ = scheduler.step(model_output, t, image) + + with torch.no_grad(): + with autocast(): + sample = autoencoder_model.decode_stage_2_outputs(image) + + return sample diff --git a/monai/brats_mri_2d/backup/scripts/ldm_trainer.py b/monai/brats_mri_2d/backup/scripts/ldm_trainer.py new file mode 100644 index 00000000..a298461e --- /dev/null +++ b/monai/brats_mri_2d/backup/scripts/ldm_trainer.py @@ -0,0 +1,380 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Callable, Iterable, Sequence + +import torch +from monai.config import IgniteInfo +from monai.engines.utils import IterationEvents, default_metric_cmp_fn, default_prepare_batch +from monai.inferers import Inferer, SimpleInferer +from monai.transforms import Transform +from monai.utils import min_version, optional_import +from monai.utils.enums import CommonKeys, GanKeys +from torch.optim.optimizer import Optimizer +from torch.utils.data import DataLoader + +if TYPE_CHECKING: + from ignite.engine import Engine, EventEnum + from ignite.metrics import Metric +else: + Engine, _ = optional_import("ignite.engine", IgniteInfo.OPT_IMPORT_VERSION, min_version, "Engine") + Metric, _ = optional_import("ignite.metrics", IgniteInfo.OPT_IMPORT_VERSION, min_version, "Metric") + EventEnum, _ = optional_import("ignite.engine", IgniteInfo.OPT_IMPORT_VERSION, min_version, "EventEnum") +from monai.engines.trainer import SupervisedTrainer, Trainer + + +class VaeGanTrainer(Trainer): + """ + Generative adversarial network training based on Goodfellow et al. 2014 https://arxiv.org/abs/1406.266, + inherits from ``Trainer`` and ``Workflow``. + Training Loop: for each batch of data size `m` + 1. Generate `m` fakes from random latent codes. + 2. Update discriminator with these fakes and current batch reals, repeated d_train_steps times. + 3. If g_update_latents, generate `m` fakes from new random latent codes. + 4. Update generator with these fakes using discriminator feedback. + Args: + device: an object representing the device on which to run. + max_epochs: the total epoch number for engine to run. + train_data_loader: Core ignite engines uses `DataLoader` for training loop batchdata. + g_network: generator (G) network architecture. + g_optimizer: G optimizer function. + g_loss_function: G loss function for optimizer. + d_network: discriminator (D) network architecture. + d_optimizer: D optimizer function. + d_loss_function: D loss function for optimizer. + epoch_length: number of iterations for one epoch, default to `len(train_data_loader)`. + g_inferer: inference method to execute G model forward. Defaults to ``SimpleInferer()``. + d_inferer: inference method to execute D model forward. Defaults to ``SimpleInferer()``. + d_train_steps: number of times to update D with real data minibatch. Defaults to ``1``. + latent_shape: size of G input latent code. Defaults to ``64``. + non_blocking: if True and this copy is between CPU and GPU, the copy may occur asynchronously + with respect to the host. For other cases, this argument has no effect. + d_prepare_batch: callback function to prepare batchdata for D inferer. + Defaults to return ``GanKeys.REALS`` in batchdata dict. for more details please refer to: + https://pytorch.org/ignite/generated/ignite.engine.create_supervised_trainer.html. + g_prepare_batch: callback function to create batch of latent input for G inferer. + Defaults to return random latents. for more details please refer to: + https://pytorch.org/ignite/generated/ignite.engine.create_supervised_trainer.html. + g_update_latents: Calculate G loss with new latent codes. Defaults to ``True``. + iteration_update: the callable function for every iteration, expect to accept `engine` + and `engine.state.batch` as inputs, return data will be stored in `engine.state.output`. + if not provided, use `self._iteration()` instead. for more details please refer to: + https://pytorch.org/ignite/generated/ignite.engine.engine.Engine.html. + postprocessing: execute additional transformation for the model output data. + Typically, several Tensor based transforms composed by `Compose`. + key_train_metric: compute metric when every iteration completed, and save average value to + engine.state.metrics when epoch completed. key_train_metric is the main metric to compare and save the + checkpoint into files. + additional_metrics: more Ignite metrics that also attach to Ignite Engine. + metric_cmp_fn: function to compare current key metric with previous best key metric value, + it must accept 2 args (current_metric, previous_best) and return a bool result: if `True`, will update + `best_metric` and `best_metric_epoch` with current metric and epoch, default to `greater than`. + train_handlers: every handler is a set of Ignite Event-Handlers, must have `attach` function, like: + CheckpointHandler, StatsHandler, etc. + decollate: whether to decollate the batch-first data to a list of data after model computation, + recommend `decollate=True` when `postprocessing` uses components from `monai.transforms`. + default to `True`. + optim_set_to_none: when calling `optimizer.zero_grad()`, instead of setting to zero, set the grads to None. + more details: https://pytorch.org/docs/stable/generated/torch.optim.Optimizer.zero_grad.html. + to_kwargs: dict of other args for `prepare_batch` API when converting the input data, except for + `device`, `non_blocking`. + amp_kwargs: dict of the args for `torch.cuda.amp.autocast()` API, for more details: + https://pytorch.org/docs/stable/amp.html#torch.cuda.amp.autocast. + """ + + def __init__( + self, + device: str | torch.device, + max_epochs: int, + train_data_loader: DataLoader, + g_network: torch.nn.Module, + g_optimizer: Optimizer, + g_loss_function: Callable, + d_network: torch.nn.Module, + d_optimizer: Optimizer, + d_loss_function: Callable, + epoch_length: int | None = None, + g_inferer: Inferer | None = None, + d_inferer: Inferer | None = None, + d_train_steps: int = 1, + latent_shape: int = 64, + non_blocking: bool = False, + d_prepare_batch: Callable = default_prepare_batch, + g_prepare_batch: Callable = default_prepare_batch, + g_update_latents: bool = True, + iteration_update: Callable[[Engine, Any], Any] | None = None, + postprocessing: Transform | None = None, + key_train_metric: dict[str, Metric] | None = None, + additional_metrics: dict[str, Metric] | None = None, + metric_cmp_fn: Callable = default_metric_cmp_fn, + train_handlers: Sequence | None = None, + decollate: bool = True, + optim_set_to_none: bool = False, + to_kwargs: dict | None = None, + amp_kwargs: dict | None = None, + ): + if not isinstance(train_data_loader, DataLoader): + raise ValueError("train_data_loader must be PyTorch DataLoader.") + + # set up Ignite engine and environments + super().__init__( + device=device, + max_epochs=max_epochs, + data_loader=train_data_loader, + epoch_length=epoch_length, + non_blocking=non_blocking, + prepare_batch=d_prepare_batch, + iteration_update=iteration_update, + key_metric=key_train_metric, + additional_metrics=additional_metrics, + metric_cmp_fn=metric_cmp_fn, + handlers=train_handlers, + postprocessing=postprocessing, + decollate=decollate, + to_kwargs=to_kwargs, + amp_kwargs=amp_kwargs, + ) + self.g_network = g_network + self.g_optimizer = g_optimizer + self.g_loss_function = g_loss_function + self.g_inferer = SimpleInferer() if g_inferer is None else g_inferer + self.d_network = d_network + self.d_optimizer = d_optimizer + self.d_loss_function = d_loss_function + self.d_inferer = SimpleInferer() if d_inferer is None else d_inferer + self.d_train_steps = d_train_steps + self.latent_shape = latent_shape + self.g_prepare_batch = g_prepare_batch + self.g_update_latents = g_update_latents + self.optim_set_to_none = optim_set_to_none + + def _iteration( + self, engine: VaeGanTrainer, batchdata: dict | Sequence + ) -> dict[str, torch.Tensor | int | float | bool]: + """ + Callback function for Adversarial Training processing logic of 1 iteration in Ignite Engine. + Args: + engine: `VaeGanTrainer` to execute operation for an iteration. + batchdata: input data for this iteration, usually can be dictionary or tuple of Tensor data. + Raises: + ValueError: must provide batch data for current iteration. + """ + if batchdata is None: + raise ValueError("must provide batch data for current iteration.") + + d_input = engine.prepare_batch(batchdata, engine.state.device, engine.non_blocking, **engine.to_kwargs)[0] + g_input = d_input + g_output, z_mu, z_sigma = engine.g_inferer(g_input, engine.g_network) + + # Train Generator + engine.g_optimizer.zero_grad(set_to_none=engine.optim_set_to_none) + g_loss = engine.g_loss_function(g_output, g_input, z_mu, z_sigma) + g_loss.backward() + engine.g_optimizer.step() + + # Train Discriminator + d_total_loss = torch.zeros(1) + for _ in range(engine.d_train_steps): + engine.d_optimizer.zero_grad(set_to_none=engine.optim_set_to_none) + dloss = engine.d_loss_function(g_output, d_input) + dloss.backward() + engine.d_optimizer.step() + d_total_loss += dloss.item() + + return { + GanKeys.REALS: d_input, + GanKeys.FAKES: g_output, + GanKeys.LATENTS: g_input, + GanKeys.GLOSS: g_loss.item(), + GanKeys.DLOSS: d_total_loss.item(), + } + + +class LDMTrainer(SupervisedTrainer): + """ + Standard supervised training method with image and label, inherits from ``Trainer`` and ``Workflow``. + Args: + device: an object representing the device on which to run. + max_epochs: the total epoch number for trainer to run. + train_data_loader: Ignite engine use data_loader to run, must be Iterable or torch.DataLoader. + network: network to train in the trainer, should be regular PyTorch `torch.nn.Module`. + optimizer: the optimizer associated to the network, should be regular PyTorch optimizer from `torch.optim` + or its subclass. + loss_function: the loss function associated to the optimizer, should be regular PyTorch loss, + which inherit from `torch.nn.modules.loss`. + epoch_length: number of iterations for one epoch, default to `len(train_data_loader)`. + non_blocking: if True and this copy is between CPU and GPU, the copy may occur asynchronously + with respect to the host. For other cases, this argument has no effect. + prepare_batch: function to parse expected data (usually `image`, `label` and other network args) + from `engine.state.batch` for every iteration, for more details please refer to: + https://pytorch.org/ignite/generated/ignite.engine.create_supervised_trainer.html. + iteration_update: the callable function for every iteration, expect to accept `engine` + and `engine.state.batch` as inputs, return data will be stored in `engine.state.output`. + if not provided, use `self._iteration()` instead. for more details please refer to: + https://pytorch.org/ignite/generated/ignite.engine.engine.Engine.html. + inferer: inference method that execute model forward on input data, like: SlidingWindow, etc. + postprocessing: execute additional transformation for the model output data. + Typically, several Tensor based transforms composed by `Compose`. + key_train_metric: compute metric when every iteration completed, and save average value to + engine.state.metrics when epoch completed. key_train_metric is the main metric to compare and save the + checkpoint into files. + additional_metrics: more Ignite metrics that also attach to Ignite Engine. + metric_cmp_fn: function to compare current key metric with previous best key metric value, + it must accept 2 args (current_metric, previous_best) and return a bool result: if `True`, will update + `best_metric` and `best_metric_epoch` with current metric and epoch, default to `greater than`. + train_handlers: every handler is a set of Ignite Event-Handlers, must have `attach` function, like: + CheckpointHandler, StatsHandler, etc. + amp: whether to enable auto-mixed-precision training, default is False. + event_names: additional custom ignite events that will register to the engine. + new events can be a list of str or `ignite.engine.events.EventEnum`. + event_to_attr: a dictionary to map an event to a state attribute, then add to `engine.state`. + for more details, check: https://pytorch.org/ignite/generated/ignite.engine.engine.Engine.html + #ignite.engine.engine.Engine.register_events. + decollate: whether to decollate the batch-first data to a list of data after model computation, + recommend `decollate=True` when `postprocessing` uses components from `monai.transforms`. + default to `True`. + optim_set_to_none: when calling `optimizer.zero_grad()`, instead of setting to zero, set the grads to None. + more details: https://pytorch.org/docs/stable/generated/torch.optim.Optimizer.zero_grad.html. + to_kwargs: dict of other args for `prepare_batch` API when converting the input data, except for + `device`, `non_blocking`. + amp_kwargs: dict of the args for `torch.cuda.amp.autocast()` API, for more details: + https://pytorch.org/docs/stable/amp.html#torch.cuda.amp.autocast. + """ + + def __init__( + self, + device: str | torch.device, + max_epochs: int, + train_data_loader: Iterable | DataLoader, + network: torch.nn.Module, + autoencoder_model: torch.nn.Module, + optimizer: Optimizer, + loss_function: Callable, + latent_shape: Sequence, + inferer: Inferer, + epoch_length: int | None = None, + non_blocking: bool = False, + prepare_batch: Callable = default_prepare_batch, + iteration_update: Callable[[Engine, Any], Any] | None = None, + postprocessing: Transform | None = None, + key_train_metric: dict[str, Metric] | None = None, + additional_metrics: dict[str, Metric] | None = None, + metric_cmp_fn: Callable = default_metric_cmp_fn, + train_handlers: Sequence | None = None, + amp: bool = False, + event_names: list[str | EventEnum | type[EventEnum]] | None = None, + event_to_attr: dict | None = None, + decollate: bool = True, + optim_set_to_none: bool = False, + to_kwargs: dict | None = None, + amp_kwargs: dict | None = None, + ) -> None: + super().__init__( + device=device, + max_epochs=max_epochs, + train_data_loader=train_data_loader, + network=network, + optimizer=optimizer, + loss_function=loss_function, + inferer=inferer, + optim_set_to_none=optim_set_to_none, + epoch_length=epoch_length, + non_blocking=non_blocking, + prepare_batch=prepare_batch, + iteration_update=iteration_update, + postprocessing=postprocessing, + key_train_metric=key_train_metric, + additional_metrics=additional_metrics, + metric_cmp_fn=metric_cmp_fn, + train_handlers=train_handlers, + amp=amp, + event_names=event_names, + event_to_attr=event_to_attr, + decollate=decollate, + to_kwargs=to_kwargs, + amp_kwargs=amp_kwargs, + ) + + self.latent_shape = latent_shape + self.autoencoder_model = autoencoder_model + + def _iteration(self, engine: LDMTrainer, batchdata: dict[str, torch.Tensor]) -> dict: + """ + Callback function for the Supervised Training processing logic of 1 iteration in Ignite Engine. + Return below items in a dictionary: + - IMAGE: image Tensor data for model input, already moved to device. + - LABEL: label Tensor data corresponding to the image, already moved to device. + - PRED: prediction result of model. + - LOSS: loss value computed by loss function. + Args: + engine: `SupervisedTrainer` to execute operation for an iteration. + batchdata: input data for this iteration, usually can be dictionary or tuple of Tensor data. + Raises: + ValueError: When ``batchdata`` is None. + """ + if batchdata is None: + raise ValueError("Must provide batch data for current iteration.") + batch = engine.prepare_batch(batchdata, engine.state.device, engine.non_blocking, **engine.to_kwargs) + if len(batch) == 2: + images, labels = batch + args: tuple = () + kwargs: dict = {} + else: + images, labels, args, kwargs = batch + # put iteration outputs into engine.state + engine.state.output = {CommonKeys.IMAGE: images} + + # generate noise + noise_shape = [images.shape[0]] + list(self.latent_shape) + noise = torch.randn(noise_shape, dtype=images.dtype).to(images.device) + engine.state.output = {"noise": noise} + + # Create timesteps + timesteps = torch.randint( + 0, engine.inferer.scheduler.num_train_timesteps, (images.shape[0],), device=images.device + ).long() + + def _compute_pred_loss(): + # predicted noise + engine.state.output[CommonKeys.PRED] = engine.inferer( + inputs=images, + autoencoder_model=self.autoencoder_model, + diffusion_model=engine.network, + noise=noise, + timesteps=timesteps, + ) + engine.fire_event(IterationEvents.FORWARD_COMPLETED) + # compute loss + engine.state.output[CommonKeys.LOSS] = engine.loss_function( + engine.state.output[CommonKeys.PRED], noise + ).mean() + engine.fire_event(IterationEvents.LOSS_COMPLETED) + + engine.network.train() + engine.optimizer.zero_grad(set_to_none=engine.optim_set_to_none) + + if engine.amp and engine.scaler is not None: + with torch.cuda.amp.autocast(**engine.amp_kwargs): + _compute_pred_loss() + engine.scaler.scale(engine.state.output[CommonKeys.LOSS]).backward() + engine.fire_event(IterationEvents.BACKWARD_COMPLETED) + engine.scaler.step(engine.optimizer) + engine.scaler.update() + else: + _compute_pred_loss() + engine.state.output[CommonKeys.LOSS].backward() + engine.fire_event(IterationEvents.BACKWARD_COMPLETED) + engine.optimizer.step() + engine.fire_event(IterationEvents.MODEL_COMPLETED) + + return engine.state.output diff --git a/monai/brats_mri_2d/backup/scripts/losses.py b/monai/brats_mri_2d/backup/scripts/losses.py new file mode 100644 index 00000000..9c824bd4 --- /dev/null +++ b/monai/brats_mri_2d/backup/scripts/losses.py @@ -0,0 +1,52 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import torch +from generative.losses import PatchAdversarialLoss + +intensity_loss = torch.nn.L1Loss() +adv_loss = PatchAdversarialLoss(criterion="least_squares") + +adv_weight = 0.5 +perceptual_weight = 1.0 +# kl_weight: important hyper-parameter. +# If too large, decoder cannot recon good results from latent space. +# If too small, latent space will not be regularized enough for the diffusion model +kl_weight = 1e-6 + + +def compute_kl_loss(z_mu, z_sigma): + kl_loss = 0.5 * torch.sum( + z_mu.pow(2) + z_sigma.pow(2) - torch.log(z_sigma.pow(2)) - 1, dim=list(range(1, len(z_sigma.shape))) + ) + return torch.sum(kl_loss) / kl_loss.shape[0] + + +def generator_loss(gen_images, real_images, z_mu, z_sigma, disc_net, loss_perceptual): + recons_loss = intensity_loss(gen_images, real_images) + kl_loss = compute_kl_loss(z_mu, z_sigma) + p_loss = loss_perceptual(gen_images.float(), real_images.float()) + loss_g = recons_loss + kl_weight * kl_loss + perceptual_weight * p_loss + + logits_fake = disc_net(gen_images)[-1] + generator_loss = adv_loss(logits_fake, target_is_real=True, for_discriminator=False) + loss_g = loss_g + adv_weight * generator_loss + + return loss_g + + +def discriminator_loss(gen_images, real_images, disc_net): + logits_fake = disc_net(gen_images.contiguous().detach())[-1] + loss_d_fake = adv_loss(logits_fake, target_is_real=False, for_discriminator=True) + logits_real = disc_net(real_images.contiguous().detach())[-1] + loss_d_real = adv_loss(logits_real, target_is_real=True, for_discriminator=True) + discriminator_loss = (loss_d_fake + loss_d_real) * 0.5 + loss_d = adv_weight * discriminator_loss + return loss_d diff --git a/monai/brats_mri_2d/backup/scripts/utils.py b/monai/brats_mri_2d/backup/scripts/utils.py new file mode 100644 index 00000000..5a5a5aa8 --- /dev/null +++ b/monai/brats_mri_2d/backup/scripts/utils.py @@ -0,0 +1,50 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import numpy as np +import torch +from monai.utils import first +from monai.utils.type_conversion import convert_to_numpy + + +def compute_scale_factor(autoencoder, train_loader, device): + with torch.no_grad(): + check_data = first(train_loader) + z = autoencoder.encode_stage_2_inputs(check_data["image"].to(device)) + scale_factor = 1 / torch.std(z) + return scale_factor.item() + + +def normalize_image_to_uint8(image): + """ + Normalize image to uint8 + Args: + image: numpy array + """ + draw_img = image + if np.amin(draw_img) < 0: + draw_img[draw_img < 0] = 0 + if np.amax(draw_img) > 0.1: + draw_img /= np.amax(draw_img) + draw_img = (255 * draw_img).astype(np.uint8) + return draw_img + + +def visualize_2d_image(image): + """ + Prepare a 2D image for visualization. + Args: + image: image numpy array, sized (H, W) + """ + image = convert_to_numpy(image) + # draw image + draw_img = normalize_image_to_uint8(image) + draw_img = np.stack([draw_img, draw_img, draw_img], axis=-1) + return draw_img diff --git a/monai/brats_mri_2d/brats_mri_2d_diff.isc b/monai/brats_mri_2d/brats_mri_2d_diff.isc new file mode 100644 index 00000000..daad1731 --- /dev/null +++ b/monai/brats_mri_2d/brats_mri_2d_diff.isc @@ -0,0 +1,6 @@ +experiment_name="brats_mri_2d_diff" +gpu_type="24GB VRAM GPU" +nnodes = 9 +venv_path = "~/.venv/bin/activate" +output_path = "~/output_brats_mri_2d_diff" +command="train_cycling_diff.py --data-path=/mnt/Datasets/Open-Datasets/MONAI --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/monai/brats_mri_2d/brats_mri_2d_gen.isc b/monai/brats_mri_2d/brats_mri_2d_gen.isc new file mode 100644 index 00000000..c57c5e55 --- /dev/null +++ b/monai/brats_mri_2d/brats_mri_2d_gen.isc @@ -0,0 +1,6 @@ +experiment_name="brats_mri_2d_gen" +gpu_type="24GB VRAM GPU" +nnodes = 9 +venv_path = "~/.venv/bin/activate" +output_path = "~/output_brats_mri_2d_gen" +command="train_cycling_gen.py --data-path=/mnt/Datasets/Open-Datasets/MONAI --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/monai/brats_mri_2d/loops.py b/monai/brats_mri_2d/loops.py new file mode 100644 index 00000000..a8c7d23f --- /dev/null +++ b/monai/brats_mri_2d/loops.py @@ -0,0 +1,282 @@ +from tqdm import tqdm +import torch +from torch.cuda.amp import GradScaler, autocast +import torch.nn.functional as F +import utils +from cycling_utils import atomic_torch_save + +## -- AUTO-ENCODER - ## + +def train_generator_one_epoch( + args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, + scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, + train_images_seen, val_images_seen, epoch_loss, gen_epoch_loss, disc_epoch_loss, val_loss + ): + + # Maybe pull these out into args later + kl_weight = 1e-6 + generator_warm_up_n_epochs = 10 + perceptual_weight = 0.001 + adv_weight = 0.01 + + generator.train() + discriminator.train() + + train_step = train_sampler.progress // train_loader.batch_size + total_steps = int((len(train_loader.dataset) / 11) / train_loader.batch_size) + print(f'\nTraining / resuming epoch {epoch} from training step {train_step}\n') + + for step, batch in enumerate(train_loader): + + images = batch["image"].to(device) + timer.report(f'train batch {train_step} to device') + + # TRAIN GENERATOR + + optimizer_g.zero_grad(set_to_none=True) + + with autocast(enabled=True): + + reconstruction, z_mu, z_sigma = generator(images) + timer.report(f'train batch {train_step} generator forward') + recons_loss = F.l1_loss(reconstruction.float(), images.float()) + timer.report(f'train batch {train_step} recons_loss') + p_loss = perceptual_loss(reconstruction.float(), images.float()) + timer.report(f'train batch {train_step} p_loss') + kl_loss = 0.5 * torch.sum(z_mu.pow(2) + z_sigma.pow(2) - torch.log(z_sigma.pow(2)) - 1, dim=[1, 2, 3]) + kl_loss = torch.sum(kl_loss) / kl_loss.shape[0] + timer.report(f'train batch {train_step} kl_loss') + loss_g = recons_loss + (kl_weight * kl_loss) + (perceptual_weight * p_loss) + timer.report(f'train batch {train_step} loss_g (1)') + + if epoch > generator_warm_up_n_epochs: # Train generator for n epochs on reconstruction, KL, and perceptual loss before introducing discriminator loss + + logits_fake = discriminator(reconstruction.contiguous().float())[-1] + timer.report(f'train batch {train_step} logits_fake from discriminator') + generator_loss = adv_loss(logits_fake, target_is_real=True, for_discriminator=False) + timer.report(f'train batch {train_step} generator_loss (adv_loss)') + loss_g += adv_weight * generator_loss + timer.report(f'train batch {train_step} loss_g (2)') + + scaler_g.scale(loss_g).backward() + scaler_g.step(optimizer_g) + scaler_g.update() + timer.report(f'train batch {train_step} generator backward') + + # TRAIN DISCRIMINATOR + + if epoch > generator_warm_up_n_epochs: # Train generator for n epochs before starting discriminator training + + with autocast(enabled=True): + + optimizer_d.zero_grad(set_to_none=True) + + logits_fake = discriminator(reconstruction.contiguous().detach())[-1] + timer.report(f'train batch {train_step} discriminator forward (fake)') + loss_d_fake = adv_loss(logits_fake, target_is_real=False, for_discriminator=True) + timer.report(f'train batch {train_step} loss_d_fake') + logits_real = discriminator(images.contiguous().detach())[-1] + timer.report(f'train batch {train_step} discriminator forward (real)') + loss_d_real = adv_loss(logits_real, target_is_real=True, for_discriminator=True) + timer.report(f'train batch {train_step} loss_d_real') + discriminator_loss = (loss_d_fake + loss_d_real) * 0.5 + timer.report(f'train batch {train_step} discriminator_loss') + loss_d = adv_weight * discriminator_loss + timer.report(f'train batch {train_step} loss_d') + + scaler_d.scale(loss_d).backward() + scaler_d.step(optimizer_d) + scaler_d.update() + timer.report(f'train batch {train_step} discriminator backward') + + epoch_loss += recons_loss.item() + if epoch > generator_warm_up_n_epochs: + gen_epoch_loss += generator_loss.item() + disc_epoch_loss += discriminator_loss.item() + timer.report(f'train batch {train_step} metrics update') + + train_images_seen += len(images) + recons_loss = epoch_loss / train_images_seen + gen_loss = gen_epoch_loss / train_images_seen + disc_loss = disc_epoch_loss / train_images_seen + print("Epoch [{}] Step [{}/{}] :: recons_loss: {:,.3f}, gen_loss: {:,.3f}, disc_loss: {:,.3f}".format(epoch, train_step, total_steps, recons_loss, gen_loss, disc_loss)) + + ## Checkpointing + print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") + train_sampler.advance(len(images)) + if utils.is_main_process() and train_step % 1 == 0: # Checkpointing every batch + checkpoint = { + # Universals + "args": args, + "epoch": epoch, + + # State variables + "generator": generator.module.state_dict(), + "discriminator": discriminator.module.state_dict(), + "optimizer_g": optimizer_g.state_dict(), + "optimizer_d": optimizer_d.state_dict(), + "scaler_g": scaler_g.state_dict(), + "scaler_d": scaler_d.state_dict(), + "train_sampler": train_sampler.state_dict(), + "val_sampler": val_sampler.state_dict(), + + # Evaluation metrics + "train_images_seen": train_images_seen, + "val_images_seen": val_images_seen, + "epoch_loss": epoch_loss, + "gen_epoch_loss": gen_epoch_loss, + "disc_epoch_loss": disc_epoch_loss, + "val_loss": val_loss, + } + timer = atomic_torch_save(checkpoint, args.resume, timer) + + return generator, timer + + +def evaluate_generator( + args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, + scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, + train_images_seen, val_images_seen, epoch_loss, gen_epoch_loss, disc_epoch_loss, val_loss + ): + + generator.eval() + + val_step = val_sampler.progress // val_loader.batch_size + print(f'\nEvaluating / resuming epoch {epoch} from training step {val_step}\n') + + with torch.no_grad(): + for step, batch in enumerate(val_loader): + + images = batch["image"].to(device) + timer.report(f'eval batch {val_step} to device') + + with autocast(enabled=True): + + reconstruction, _, _ = generator(images) + timer.report(f'eval batch {val_step} forward') + recons_loss = F.l1_loss(images.float(), reconstruction.float()) + timer.report(f'eval batch {val_step} recons_loss') + + val_loss += recons_loss.item() + val_images_seen += len(images) + timer.report(f'eval batch {val_step} metrics update') + + ## Checkpointing + print(f"Saving checkpoint at epoch {epoch} val batch {val_step}") + val_sampler.advance(len(images)) + if utils.is_main_process() and val_step % 1 == 0: # Checkpointing every batch + print(f"Saving checkpoint at epoch {epoch} train batch {val_step}") + checkpoint = { + # Universals + "args": args, + "epoch": epoch, + + # State variables + "generator": generator.module.state_dict(), + "discriminator": discriminator.module.state_dict(), + "optimizer_g": optimizer_g.state_dict(), + "optimizer_d": optimizer_d.state_dict(), + "scaler_g": scaler_g.state_dict(), + "scaler_d": scaler_d.state_dict(), + "train_sampler": train_sampler.state_dict(), + "val_sampler": val_sampler.state_dict(), + + # Evaluation metrics + "train_images_seen": train_images_seen, + "val_images_seen": val_images_seen, + "epoch_loss": epoch_loss, + "gen_epoch_loss": gen_epoch_loss, + "disc_epoch_loss": disc_epoch_loss, + "val_loss": val_loss, + } + timer = atomic_torch_save(checkpoint, args.resume, timer) + + val_loss /= val_images_seen + print(f"Epoch {epoch} val loss: {val_loss:.4f}") + + return timer # ?? + + + + + + +# ## -- DIFFUSION MODEL - ## + +# def train_diffusion_one_epoch(epoch, unet, generator, optimizer, inferer, scaler, train_loader, device): + +# unet.train() +# generator.eval() + +# epoch_losses = [] + +# epoch_loss = 0 +# train_images_seen = 0 + +# progress_bar = tqdm(enumerate(train_loader), total=len(train_loader), ncols=70) +# progress_bar.set_description(f"Epoch (train diff) {epoch}") + +# for step, batch in progress_bar: + +# images = batch["image"].to(device) +# optimizer.zero_grad(set_to_none=True) + +# with autocast(enabled=True): + +# z_mu, z_sigma = generator.encode(images) +# z = generator.sampling(z_mu, z_sigma) +# noise = torch.randn_like(z).to(device) +# timesteps = torch.randint(0, inferer.scheduler.num_train_timesteps, (z.shape[0],), device=z.device).long() +# noise_pred = inferer(inputs=images, diffusion_model=unet, noise=noise, timesteps=timesteps, autoencoder_model=generator) +# loss = F.mse_loss(noise_pred.float(), noise.float()) + +# scaler.scale(loss).backward() +# scaler.step(optimizer) +# scaler.update() + +# epoch_loss += loss.item() +# train_images_seen += len(images) + +# progress_bar.set_postfix({"loss": epoch_loss / train_images_seen}) + +# epoch_losses.append(epoch_loss / train_images_seen) + +# return unet, epoch_losses + + +# def evaluate_diffusion(epoch, unet, generator, inferer, val_loader, device): + +# unet.eval() + +# val_losses = [] + +# val_loss = 0 +# val_images_seen = 0 + +# progress_bar = tqdm(enumerate(val_loader), total=len(val_loader), ncols=110) +# progress_bar.set_description(f"Epoch (eval diff) {epoch}") + +# with torch.no_grad(): +# for val_step, batch in enumerate(val_loader, start=1): +# images = batch["image"].to(device) + +# with autocast(enabled=True): +# z_mu, z_sigma = generator.encode(images) +# z = generator.sampling(z_mu, z_sigma) + +# noise = torch.randn_like(z).to(device) +# timesteps = torch.randint(0, inferer.scheduler.num_train_timesteps, (z.shape[0],), device=z.device).long() +# noise_pred = inferer(inputs=images,diffusion_model=unet,noise=noise,timesteps=timesteps,autoencoder_model=generator) + +# loss = F.mse_loss(noise_pred.float(), noise.float()) + +# val_loss += loss.item() +# val_images_seen += len(images) + +# val_loss /= val_images_seen +# val_losses.append(val_loss) +# print(f"Epoch {epoch} diff val loss: {val_loss:.4f}") + +# progress_bar.close() + +# return val_losses # ?? diff --git a/monai/brats_mri_2d/models_DIS.py b/monai/brats_mri_2d/models_DIS.py new file mode 100644 index 00000000..c6f23c2d --- /dev/null +++ b/monai/brats_mri_2d/models_DIS.py @@ -0,0 +1,23 @@ +# import os +# import pandas as pd +# from torchvision.io import read_image + +# class CustomImageDataset(Dataset): +# def __init__(self, annotations_file, img_dir, transform=None, target_transform=None): +# self.img_labels = pd.read_csv(annotations_file) +# self.img_dir = img_dir +# self.transform = transform +# self.target_transform = target_transform + +# def __len__(self): +# return len(self.img_labels) + +# def __getitem__(self, idx): +# img_path = os.path.join(self.img_dir, self.img_labels.iloc[idx, 0]) +# image = read_image(img_path) +# label = self.img_labels.iloc[idx, 1] +# if self.transform: +# image = self.transform(image) +# if self.target_transform: +# label = self.target_transform(label) +# return image, label \ No newline at end of file diff --git a/monai/brats_mri_2d/prep.py b/monai/brats_mri_2d/prep.py new file mode 100644 index 00000000..17766722 --- /dev/null +++ b/monai/brats_mri_2d/prep.py @@ -0,0 +1,4 @@ +# Obtain the dataset +from monai.apps import DecathlonDataset + +_ = DecathlonDataset(root_dir="/mnt/Datasets/Open-Datasets/MONAI", task="Task01_BrainTumour", section="training", download=True) \ No newline at end of file diff --git a/monai/brats_mri_2d/train_cycling_diff.py b/monai/brats_mri_2d/train_cycling_diff.py new file mode 100644 index 00000000..a79370a2 --- /dev/null +++ b/monai/brats_mri_2d/train_cycling_diff.py @@ -0,0 +1,260 @@ +from cycling_utils import Timer + +timer = Timer() +timer.report('importing Timer') + +import os + +import matplotlib.pyplot as plt +import numpy as np +import torch +import torch.nn.functional as F +from monai import transforms +from monai.apps import DecathlonDataset +from monai.config import print_config +from monai.data import DataLoader, Dataset +from monai.utils import first, set_determinism +from torch.cuda.amp import GradScaler, autocast +from pathlib import Path +from tqdm import tqdm + +from generative.inferers import LatentDiffusionInferer +from generative.losses.adversarial_loss import PatchAdversarialLoss +from generative.losses.perceptual import PerceptualLoss +from generative.networks.nets import AutoencoderKL, DiffusionModelUNet, PatchDiscriminator +from generative.networks.schedulers import DDPMScheduler + +from cycling_utils import InterruptableDistributedSampler, Timer +# from loops import train_generator_one_epoch, evaluate_generator +from loops import train_diffusion_one_epoch, evaluate_diffusion +import utils + +def get_args_parser(add_help=True): + import argparse + + parser = argparse.ArgumentParser(description="Latent Diffusion Model Training", add_help=add_help) + + parser.add_argument("--resume", type=str, help="path of checkpoint", required=True) # for checkpointing + # parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") + parser.add_argument("--data-path", default="/mnt/Datasets/Open-Datasets/MONAI", type=str, help="dataset path", dest="data_path") + parser.add_argument("--device", default="cuda", type=str, help="device (Use cuda or cpu Default: cuda)") + parser.add_argument("--start-epoch", default=0, type=int, metavar="N", help="start epoch") + # parser.add_argument("-b", "--batch-size", default=8, type=int, help="images per gpu, the total batch size is $NGPU x batch_size", dest="batch_size") + # parser.add_argument("--epochs", default=30, type=int, metavar="N", help="number of total epochs to run") + # parser.add_argument("--print-freq", default=1, type=int, help="print frequency") + parser.add_argument("--dist-url", default="env://", type=str, help="url used to set up distributed training") + + return parser + +channel = 0 # 0 = Flair +assert channel in [0, 1, 2, 3], "Choose a valid channel" +train_transforms = transforms.Compose([ + transforms.LoadImaged(keys=["image", "label"]), + transforms.EnsureChannelFirstd(keys=["image", "label"]), + transforms.Lambdad(keys=["image"], func=lambda x: x[channel, None, :, :, :]), + transforms.EnsureTyped(keys=["image", "label"]), + transforms.Orientationd(keys=["image", "label"], axcodes="RAS"), + transforms.Spacingd(keys=["image", "label"], pixdim=(3.0, 3.0, 2.0), mode=("bilinear", "nearest")), + transforms.CenterSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 44)), + transforms.ScaleIntensityRangePercentilesd(keys="image", lower=0, upper=99.5, b_min=0, b_max=1), + transforms.RandSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 1), random_size=False), + transforms.Lambdad(keys=["image", "label"], func=lambda x: x.squeeze(-1)), + transforms.CopyItemsd(keys=["label"], times=1, names=["slice_label"]), + transforms.Lambdad(keys=["slice_label"], func=lambda x: 1.0 if x.sum() > 0 else 0.0), +]) + +timer.report('importing everything else and prepping transforms') + +def main(args, timer): + + ## Distributed training prelims + if args.output_dir: + utils.mkdir(args.output_dir) + + utils.init_distributed_mode(args) # Sets args.distributed among other things + assert args.distributed # don't support cycling when not distributed for simplicity + + device = torch.device(args.device) + + timer.report('preliminaries') + + # Maybe this will work? + set_determinism(42) + + train_ds = DecathlonDataset( + root_dir=args.data_path, task="Task01_BrainTumour", section="training", cache_rate=1.0, + num_workers=4, download=False, seed=0, transform=train_transforms, + ) + val_ds = DecathlonDataset( + root_dir=args.data_path, task="Task01_BrainTumour", section="validation", cache_rate=1.0, + num_workers=4, download=False, seed=0, transform=train_transforms, + ) + + timer.report('build datasets') + + train_sampler = InterruptableDistributedSampler(train_ds) + val_sampler = InterruptableDistributedSampler(val_ds) + + timer.report('build samplers') + + train_loader = DataLoader(train_ds, batch_size=64, sampler=train_sampler, shuffle=True, num_workers=4, persistent_workers=True) + val_loader = DataLoader(val_ds, batch_size=64, sampler=val_sampler, shuffle=True, num_workers=4, persistent_workers=True) + check_data = first(train_loader) # Used later + + timer.report('build dataloaders') + + # # Auto-encoder definition + # generator = AutoencoderKL( + # spatial_dims=2, in_channels=1, out_channels=1, num_channels=(128, 128, 256), latent_channels=3, + # num_res_blocks=2, attention_levels=(False, False, False),with_encoder_nonlocal_attn=False, + # with_decoder_nonlocal_attn=False, + # ) + # generator = generator.to(device) + + # timer.report('generator to device') + + # # Discriminator definition + # discriminator = PatchDiscriminator( + # spatial_dims=2, num_layers_d=3, num_channels=64, + # in_channels=1, out_channels=1 + # ) + # discriminator = discriminator.to(device) + + # timer.report('discriminator to device') + + # Diffusion model (unet) + unet = DiffusionModelUNet( + spatial_dims=2, in_channels=3, out_channels=3, num_res_blocks=2, + num_channels=(128, 256, 512),attention_levels=(False, True, True), + num_head_channels=(0, 256, 512), + ) + unet = unet.to(device) + + timer.report('unet to device') + + # # Autoencoder loss functions + # adv_loss = PatchAdversarialLoss(criterion="least_squares") + # perceptual_loss = PerceptualLoss(spatial_dims=2, network_type="alex") + # perceptual_loss.to(device) + + # timer.report('loss functions') + + # Prepare for distributed training + # generator_without_ddp = generator + # discriminator_without_ddp = discriminator + unet_without_ddp = unet + # perceptual_loss_without_ddp = perceptual_loss + if args.distributed: + # generator = torch.nn.parallel.DistributedDataParallel(generator, device_ids=[args.gpu]) + # discriminator = torch.nn.parallel.DistributedDataParallel(discriminator, device_ids=[args.gpu]) + unet = torch.nn.parallel.DistributedDataParallel(unet, device_ids=[args.gpu]) + # perceptual_loss = torch.nn.parallel.DistributedDataParallel(perceptual_loss, device_ids=[args.gpu]) + # generator_without_ddp = generator.module + # discriminator_without_ddp = discriminator.module + unet_without_ddp = unet.module + # perceptual_loss_without_ddp = perceptual_loss.module + + timer.report('models prepped for distribution') + + # Optimizers + # optimizer_g = torch.optim.Adam(generator_without_ddp.parameters(), lr=1e-4) + # optimizer_d = torch.optim.Adam(discriminator_without_ddp.parameters(), lr=5e-4) + optimizer_u = torch.optim.Adam(unet_without_ddp.parameters(), lr=1e-4) + + timer.report('optimizers') + + # For mixed precision training + # scaler_g = GradScaler() + # scaler_d = GradScaler() + scaler_u = GradScaler() + + timer.report('grad scalers') + + # RETRIEVE CHECKPOINT + Path(args.resume).parent.mkdir(parents=True, exist_ok=True) + if args.resume and os.path.isfile(args.resume): # If we're resuming... + + checkpoint = torch.load(args.resume, map_location="cpu") + # generator_without_ddp.load_state_dict(checkpoint["generator"], strict=not args.test_only) + # discriminator_without_ddp.load_state_dict(checkpoint["discriminator"], strict=not args.test_only) + unet_without_ddp.load_state_dict(checkpoint["unet"], strict=not args.test_only) + + args.start_epoch = checkpoint["epoch"] + + # optimizer.load_state_dict(checkpoint["optimizer"]) + # lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) + # train_sampler.load_state_dict(checkpoint["train_sampler"]) + # if args.amp: # Could align this syntactically... + # scaler.load_state_dict(checkpoint["scaler"]) + + # test_sampler.load_state_dict(checkpoint["test_sampler"]) + # confmat.mat = checkpoint["confmat"] + # confmat.temp_mat = checkpoint["confmat_temp"] + + timer.report('checkpoint retrieval') + + # ## -- TRAINING THE AUTO-ENCODER - ## + + # n_gen_epochs = 100 + # gen_val_interval = 1 + + # for epoch in range(n_gen_epochs): + + # print('\n') + # print(f"EPOCH (gen) :: {epoch}") + # print('\n') + + # with train_sampler.in_epoch(epoch): + # timer = Timer() + # generator, timer, _, _, _ = train_generator_one_epoch( + # epoch, generator, discriminator, optimizer_g, optimizer_d, + # scaler_g, scaler_d, train_loader, perceptual_loss, adv_loss, device, timer + # ) + # timer.report(f'training generator for epoch {epoch}') + + # if epoch % gen_val_interval == 0: # Eval every epoch + # with val_sampler.in_epoch(epoch): + # timer = Timer() + # timer, _ = evaluate_generator(epoch, generator, val_loader, device, timer) + # timer.report(f'evaluating generator for epoch {epoch}') + + + ## -- TRAINING THE DIFFUSION MODEL - ## + + n_diff_epochs = 200 + diff_val_interval = 1 + + # Prepare LatentDiffusionInferer + scheduler = DDPMScheduler(num_train_timesteps=1000, schedule="linear_beta", beta_start=0.0015, beta_end=0.0195) + with torch.no_grad(): + with autocast(enabled=True): + z = generator.encode_stage_2_inputs(check_data["image"].to(device)) + scale_factor = 1 / torch.std(z) + inferer = LatentDiffusionInferer(scheduler, scale_factor=scale_factor) + + timer.report('building inferer') + + for epoch in range(n_diff_epochs): + + print('\n') + print(f"EPOCH (diff) :: {epoch}") + print('\n') + + with train_sampler.in_epoch(epoch): + timer = Timer() + unet, timer, _ = train_diffusion_one_epoch( + epoch, unet, generator, optimizer_u, + inferer, scaler_u, train_loader, device, timer + ) + timer.report(f'training unet for epoch {epoch}') + + if epoch % diff_val_interval == 0: + with val_sampler.in_epoch(epoch): + timer = Timer() + _ = evaluate_diffusion(epoch, unet, generator, inferer, val_loader, device) + timer.report(f'evaluating unet for epoch {epoch}') + + +if __name__ == "__main__": + args = get_args_parser().parse_args() + main(args, timer) diff --git a/monai/brats_mri_2d/train_cycling_gen.py b/monai/brats_mri_2d/train_cycling_gen.py new file mode 100644 index 00000000..5dc5fcd9 --- /dev/null +++ b/monai/brats_mri_2d/train_cycling_gen.py @@ -0,0 +1,279 @@ +from cycling_utils import Timer + +timer = Timer() +timer.report('importing Timer') + +import os + +# import matplotlib.pyplot as plt +# import numpy as np +import torch +# import torch.nn.functional as F +from monai import transforms +from monai.apps import DecathlonDataset +# from monai.config import print_config +from monai.data import DataLoader# , Dataset +from monai.utils import first, set_determinism +from torch.cuda.amp import GradScaler# , autocast +from pathlib import Path +from tqdm import tqdm + +# from generative.inferers import LatentDiffusionInferer +from generative.losses.adversarial_loss import PatchAdversarialLoss +from generative.losses.perceptual import PerceptualLoss +from generative.networks.nets import AutoencoderKL, PatchDiscriminator # , DiffusionModelUNet +# from generative.networks.schedulers import DDPMScheduler + +from cycling_utils import InterruptableDistributedSampler, Timer +from loops import train_generator_one_epoch, evaluate_generator +# from loops import train_diffusion_one_epoch, evaluate_diffusion +import utils + +def get_args_parser(add_help=True): + import argparse + + parser = argparse.ArgumentParser(description="Latent Diffusion Model Training", add_help=add_help) + + parser.add_argument("--resume", type=str, help="path of checkpoint", required=True) # for checkpointing + # parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") + parser.add_argument("--data-path", default="/mnt/Datasets/Open-Datasets/MONAI", type=str, help="dataset path", dest="data_path") + parser.add_argument("--device", default="cuda", type=str, help="device (Use cuda or cpu Default: cuda)") + parser.add_argument("--start-epoch", default=0, type=int, metavar="N", help="start epoch") + # parser.add_argument("-b", "--batch-size", default=8, type=int, help="images per gpu, the total batch size is $NGPU x batch_size", dest="batch_size") + # parser.add_argument("--epochs", default=30, type=int, metavar="N", help="number of total epochs to run") + # parser.add_argument("--print-freq", default=1, type=int, help="print frequency") + parser.add_argument("--dist-url", default="env://", type=str, help="url used to set up distributed training") + parser.add_argument("-j", "--workers", default=16, type=int, metavar="N", help="number of data loading workers (default: 16)") + + return parser + +timer.report('importing everything else') + +def main(args, timer): + + # ## Distributed training prelims + # if args.output_dir: + # utils.mkdir(args.output_dir) + + utils.init_distributed_mode(args) # Sets args.distributed among other things + assert args.distributed # don't support cycling when not distributed for simplicity + + device = torch.device(args.device) + + timer.report('preliminaries') + + # Maybe this will work? + set_determinism(42) + + channel = 0 # 0 = Flair + assert channel in [0, 1, 2, 3], "Choose a valid channel" + train_transforms = transforms.Compose([ + transforms.LoadImaged(keys=["image", "label"]), + transforms.EnsureChannelFirstd(keys=["image", "label"]), + transforms.Lambdad(keys=["image"], func=lambda x: x[channel, None, :, :, :]), + transforms.EnsureTyped(keys=["image", "label"]), + transforms.Orientationd(keys=["image", "label"], axcodes="RAS"), + transforms.Spacingd(keys=["image", "label"], pixdim=(3.0, 3.0, 2.0), mode=("bilinear", "nearest")), + transforms.CenterSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 44)), + transforms.ScaleIntensityRangePercentilesd(keys="image", lower=0, upper=99.5, b_min=0, b_max=1), + transforms.RandSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 1), random_size=False), + transforms.Lambdad(keys=["image", "label"], func=lambda x: x.squeeze(-1)), + transforms.CopyItemsd(keys=["label"], times=1, names=["slice_label"]), + transforms.Lambdad(keys=["slice_label"], func=lambda x: 1.0 if x.sum() > 0 else 0.0), + ]) + + train_ds = DecathlonDataset( + root_dir=args.data_path, task="Task01_BrainTumour", section="training", cache_rate=0.0, + num_workers=4, download=False, seed=0, transform=train_transforms, + ) + val_ds = DecathlonDataset( + root_dir=args.data_path, task="Task01_BrainTumour", section="validation", cache_rate=0.0, + num_workers=4, download=False, seed=0, transform=train_transforms, + ) + + # ## SUBSET FOR TESTING + # train_ds = torch.utils.data.Subset(train_ds, torch.arange(2*9*3)) # batch_size x nodes x iterations + # val_ds = torch.utils.data.Subset(val_ds, torch.arange(1*9*2)) # batch_size x nodes x iterations + + timer.report('build datasets') + + train_sampler = InterruptableDistributedSampler(train_ds) + val_sampler = InterruptableDistributedSampler(val_ds) + + timer.report('build samplers') + + train_loader = DataLoader(train_ds, batch_size=2, sampler=train_sampler, num_workers=args.workers) + val_loader = DataLoader(val_ds, batch_size=1, sampler=val_sampler, num_workers=args.workers) + # check_data = first(train_loader) # Used later + + timer.report('build dataloaders') + + # Auto-encoder definition + generator = AutoencoderKL( + spatial_dims=2, in_channels=1, out_channels=1, num_channels=(128, 128, 256), + latent_channels=3, num_res_blocks=2, attention_levels=(False, False, False), + with_encoder_nonlocal_attn=False, with_decoder_nonlocal_attn=False, + ) + generator = generator.to(device) + + timer.report('generator to device') + + # Discriminator definition + discriminator = PatchDiscriminator( + spatial_dims=2, num_layers_d=3, num_channels=64, + in_channels=1, out_channels=1 + ) + discriminator = discriminator.to(device) + + timer.report('discriminator to device') + + # # Diffusion model (unet) + # unet = DiffusionModelUNet( + # spatial_dims=2, in_channels=3, out_channels=3, num_res_blocks=2, + # num_channels=(128, 256, 512),attention_levels=(False, True, True), + # num_head_channels=(0, 256, 512), + # ) + # unet = unet.to(device) + + # timer.report('unet to device') + + # Autoencoder loss functions + adv_loss = PatchAdversarialLoss(criterion="least_squares") + perceptual_loss = PerceptualLoss(spatial_dims=2, network_type="alex") + perceptual_loss.to(device) + + timer.report('loss functions') + + # Prepare for distributed training + generator_without_ddp = generator + discriminator_without_ddp = discriminator + # unet_without_ddp = unet + # perceptual_loss_without_ddp = perceptual_loss + if args.distributed: + print('Confirm distributed training.') + generator = torch.nn.parallel.DistributedDataParallel(generator, device_ids=[args.gpu]) + discriminator = torch.nn.parallel.DistributedDataParallel(discriminator, device_ids=[args.gpu]) + # unet = torch.nn.parallel.DistributedDataParallel(unet, device_ids=[args.gpu]) + # perceptual_loss = torch.nn.parallel.DistributedDataParallel(perceptual_loss, device_ids=[args.gpu]) + generator_without_ddp = generator.module + discriminator_without_ddp = discriminator.module + # unet_without_ddp = unet.module + # perceptual_loss_without_ddp = perceptual_loss.module + + timer.report('models prepped for distribution') + + # Optimizers + optimizer_g = torch.optim.Adam(generator_without_ddp.parameters(), lr=1e-4) + optimizer_d = torch.optim.Adam(discriminator_without_ddp.parameters(), lr=5e-4) + # optimizer_u = torch.optim.Adam(unet_without_ddp.parameters(), lr=1e-4) + + timer.report('optimizers') + + # For mixed precision training + scaler_g = GradScaler() + scaler_d = GradScaler() + # scaler_u = GradScaler() + + timer.report('grad scalers') + + # Init tracking metrics + train_images_seen = 0 + val_images_seen = 0 + epoch_loss = 0 + gen_epoch_loss = 0 + disc_epoch_loss = 0 + val_loss = 0 + + # RETRIEVE CHECKPOINT + + Path(args.resume).parent.mkdir(parents=True, exist_ok=True) + if args.resume and os.path.isfile(args.resume): # If we're resuming... + checkpoint = torch.load(args.resume, map_location="cpu") + args.start_epoch = checkpoint["epoch"] + generator_without_ddp.load_state_dict(checkpoint["generator"]) + discriminator_without_ddp.load_state_dict(checkpoint["discriminator"]) + optimizer_g.load_state_dict(checkpoint["optimizer_g"]) + optimizer_d.load_state_dict(checkpoint["optimizer_d"]) + scaler_g.load_state_dict(checkpoint["scaler_g"]) + scaler_d.load_state_dict(checkpoint["scaler_d"]) + train_sampler.load_state_dict(checkpoint["train_sampler"]) + val_sampler.load_state_dict(checkpoint["val_sampler"]) + train_images_seen = checkpoint["train_images_seen"] + val_images_seen = checkpoint["val_images_seen"] + epoch_loss = checkpoint["epoch_loss"] + gen_epoch_loss = checkpoint["gen_epoch_loss"] + disc_epoch_loss = checkpoint["disc_epoch_loss"] + val_loss = checkpoint["val_loss"] + + timer.report('checkpoint retrieval') + + ## -- TRAINING THE AUTO-ENCODER - ## + + n_gen_epochs = 100 + gen_val_interval = 1 + + for epoch in range(args.start_epoch, n_gen_epochs): + + print('\n') + print(f"EPOCH (gen) :: {epoch}") + print('\n') + + with train_sampler.in_epoch(epoch): + timer = Timer("Start training") + generator, timer = train_generator_one_epoch( + args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, + scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, + train_images_seen, val_images_seen, epoch_loss, gen_epoch_loss, disc_epoch_loss, val_loss + ) + timer.report(f'training generator for epoch {epoch}') + + if epoch % gen_val_interval == 0: # Eval every epoch + with val_sampler.in_epoch(epoch): + timer = Timer("Start evaluation") + timer = evaluate_generator( + args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, + scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, + train_images_seen, val_images_seen, epoch_loss, gen_epoch_loss, disc_epoch_loss, val_loss + ) + timer.report(f'evaluating generator for epoch {epoch}') + + + # ## -- TRAINING THE DIFFUSION MODEL - ## + + # n_diff_epochs = 200 + # diff_val_interval = 1 + + # # Prepare LatentDiffusionInferer + # scheduler = DDPMScheduler(num_train_timesteps=1000, schedule="linear_beta", beta_start=0.0015, beta_end=0.0195) + # with torch.no_grad(): + # with autocast(enabled=True): + # z = generator.encode_stage_2_inputs(check_data["image"].to(device)) + # scale_factor = 1 / torch.std(z) + # inferer = LatentDiffusionInferer(scheduler, scale_factor=scale_factor) + + # timer.report('building inferer') + + # for epoch in range(n_diff_epochs): + + # print('\n') + # print(f"EPOCH (diff) :: {epoch}") + # print('\n') + + # with train_sampler.in_epoch(epoch): + # timer = Timer() + # unet, timer, _ = train_diffusion_one_epoch( + # epoch, unet, generator, optimizer_u, + # inferer, scaler_u, train_loader, device, timer + # ) + # timer.report(f'training unet for epoch {epoch}') + + # if epoch % diff_val_interval == 0: + # with val_sampler.in_epoch(epoch): + # timer = Timer() + # _ = evaluate_diffusion(epoch, unet, generator, inferer, val_loader, device) + # timer.report(f'evaluating unet for epoch {epoch}') + + +if __name__ == "__main__": + args = get_args_parser().parse_args() + main(args, timer) diff --git a/monai/brats_mri_2d/utils.py b/monai/brats_mri_2d/utils.py new file mode 100644 index 00000000..1b268b82 --- /dev/null +++ b/monai/brats_mri_2d/utils.py @@ -0,0 +1,72 @@ +import torch, os, errno +import torch.distributed as dist + +def mkdir(path): + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + import builtins as __builtin__ + + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop("force", False) + if is_master or force: + builtin_print(*args, **kwargs) + + __builtin__.print = print + +def init_distributed_mode(args): + if "RANK" in os.environ and "WORLD_SIZE" in os.environ: + args.rank = int(os.environ["RANK"]) + args.world_size = int(os.environ["WORLD_SIZE"]) + args.gpu = int(os.environ["LOCAL_RANK"]) + elif "SLURM_PROCID" in os.environ: + args.rank = int(os.environ["SLURM_PROCID"]) + args.gpu = args.rank % torch.cuda.device_count() + else: + print("Not using distributed mode") + args.distributed = False + return + + args.distributed = True + + torch.cuda.set_device(args.gpu) + args.dist_backend = "nccl" + print(f"| distributed init (rank {args.rank}): {args.dist_url}", flush=True) + torch.distributed.init_process_group( + backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank + ) + torch.distributed.barrier() + setup_for_distributed(args.rank == 0) + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def is_main_process(): + return get_rank() == 0 \ No newline at end of file diff --git a/tv-detection/SS/COCO_edits/COCO_interface.py b/tv-detection/SS/COCO_edits/COCO_interface.py new file mode 100644 index 00000000..c1b48eb0 --- /dev/null +++ b/tv-detection/SS/COCO_edits/COCO_interface.py @@ -0,0 +1,521 @@ + +import os.path +from typing import Any, Callable, List, Optional, Tuple + +from PIL import Image + +from torchvision.datasets import VisionDataset + + +class torch_CocoDetection(VisionDataset): + """`MS Coco Detection `_ Dataset. + + It requires the `COCO API to be installed `_. + + Args: + root (string): Root directory where images are downloaded to. + annFile (string): Path to json annotation file. + transform (callable, optional): A function/transform that takes in an PIL image + and returns a transformed version. E.g, ``transforms.PILToTensor`` + target_transform (callable, optional): A function/transform that takes in the + target and transforms it. + transforms (callable, optional): A function/transform that takes input sample and its target as entry + and returns a transformed version. + """ + + def __init__( + self, + root: str, + annFile: str, + transform: Optional[Callable] = None, + target_transform: Optional[Callable] = None, + transforms: Optional[Callable] = None, + ) -> None: + super().__init__(root, transforms, transform, target_transform) + # from pycocotools.coco import COCO + + self.coco = COCO(annFile) + self.ids = list(sorted(self.coco.imgs.keys())) + + def _load_image(self, id: int) -> Image.Image: + path = self.coco.loadImgs(id)[0]["file_name"] + return Image.open(os.path.join(self.root, path)).convert("RGB") + + def _load_target(self, id: int) -> List[Any]: + return self.coco.loadAnns(self.coco.getAnnIds(id)) + + def __getitem__(self, index: int) -> Tuple[Any, Any]: + id = self.ids[index] + image = self._load_image(id) + target = self._load_target(id) + + if self.transforms is not None: + image, target = self.transforms(image, target) + + return image, target + + + def __len__(self) -> int: + return len(self.ids) + + + + +__author__ = 'tylin' +__version__ = '2.0' +# Interface for accessing the Microsoft COCO dataset. + +# Microsoft COCO is a large image dataset designed for object detection, +# segmentation, and caption generation. pycocotools is a Python API that +# assists in loading, parsing and visualizing the annotations in COCO. +# Please visit http://mscoco.org/ for more information on COCO, including +# for the data, paper, and tutorials. The exact format of the annotations +# is also described on the COCO website. For example usage of the pycocotools +# please see pycocotools_demo.ipynb. In addition to this API, please download both +# the COCO images and annotations in order to run the demo. + +# An alternative to using the API is to load the annotations directly +# into Python dictionary +# Using the API provides additional utility functions. Note that this API +# supports both *instance* and *caption* annotations. In the case of +# captions not all functions are defined (e.g. categories are undefined). + +# The following API functions are defined: +# COCO - COCO api class that loads COCO annotation file and prepare data structures. +# decodeMask - Decode binary mask M encoded via run-length encoding. +# encodeMask - Encode binary mask M using run-length encoding. +# getAnnIds - Get ann ids that satisfy given filter conditions. +# getCatIds - Get cat ids that satisfy given filter conditions. +# getImgIds - Get img ids that satisfy given filter conditions. +# loadAnns - Load anns with the specified ids. +# loadCats - Load cats with the specified ids. +# loadImgs - Load imgs with the specified ids. +# annToMask - Convert segmentation in an annotation to binary mask. +# showAnns - Display the specified annotations. +# loadRes - Load algorithm results and create API for accessing them. +# download - Download COCO images from mscoco.org server. +# Throughout the API "ann"=annotation, "cat"=category, and "img"=image. +# Help on each functions can be accessed by: "help COCO>function". + +# See also COCO>decodeMask, +# COCO>encodeMask, COCO>getAnnIds, COCO>getCatIds, +# COCO>getImgIds, COCO>loadAnns, COCO>loadCats, +# COCO>loadImgs, COCO>annToMask, COCO>showAnns + +# Microsoft COCO Toolbox. version 2.0 +# Data, paper, and tutorials available at: http://mscoco.org/ +# Code written by Piotr Dollar and Tsung-Yi Lin, 2014. +# Licensed under the Simplified BSD License [see bsd.txt] + +import json +import time +import matplotlib.pyplot as plt +from matplotlib.collections import PatchCollection +from matplotlib.patches import Polygon +import numpy as np +import copy +import itertools +from pycocotools import mask as maskUtils +import os +from collections import defaultdict +from itertools import groupby +import sys +PYTHON_VERSION = sys.version_info[0] +if PYTHON_VERSION == 2: + from urllib import urlretrieve +elif PYTHON_VERSION == 3: + from urllib.request import urlretrieve + + +def _isArrayLike(obj): + return hasattr(obj, '__iter__') and hasattr(obj, '__len__') + + +class COCO: + def __init__(self, annotation_file=None): + """ + Constructor of Microsoft COCO helper class for reading and visualizing annotations. + :param annotation_file (str): location of annotation file + :param image_folder (str): location to the folder that hosts images. + :return: + """ + # load dataset + self.dataset,self.anns,self.cats,self.imgs = dict(),dict(),dict(),dict() + self.imgToAnns, self.catToImgs = defaultdict(list), defaultdict(list) + if not annotation_file == None: + print('loading annotations into memory...') + tic = time.time() + dataset = json.load(open(annotation_file, 'r')) + assert type(dataset)==dict, 'annotation file format {} not supported'.format(type(dataset)) + print('Done (t={:0.2f}s)'.format(time.time()- tic)) + self.dataset = dataset + self.createIndex() + + def createIndex(self): + # create index + print('creating index...') + # anns, cats, imgs = {}, {}, {} + # imgToAnns,catToImgs = defaultdict(list),defaultdict(list) + + # self.dataset = { + # 'images': [{'id': ...}], + # 'annotations': [{'image_id': ..., 'id': ..., }, {...}, ...], + # 'categories': [...], + # } + + if 'annotations' in self.dataset: + # for ann in self.dataset['annotations']: + # imgToAnns[ann['image_id']].append(ann) + # anns[ann['id']] = ann + + anns_by_iid = groupby(self.dataset['annotations'], lambda ann: ann['image_id']) + self.imgToAnns = {iid:anns_group for iid,anns_group in anns_by_iid} + self.anns = {ann['id']:ann for ann in self.dataset['annotations']} + + if 'images' in self.dataset: + # for img in self.dataset['images']: + # imgs[img['id']] = img + self.imgs = {img['id']:img for img in self.dataset['images']} + + if 'categories' in self.dataset: + # for cat in self.dataset['categories']: + # cats[cat['id']] = cat + self.cats = {cat['id']:cat for cat in self.dataset['categories']} + + if 'annotations' in self.dataset and 'categories' in self.dataset: + # for ann in self.dataset['annotations']: + # catToImgs[ann['category_id']].append(ann['image_id']) + anns_by_cid = groupby(self.dataset['annotations'], lambda ann: ann['category_id']) + self.catToImgs = {cid:anns_group for cid,anns_group in anns_by_cid} + + + print('index created!') + + # create class members + # self.anns = anns + # self.imgToAnns = imgToAnns + # self.catToImgs = catToImgs + # self.imgs = imgs + # self.cats = cats + + def info(self): + """ + Print information about the annotation file. + :return: + """ + for key, value in self.dataset['info'].items(): + print('{}: {}'.format(key, value)) + + def getAnnIds(self, imgIds=[], catIds=[], areaRng=[], iscrowd=None): + """ + Get ann ids that satisfy given filter conditions. default skips that filter + :param imgIds (int array) : get anns for given imgs + catIds (int array) : get anns for given cats + areaRng (float array) : get anns for given area range (e.g. [0 inf]) + iscrowd (boolean) : get anns for given crowd label (False or True) + :return: ids (int array) : integer array of ann ids + """ + imgIds = imgIds if _isArrayLike(imgIds) else [imgIds] + catIds = catIds if _isArrayLike(catIds) else [catIds] + + if len(imgIds) == len(catIds) == len(areaRng) == 0: + anns = self.dataset['annotations'] + else: + if not len(imgIds) == 0: + lists = [self.imgToAnns[imgId] for imgId in imgIds if imgId in self.imgToAnns] + anns = list(itertools.chain.from_iterable(lists)) + else: + anns = self.dataset['annotations'] + anns = anns if len(catIds) == 0 else [ann for ann in anns if ann['category_id'] in catIds] + anns = anns if len(areaRng) == 0 else [ann for ann in anns if ann['area'] > areaRng[0] and ann['area'] < areaRng[1]] + if not iscrowd == None: + ids = [ann['id'] for ann in anns if ann['iscrowd'] == iscrowd] + else: + ids = [ann['id'] for ann in anns] + return ids + + def getCatIds(self, catNms=[], supNms=[], catIds=[]): + """ + filtering parameters. default skips that filter. + :param catNms (str array) : get cats for given cat names + :param supNms (str array) : get cats for given supercategory names + :param catIds (int array) : get cats for given cat ids + :return: ids (int array) : integer array of cat ids + """ + catNms = catNms if _isArrayLike(catNms) else [catNms] + supNms = supNms if _isArrayLike(supNms) else [supNms] + catIds = catIds if _isArrayLike(catIds) else [catIds] + + if len(catNms) == len(supNms) == len(catIds) == 0: + cats = self.dataset['categories'] + else: + cats = self.dataset['categories'] + cats = cats if len(catNms) == 0 else [cat for cat in cats if cat['name'] in catNms] + cats = cats if len(supNms) == 0 else [cat for cat in cats if cat['supercategory'] in supNms] + cats = cats if len(catIds) == 0 else [cat for cat in cats if cat['id'] in catIds] + ids = [cat['id'] for cat in cats] + return ids + + def getImgIds(self, imgIds=[], catIds=[]): + ''' + Get img ids that satisfy given filter conditions. + :param imgIds (int array) : get imgs for given ids + :param catIds (int array) : get imgs with all given cats + :return: ids (int array) : integer array of img ids + ''' + imgIds = imgIds if _isArrayLike(imgIds) else [imgIds] + catIds = catIds if _isArrayLike(catIds) else [catIds] + + if len(imgIds) == len(catIds) == 0: + ids = self.imgs.keys() + else: + ids = set(imgIds) + for i, catId in enumerate(catIds): + if i == 0 and len(ids) == 0: + ids = set(self.catToImgs[catId]) + else: + ids &= set(self.catToImgs[catId]) + return list(ids) + + def loadAnns(self, ids=[]): + """ + Load anns with the specified ids. + :param ids (int array) : integer ids specifying anns + :return: anns (object array) : loaded ann objects + """ + if _isArrayLike(ids): + return [self.anns[id] for id in ids] + elif type(ids) == int: + return [self.anns[ids]] + + def loadCats(self, ids=[]): + """ + Load cats with the specified ids. + :param ids (int array) : integer ids specifying cats + :return: cats (object array) : loaded cat objects + """ + if _isArrayLike(ids): + return [self.cats[id] for id in ids] + elif type(ids) == int: + return [self.cats[ids]] + + def loadImgs(self, ids=[]): + """ + Load anns with the specified ids. + :param ids (int array) : integer ids specifying img + :return: imgs (object array) : loaded img objects + """ + if _isArrayLike(ids): + return [self.imgs[id] for id in ids] + elif type(ids) == int: + return [self.imgs[ids]] + + def showAnns(self, anns, draw_bbox=False): + """ + Display the specified annotations. + :param anns (array of object): annotations to display + :return: None + """ + if len(anns) == 0: + return 0 + if 'segmentation' in anns[0] or 'keypoints' in anns[0]: + datasetType = 'instances' + elif 'caption' in anns[0]: + datasetType = 'captions' + else: + raise Exception('datasetType not supported') + if datasetType == 'instances': + ax = plt.gca() + ax.set_autoscale_on(False) + polygons = [] + color = [] + for ann in anns: + c = (np.random.random((1, 3))*0.6+0.4).tolist()[0] + if 'segmentation' in ann: + if type(ann['segmentation']) == list: + # polygon + for seg in ann['segmentation']: + poly = np.array(seg).reshape((int(len(seg)/2), 2)) + polygons.append(Polygon(poly)) + color.append(c) + else: + # mask + t = self.imgs[ann['image_id']] + if type(ann['segmentation']['counts']) == list: + rle = maskUtils.frPyObjects([ann['segmentation']], t['height'], t['width']) + else: + rle = [ann['segmentation']] + m = maskUtils.decode(rle) + img = np.ones( (m.shape[0], m.shape[1], 3) ) + if ann['iscrowd'] == 1: + color_mask = np.array([2.0,166.0,101.0])/255 + if ann['iscrowd'] == 0: + color_mask = np.random.random((1, 3)).tolist()[0] + for i in range(3): + img[:,:,i] = color_mask[i] + ax.imshow(np.dstack( (img, m*0.5) )) + if 'keypoints' in ann and type(ann['keypoints']) == list: + # turn skeleton into zero-based index + sks = np.array(self.loadCats(ann['category_id'])[0]['skeleton'])-1 + kp = np.array(ann['keypoints']) + x = kp[0::3] + y = kp[1::3] + v = kp[2::3] + for sk in sks: + if np.all(v[sk]>0): + plt.plot(x[sk],y[sk], linewidth=3, color=c) + plt.plot(x[v>0], y[v>0],'o',markersize=8, markerfacecolor=c, markeredgecolor='k',markeredgewidth=2) + plt.plot(x[v>1], y[v>1],'o',markersize=8, markerfacecolor=c, markeredgecolor=c, markeredgewidth=2) + + if draw_bbox: + [bbox_x, bbox_y, bbox_w, bbox_h] = ann['bbox'] + poly = [[bbox_x, bbox_y], [bbox_x, bbox_y+bbox_h], [bbox_x+bbox_w, bbox_y+bbox_h], [bbox_x+bbox_w, bbox_y]] + np_poly = np.array(poly).reshape((4,2)) + polygons.append(Polygon(np_poly)) + color.append(c) + + p = PatchCollection(polygons, facecolor=color, linewidths=0, alpha=0.4) + ax.add_collection(p) + p = PatchCollection(polygons, facecolor='none', edgecolors=color, linewidths=2) + ax.add_collection(p) + elif datasetType == 'captions': + for ann in anns: + print(ann['caption']) + + def loadRes(self, resFile): + """ + Load result file and return a result api object. + :param resFile (str) : file name of result file + :return: res (obj) : result api object + """ + res = COCO() + res.dataset['images'] = [img for img in self.dataset['images']] + + print('Loading and preparing results...') + tic = time.time() + if type(resFile) == str or (PYTHON_VERSION == 2 and type(resFile) == unicode): + anns = json.load(open(resFile)) + elif type(resFile) == np.ndarray: + anns = self.loadNumpyAnnotations(resFile) + else: + anns = resFile + assert type(anns) == list, 'results in not an array of objects' + annsImgIds = [ann['image_id'] for ann in anns] + assert set(annsImgIds) == (set(annsImgIds) & set(self.getImgIds())), \ + 'Results do not correspond to current coco set' + if 'caption' in anns[0]: + imgIds = set([img['id'] for img in res.dataset['images']]) & set([ann['image_id'] for ann in anns]) + res.dataset['images'] = [img for img in res.dataset['images'] if img['id'] in imgIds] + for id, ann in enumerate(anns): + ann['id'] = id+1 + elif 'bbox' in anns[0] and not anns[0]['bbox'] == []: + res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) + for id, ann in enumerate(anns): + bb = ann['bbox'] + x1, x2, y1, y2 = [bb[0], bb[0]+bb[2], bb[1], bb[1]+bb[3]] + if not 'segmentation' in ann: + ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]] + ann['area'] = bb[2]*bb[3] + ann['id'] = id+1 + ann['iscrowd'] = 0 + elif 'segmentation' in anns[0]: + res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) + for id, ann in enumerate(anns): + # now only support compressed RLE format as segmentation results + ann['area'] = maskUtils.area(ann['segmentation']) + if not 'bbox' in ann: + ann['bbox'] = maskUtils.toBbox(ann['segmentation']) + ann['id'] = id+1 + ann['iscrowd'] = 0 + elif 'keypoints' in anns[0]: + res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) + for id, ann in enumerate(anns): + s = ann['keypoints'] + x = s[0::3] + y = s[1::3] + x0,x1,y0,y1 = np.min(x), np.max(x), np.min(y), np.max(y) + ann['area'] = (x1-x0)*(y1-y0) + ann['id'] = id + 1 + ann['bbox'] = [x0,y0,x1-x0,y1-y0] + print('DONE (t={:0.2f}s)'.format(time.time()- tic)) + + res.dataset['annotations'] = anns + res.createIndex() + return res + + def download(self, tarDir = None, imgIds = [] ): + ''' + Download COCO images from mscoco.org server. + :param tarDir (str): COCO results directory name + imgIds (list): images to be downloaded + :return: + ''' + if tarDir is None: + print('Please specify target directory') + return -1 + if len(imgIds) == 0: + imgs = self.imgs.values() + else: + imgs = self.loadImgs(imgIds) + N = len(imgs) + if not os.path.exists(tarDir): + os.makedirs(tarDir) + for i, img in enumerate(imgs): + tic = time.time() + fname = os.path.join(tarDir, img['file_name']) + if not os.path.exists(fname): + urlretrieve(img['coco_url'], fname) + print('downloaded {}/{} images (t={:0.1f}s)'.format(i, N, time.time()- tic)) + + def loadNumpyAnnotations(self, data): + """ + Convert result data from a numpy array [Nx7] where each row contains {imageID,x1,y1,w,h,score,class} + :param data (numpy.ndarray) + :return: annotations (python nested list) + """ + print('Converting ndarray to lists...') + assert(type(data) == np.ndarray) + print(data.shape) + assert(data.shape[1] == 7) + N = data.shape[0] + ann = [] + for i in range(N): + if i % 1000000 == 0: + print('{}/{}'.format(i,N)) + ann += [{ + 'image_id' : int(data[i, 0]), + 'bbox' : [ data[i, 1], data[i, 2], data[i, 3], data[i, 4] ], + 'score' : data[i, 5], + 'category_id': int(data[i, 6]), + }] + return ann + + def annToRLE(self, ann): + """ + Convert annotation which can be polygons, uncompressed RLE to RLE. + :return: binary mask (numpy 2D array) + """ + t = self.imgs[ann['image_id']] + h, w = t['height'], t['width'] + segm = ann['segmentation'] + if type(segm) == list: + # polygon -- a single object might consist of multiple parts + # we merge all parts into one mask rle code + rles = maskUtils.frPyObjects(segm, h, w) + rle = maskUtils.merge(rles) + elif type(segm['counts']) == list: + # uncompressed RLE + rle = maskUtils.frPyObjects(segm, h, w) + else: + # rle + rle = ann['segmentation'] + return rle + + def annToMask(self, ann): + """ + Convert annotation which can be polygons, uncompressed RLE, or RLE to binary mask. + :return: binary mask (numpy 2D array) + """ + rle = self.annToRLE(ann) + m = maskUtils.decode(rle) + return m \ No newline at end of file diff --git a/tv-detection/SS/COCO_edits/coco_utils_SC.py b/tv-detection/SS/COCO_edits/coco_utils_SC.py new file mode 100644 index 00000000..a807e3a8 --- /dev/null +++ b/tv-detection/SS/COCO_edits/coco_utils_SC.py @@ -0,0 +1,234 @@ +import os + +import torch +import torch.utils.data +import torchvision +import transforms as T +from pycocotools import mask as coco_mask +from COCO_interface import torch_CocoDetection, COCO + + +def convert_coco_poly_to_mask(segmentations, height, width): + masks = [] + for polygons in segmentations: + rles = coco_mask.frPyObjects(polygons, height, width) + mask = coco_mask.decode(rles) + if len(mask.shape) < 3: + mask = mask[..., None] + mask = torch.as_tensor(mask, dtype=torch.uint8) + mask = mask.any(dim=2) + masks.append(mask) + if masks: + masks = torch.stack(masks, dim=0) + else: + masks = torch.zeros((0, height, width), dtype=torch.uint8) + return masks + + +class ConvertCocoPolysToMask: + def __call__(self, image, target): + w, h = image.size + + image_id = target["image_id"] + + anno = target["annotations"] + + anno = [obj for obj in anno if obj["iscrowd"] == 0] + + boxes = [obj["bbox"] for obj in anno] + # guard against no boxes via resizing + boxes = torch.as_tensor(boxes, dtype=torch.float32).reshape(-1, 4) + boxes[:, 2:] += boxes[:, :2] + boxes[:, 0::2].clamp_(min=0, max=w) + boxes[:, 1::2].clamp_(min=0, max=h) + + classes = [obj["category_id"] for obj in anno] + classes = torch.tensor(classes, dtype=torch.int64) + + segmentations = [obj["segmentation"] for obj in anno] + masks = convert_coco_poly_to_mask(segmentations, h, w) + + keypoints = None + if anno and "keypoints" in anno[0]: + keypoints = [obj["keypoints"] for obj in anno] + keypoints = torch.as_tensor(keypoints, dtype=torch.float32) + num_keypoints = keypoints.shape[0] + if num_keypoints: + keypoints = keypoints.view(num_keypoints, -1, 3) + + keep = (boxes[:, 3] > boxes[:, 1]) & (boxes[:, 2] > boxes[:, 0]) + boxes = boxes[keep] + classes = classes[keep] + masks = masks[keep] + if keypoints is not None: + keypoints = keypoints[keep] + + target = {} + target["boxes"] = boxes + target["labels"] = classes + target["masks"] = masks + target["image_id"] = image_id + if keypoints is not None: + target["keypoints"] = keypoints + + # for conversion to coco api + area = torch.tensor([obj["area"] for obj in anno]) + iscrowd = torch.tensor([obj["iscrowd"] for obj in anno]) + target["area"] = area + target["iscrowd"] = iscrowd + + return image, target + + +def _coco_remove_images_without_annotations(dataset, cat_list=None): + def _has_only_empty_bbox(anno): + return all(any(o <= 1 for o in obj["bbox"][2:]) for obj in anno) + + def _count_visible_keypoints(anno): + return sum(sum(1 for v in ann["keypoints"][2::3] if v > 0) for ann in anno) + + min_keypoints_per_image = 10 + + def _has_valid_annotation(anno): + # if it's empty, there is no annotation + if len(anno) == 0: + return False + # if all boxes have close to zero area, there is no annotation + if _has_only_empty_bbox(anno): + return False + # keypoints task have a slight different criteria for considering + # if an annotation is valid + if "keypoints" not in anno[0]: + return True + # for keypoint detection tasks, only consider valid images those + # containing at least min_keypoints_per_image + if _count_visible_keypoints(anno) >= min_keypoints_per_image: + return True + return False + + ids = [] + for ds_idx, img_id in enumerate(dataset.ids): + ann_ids = dataset.coco.getAnnIds(imgIds=img_id, iscrowd=None) + anno = dataset.coco.loadAnns(ann_ids) + if cat_list: + anno = [obj for obj in anno if obj["category_id"] in cat_list] + if _has_valid_annotation(anno): + ids.append(ds_idx) + + dataset = torch.utils.data.Subset(dataset, ids) + return dataset + + +def convert_to_coco_api(ds): + coco_ds = COCO() + # annotation IDs need to start at 1, not 0, see torchvision issue #1530 + ann_id = 1 + dataset = {"images": [], "categories": [], "annotations": []} + categories = set() + for img_idx in range(len(ds)): + # find better way to get target + # targets = ds.get_annotations(img_idx) + img, targets = ds[img_idx] + image_id = targets["image_id"] + img_dict = {} + img_dict["id"] = image_id + img_dict["height"] = img.shape[-2] + img_dict["width"] = img.shape[-1] + dataset["images"].append(img_dict) + bboxes = targets["boxes"].clone() + bboxes[:, 2:] -= bboxes[:, :2] + bboxes = bboxes.tolist() + labels = targets["labels"].tolist() + areas = targets["area"].tolist() + iscrowd = targets["iscrowd"].tolist() + if "masks" in targets: + masks = targets["masks"] + # make masks Fortran contiguous for coco_mask + masks = masks.permute(0, 2, 1).contiguous().permute(0, 2, 1) + if "keypoints" in targets: + keypoints = targets["keypoints"] + keypoints = keypoints.reshape(keypoints.shape[0], -1).tolist() + num_objs = len(bboxes) + for i in range(num_objs): + ann = {} + ann["image_id"] = image_id + ann["bbox"] = bboxes[i] + ann["category_id"] = labels[i] + categories.add(labels[i]) + ann["area"] = areas[i] + ann["iscrowd"] = iscrowd[i] + ann["id"] = ann_id + if "masks" in targets: + ann["segmentation"] = coco_mask.encode(masks[i].numpy()) + if "keypoints" in targets: + ann["keypoints"] = keypoints[i] + ann["num_keypoints"] = sum(k != 0 for k in keypoints[i][2::3]) + dataset["annotations"].append(ann) + ann_id += 1 + dataset["categories"] = [{"id": i} for i in sorted(categories)] + coco_ds.dataset = dataset + coco_ds.createIndex() + return coco_ds + + +def get_coco_api_from_dataset(dataset): + # FIXME: This is... awful? + for _ in range(10): + if isinstance(dataset, torch_CocoDetection): + break + if isinstance(dataset, torch.utils.data.Subset): + dataset = dataset.dataset + if isinstance(dataset, torch_CocoDetection): + return dataset.coco + return convert_to_coco_api(dataset) + + +class CocoDetection(torch_CocoDetection): + def __init__(self, img_folder, ann_file, transforms): + super().__init__(img_folder, ann_file) + self._transforms = transforms + + def __getitem__(self, idx): + img, target = super().__getitem__(idx) + image_id = self.ids[idx] + target = dict(image_id=image_id, annotations=target) + if self._transforms is not None: + img, target = self._transforms(img, target) + return img, target + + +def get_coco(root, image_set, transforms, mode="instances", use_v2=False, with_masks=False): + anno_file_template = "{}_{}2017.json" + PATHS = { + "train": ("train2017", os.path.join("annotations", anno_file_template.format(mode, "train"))), + "val": ("val2017", os.path.join("annotations", anno_file_template.format(mode, "val"))), + # "train": ("val2017", os.path.join("annotations", anno_file_template.format(mode, "val"))) + } + + img_folder, ann_file = PATHS[image_set] + img_folder = os.path.join(root, img_folder) + ann_file = os.path.join(root, ann_file) + + if use_v2: + from torchvision.datasets import wrap_dataset_for_transforms_v2 + + dataset = torch_CocoDetection(img_folder, ann_file, transforms=transforms) + target_keys = ["boxes", "labels", "image_id"] + if with_masks: + target_keys += ["masks"] + dataset = wrap_dataset_for_transforms_v2(dataset, target_keys=target_keys) + else: + # TODO: handle with_masks for V1? + t = [ConvertCocoPolysToMask()] + if transforms is not None: + t.append(transforms) + transforms = T.Compose(t) + + dataset = CocoDetection(img_folder, ann_file, transforms=transforms) + + if image_set == "train": + dataset = _coco_remove_images_without_annotations(dataset) + + # dataset = torch.utils.data.Subset(dataset, [i for i in range(500)]) + + return dataset diff --git a/tv-detection/SS/COCO_edits/group_by_aspect_ratio_SC.py b/tv-detection/SS/COCO_edits/group_by_aspect_ratio_SC.py new file mode 100644 index 00000000..2fe5bfd0 --- /dev/null +++ b/tv-detection/SS/COCO_edits/group_by_aspect_ratio_SC.py @@ -0,0 +1,198 @@ +import bisect +import copy +import math +from collections import defaultdict +from itertools import chain, repeat + +import numpy as np +import torch +import torch.utils.data +import torchvision +from PIL import Image +from torch.utils.data.sampler import BatchSampler, Sampler +from torch.utils.model_zoo import tqdm + +from COCO_interface import torch_CocoDetection + + +def _repeat_to_at_least(iterable, n): + repeat_times = math.ceil(n / len(iterable)) + repeated = chain.from_iterable(repeat(iterable, repeat_times)) + return list(repeated) + + +class GroupedBatchSampler(BatchSampler): + """ + Wraps another sampler to yield a mini-batch of indices. + It enforces that the batch only contain elements from the same group. + It also tries to provide mini-batches which follows an ordering which is + as close as possible to the ordering from the original sampler. + Args: + sampler (Sampler): Base sampler. + group_ids (list[int]): If the sampler produces indices in range [0, N), + `group_ids` must be a list of `N` ints which contains the group id of each sample. + The group ids must be a continuous set of integers starting from + 0, i.e. they must be in the range [0, num_groups). + batch_size (int): Size of mini-batch. + """ + + def __init__(self, sampler, group_ids, batch_size): + if not isinstance(sampler, Sampler): + raise ValueError(f"sampler should be an instance of torch.utils.data.Sampler, but got sampler={sampler}") + self.sampler = sampler + self.group_ids = group_ids + self.batch_size = batch_size + + def __iter__(self): + buffer_per_group = defaultdict(list) + samples_per_group = defaultdict(list) + + num_batches = 0 + for idx in self.sampler: + group_id = self.group_ids[idx] + buffer_per_group[group_id].append(idx) + samples_per_group[group_id].append(idx) + if len(buffer_per_group[group_id]) == self.batch_size: + yield buffer_per_group[group_id] + num_batches += 1 + del buffer_per_group[group_id] + assert len(buffer_per_group[group_id]) < self.batch_size + + # now we have run out of elements that satisfy + # the group criteria, let's return the remaining + # elements so that the size of the sampler is + # deterministic + expected_num_batches = len(self) + num_remaining = expected_num_batches - num_batches + if num_remaining > 0: + # for the remaining batches, take first the buffers with the largest number + # of elements + for group_id, _ in sorted(buffer_per_group.items(), key=lambda x: len(x[1]), reverse=True): + remaining = self.batch_size - len(buffer_per_group[group_id]) + samples_from_group_id = _repeat_to_at_least(samples_per_group[group_id], remaining) + buffer_per_group[group_id].extend(samples_from_group_id[:remaining]) + assert len(buffer_per_group[group_id]) == self.batch_size + yield buffer_per_group[group_id] + num_remaining -= 1 + if num_remaining == 0: + break + assert num_remaining == 0 + + def __len__(self): + return len(self.sampler) // self.batch_size + + +def _compute_aspect_ratios_slow(dataset, indices=None): + print( + "Your dataset doesn't support the fast path for " + "computing the aspect ratios, so will iterate over " + "the full dataset and load every image instead. " + "This might take some time..." + ) + if indices is None: + indices = range(len(dataset)) + + class SubsetSampler(Sampler): + def __init__(self, indices): + self.indices = indices + + def __iter__(self): + return iter(self.indices) + + def __len__(self): + return len(self.indices) + + sampler = SubsetSampler(indices) + data_loader = torch.utils.data.DataLoader( + dataset, + batch_size=1, + sampler=sampler, + num_workers=14, # you might want to increase it for faster processing + collate_fn=lambda x: x[0], + ) + aspect_ratios = [] + with tqdm(total=len(dataset)) as pbar: + for _i, (img, _) in enumerate(data_loader): + pbar.update(1) + height, width = img.shape[-2:] + aspect_ratio = float(width) / float(height) + aspect_ratios.append(aspect_ratio) + return aspect_ratios + + +def _compute_aspect_ratios_custom_dataset(dataset, indices=None): + if indices is None: + indices = range(len(dataset)) + aspect_ratios = [] + for i in indices: + height, width = dataset.get_height_and_width(i) + aspect_ratio = float(width) / float(height) + aspect_ratios.append(aspect_ratio) + return aspect_ratios + + +def _compute_aspect_ratios_coco_dataset(dataset, indices=None): + if indices is None: + indices = range(len(dataset)) + aspect_ratios = [] + for i in indices: + img_info = dataset.coco.imgs[dataset.ids[i]] + aspect_ratio = float(img_info["width"]) / float(img_info["height"]) + aspect_ratios.append(aspect_ratio) + return aspect_ratios + + +def _compute_aspect_ratios_voc_dataset(dataset, indices=None): + if indices is None: + indices = range(len(dataset)) + aspect_ratios = [] + for i in indices: + # this doesn't load the data into memory, because PIL loads it lazily + width, height = Image.open(dataset.images[i]).size + aspect_ratio = float(width) / float(height) + aspect_ratios.append(aspect_ratio) + return aspect_ratios + + +def _compute_aspect_ratios_subset_dataset(dataset, indices=None): + if indices is None: + indices = range(len(dataset)) + + ds_indices = [dataset.indices[i] for i in indices] + return compute_aspect_ratios(dataset.dataset, ds_indices) + + +def compute_aspect_ratios(dataset, indices=None): + if hasattr(dataset, "get_height_and_width"): + return _compute_aspect_ratios_custom_dataset(dataset, indices) + + if isinstance(dataset, torch_CocoDetection): + return _compute_aspect_ratios_coco_dataset(dataset, indices) + + if isinstance(dataset, torchvision.datasets.VOCDetection): + return _compute_aspect_ratios_voc_dataset(dataset, indices) + + if isinstance(dataset, torch.utils.data.Subset): + return _compute_aspect_ratios_subset_dataset(dataset, indices) + + # slow path + return _compute_aspect_ratios_slow(dataset, indices) + + +def _quantize(x, bins): + bins = copy.deepcopy(bins) + bins = sorted(bins) + quantized = list(map(lambda y: bisect.bisect_right(bins, y), x)) + return quantized + + +def create_aspect_ratio_groups(dataset, k=0): + aspect_ratios = compute_aspect_ratios(dataset) # list of aspect ratios for each image in the dataset + bins = (2 ** np.linspace(-1, 1, 2 * k + 1)).tolist() if k > 0 else [1.0] + groups = _quantize(aspect_ratios, bins) # list of bin indexes to which each image belongs + # count number of elements per group + counts = np.unique(groups, return_counts=True)[1] + fbins = [0] + bins + [np.inf] + print(f"Using {fbins} as bins for aspect ratio quantization") + print(f"Count of instances per bin: {counts}") + return groups diff --git a/tv-detection/maskrcnn_resnet50_fpn_small.isc b/tv-detection/maskrcnn_resnet50_fpn_small.isc new file mode 100644 index 00000000..32d8d5d2 --- /dev/null +++ b/tv-detection/maskrcnn_resnet50_fpn_small.isc @@ -0,0 +1,6 @@ +experiment_name="maskrcnn_resnet50_fpn_small" +gpu_type="24GB VRAM GPU" +nnodes = 11 +venv_path = "~/.venv/bin/activate" +output_path = "~/output_maskrcnn_resnet50_fpn" +command="train_cycling_small.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/Datasets/Open-Datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/tv-detection/train_cycling_small.py b/tv-detection/train_cycling_small.py new file mode 100644 index 00000000..10c7622d --- /dev/null +++ b/tv-detection/train_cycling_small.py @@ -0,0 +1,335 @@ +r"""PyTorch Detection Training. + +To run in a multi-gpu environment, use the distributed launcher:: + + python -m torch.distributed.launch --nproc_per_node=$NGPU --use_env \ + train.py ... --world-size $NGPU + +The default hyperparameters are tuned for training on 8 gpus and 2 images per gpu. + --lr 0.02 --batch-size 2 --world-size 8 +If you use different number of gpus, the learning rate should be changed to 0.02/8*$NGPU. + +On top of that, for training Faster/Mask R-CNN, the default hyperparameters are + --epochs 26 --lr-steps 16 22 --aspect-ratio-group-factor 3 + +Also, if you train Keypoint R-CNN, the default hyperparameters are + --epochs 46 --lr-steps 36 43 --aspect-ratio-group-factor 3 +Because the number of images is smaller in the person keypoint subset of COCO, +the number of epochs should be adapted so that we have the same number of iterations. +""" +from cycling_utils import Timer + +timer = Timer() +timer.report('importing Timer') + +import datetime +import os +import time +# import warnings + +from pathlib import Path +import presets +import torch +import torch.utils.data +import torchvision +import utils +from coco_utils import get_coco + +import torchvision.models.detection +import torchvision.models.detection.mask_rcnn +from engine import evaluate, train_one_epoch +from group_by_aspect_ratio_SC import create_aspect_ratio_groups, GroupedBatchSampler +from torchvision.transforms import InterpolationMode +from transforms import SimpleCopyPaste + +from cycling_utils import InterruptableDistributedSampler, InterruptableDistributedGroupedBatchSampler, atomic_torch_save + +timer.report('importing everything else') + +def copypaste_collate_fn(batch): + copypaste = SimpleCopyPaste(blending=True, resize_interpolation=InterpolationMode.BILINEAR) + return copypaste(*utils.collate_fn(batch)) + +def get_dataset(is_train, args): + image_set = "train" if is_train else "val" + num_classes, mode = {"coco": (91, "instances"), "coco_kp": (2, "person_keypoints")}[args.dataset] + with_masks = "mask" in args.model + ds = get_coco( + root=args.data_path, + image_set=image_set, + transforms=get_transform(is_train, args), + mode=mode, + use_v2=args.use_v2, + with_masks=with_masks, + ) + return ds, num_classes + +def get_transform(is_train, args): + if is_train: + return presets.DetectionPresetTrain( + data_augmentation=args.data_augmentation, backend=args.backend, use_v2=args.use_v2 + ) + elif args.weights and args.test_only: + weights = torchvision.models.get_weight(args.weights) + trans = weights.transforms() + return lambda img, target: (trans(img), target) + else: + return presets.DetectionPresetEval(backend=args.backend, use_v2=args.use_v2) + +def _get_iou_types(model): # intersection over union (iou) types + model_without_ddp = model + if isinstance(model, torch.nn.parallel.DistributedDataParallel): + model_without_ddp = model.module + iou_types = ["bbox"] + if isinstance(model_without_ddp, torchvision.models.detection.MaskRCNN): + iou_types.append("segm") + if isinstance(model_without_ddp, torchvision.models.detection.KeypointRCNN): + iou_types.append("keypoints") + return iou_types + +timer.report('defined other functions') + +def main(args, timer): + + if args.backend.lower() == "tv_tensor" and not args.use_v2: + raise ValueError("Use --use-v2 if you want to use the tv_tensor backend.") + if args.dataset not in ("coco", "coco_kp"): + raise ValueError(f"Dataset should be coco or coco_kp, got {args.dataset}") + if "keypoint" in args.model and args.dataset != "coco_kp": + raise ValueError("Oops, if you want Keypoint detection, set --dataset coco_kp") + if args.dataset == "coco_kp" and args.use_v2: + raise ValueError("KeyPoint detection doesn't support V2 transforms yet") + + if args.output_dir: + utils.mkdir(args.output_dir) + + utils.init_distributed_mode(args) + print(args) + + device = torch.device(args.device) + + if args.use_deterministic_algorithms: + torch.use_deterministic_algorithms(True) + + timer.report('main preliminaries') + + # Data loading code + dataset_train, num_classes = get_dataset(is_train=True, args=args) + dataset_test, _ = get_dataset(is_train=False, args=args) + + ## SUBSET FOR TESTING + dataset_train = torch.utils.data.Subset(dataset_train, torch.arange(2*11*5)) # batch_size x nodes x iterations + dataset_test = torch.utils.data.Subset(dataset_test, torch.arange(1*11*2)) + + timer.report('loading data') + + # if args.distributed: + # train_sampler = torch.utils.data.distributed.DistributedSampler(dataset) + # test_sampler = torch.utils.data.distributed.DistributedSampler(dataset_test, shuffle=False) + # else: + # train_sampler = torch.utils.data.RandomSampler(dataset) + # test_sampler = torch.utils.data.SequentialSampler(dataset_test) + + # if args.aspect_ratio_group_factor >= 0: # default == 3 + # group_ids = create_aspect_ratio_groups(dataset_train, k=args.aspect_ratio_group_factor) + # train_batch_sampler = GroupedBatchSampler(train_sampler, group_ids, args.batch_size) + # else: + # train_batch_sampler = torch.utils.data.BatchSampler(train_sampler, args.batch_size, drop_last=True) + + group_ids = create_aspect_ratio_groups(dataset_train, k=args.aspect_ratio_group_factor) + train_sampler = InterruptableDistributedGroupedBatchSampler(dataset_train, group_ids, args.batch_size) + test_sampler = InterruptableDistributedSampler(dataset_test) + + timer.report('creating data samplers') + + train_collate_fn = utils.collate_fn + if args.use_copypaste: + if args.data_augmentation != "lsj": + raise RuntimeError("SimpleCopyPaste algorithm currently only supports the 'lsj' data augmentation policies") + print("Using copypaste_collate_fn for train_collate_fn") + train_collate_fn = copypaste_collate_fn + + data_loader_train = torch.utils.data.DataLoader( + dataset_train, batch_sampler=train_sampler, num_workers=args.workers, collate_fn=train_collate_fn + ) + data_loader_test = torch.utils.data.DataLoader( + dataset_test, batch_size=1, sampler=test_sampler, num_workers=args.workers, collate_fn=utils.collate_fn + ) + + timer.report('creating data loaders') + + kwargs = {"trainable_backbone_layers": args.trainable_backbone_layers} + if args.data_augmentation in ["multiscale", "lsj"]: + kwargs["_skip_resize"] = True + if "rcnn" in args.model: + if args.rpn_score_thresh is not None: + kwargs["rpn_score_thresh"] = args.rpn_score_thresh + + model = torchvision.models.get_model( + args.model, weights=args.weights, weights_backbone=args.weights_backbone, num_classes=num_classes, **kwargs + ) + model.to(device) + + timer.report('creating model and .to(device)') + + if args.distributed and args.sync_bn: + model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model) + + model_without_ddp = model + if args.distributed: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) + model_without_ddp = model.module + + timer.report('preparing model for distributed training') + + if args.norm_weight_decay is None: + parameters = [p for p in model.parameters() if p.requires_grad] + else: + param_groups = torchvision.ops._utils.split_normalization_params(model) + wd_groups = [args.norm_weight_decay, args.weight_decay] + parameters = [{"params": p, "weight_decay": w} for p, w in zip(param_groups, wd_groups) if p] + + opt_name = args.opt.lower() + if opt_name.startswith("sgd"): + optimizer = torch.optim.SGD( + parameters, + lr=args.lr, + momentum=args.momentum, + weight_decay=args.weight_decay, + nesterov="nesterov" in opt_name, + ) + elif opt_name == "adamw": + optimizer = torch.optim.AdamW(parameters, lr=args.lr, weight_decay=args.weight_decay) + else: + raise RuntimeError(f"Invalid optimizer {args.opt}. Only SGD and AdamW are supported.") + + scaler = torch.cuda.amp.GradScaler() if args.amp else None + + timer.report('optimizer and scaler') + + ## OUTER LR_SCHEDULER + args.lr_scheduler = args.lr_scheduler.lower() + if args.lr_scheduler == "multisteplr": + lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=args.lr_steps, gamma=args.lr_gamma) + elif args.lr_scheduler == "cosineannealinglr": + lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=args.epochs) + else: + raise RuntimeError( + f"Invalid lr scheduler '{args.lr_scheduler}'. Only MultiStepLR and CosineAnnealingLR are supported." + ) + + ## WARMUP LR_SCHEDULER + warmup_factor = 1.0 / 1000 + warmup_iters = min(1000, len(data_loader_train) - 1) + warmup_lr_scheduler = torch.optim.lr_scheduler.LinearLR( + optimizer, start_factor=warmup_factor, total_iters=warmup_iters + ) + + timer.report('learning rate schedulers') + + from coco_eval import CocoEvaluator + from coco_utils import get_coco_api_from_dataset + coco = get_coco_api_from_dataset(data_loader_test.dataset) + iou_types = _get_iou_types(model) + coco_evaluator = CocoEvaluator(coco, iou_types) + + timer.report('init coco evaluator') + + Path(args.resume).parent.mkdir(parents=True, exist_ok=True) + if args.resume and os.path.isfile(args.resume): + + checkpoint = torch.load(args.resume, map_location="cpu") + model_without_ddp.load_state_dict(checkpoint["model"]) + args.start_epoch = checkpoint["epoch"] + + optimizer.load_state_dict(checkpoint["optimizer"]) + lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) + warmup_lr_scheduler.load_state_dict(checkpoint["warmup_lr_scheduler"]) + train_sampler.load_state_dict(checkpoint["train_sampler"]) + if args.amp: + scaler.load_state_dict(checkpoint["scaler"]) + + test_sampler.load_state_dict(checkpoint["test_sampler"]) + + # Evaluator state variables + coco_evaluator.img_ids = checkpoint["img_ids"] + coco_evaluator.eval_imgs = checkpoint["eval_imgs"] + + timer.report('retrieving checkpoint') + + if args.test_only: + # We disable the cudnn benchmarking because it can noticeably affect the accuracy + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + coco_evaluator, timer = evaluate(model, data_loader_test, device, timer) + return + + for epoch in range(args.start_epoch, args.epochs): + + print('\n') + print(f"EPOCH :: {epoch}") + print('\n') + + with train_sampler.in_epoch(epoch): + timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch + metric_logger, timer = train_one_epoch(model, optimizer, data_loader_train, train_sampler, test_sampler, lr_scheduler, warmup_lr_scheduler, args, device, coco_evaluator, epoch, scaler, timer) + + # NEST THE TEST SAMPLER IN TRAIN SAMPLER CONTEXT TO AVOID EPOCH RESTART? + with test_sampler.in_epoch(epoch): + timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch + coco_evaluator, timer = evaluate(model, data_loader_test, epoch, test_sampler, args, coco_evaluator, optimizer, lr_scheduler, warmup_lr_scheduler, train_sampler, device, scaler, timer) + +def get_args_parser(add_help=True): + import argparse + + parser = argparse.ArgumentParser(description="PyTorch Detection Training", add_help=add_help) + + parser.add_argument("--data-path", default="/datasets01/COCO/022719/", type=str, help="dataset path") + parser.add_argument("--dataset",default="coco",type=str,help="dataset name. Use coco for object detection and instance segmentation and coco_kp for Keypoint detection",) + parser.add_argument("--model", default="maskrcnn_resnet50_fpn", type=str, help="model name") + parser.add_argument("--device", default="cuda", type=str, help="device (Use cuda or cpu Default: cuda)") + parser.add_argument("-b", "--batch-size", default=2, type=int, help="images per gpu, the total batch size is $NGPU x batch_size") + parser.add_argument("--epochs", default=26, type=int, metavar="N", help="number of total epochs to run") + parser.add_argument("-j", "--workers", default=4, type=int, metavar="N", help="number of data loading workers (default: 4)") + parser.add_argument("--opt", default="sgd", type=str, help="optimizer") + parser.add_argument("--lr",default=0.02,type=float,help="initial learning rate, 0.02 is the default value for training on 8 gpus and 2 images_per_gpu") + parser.add_argument("--momentum", default=0.9, type=float, metavar="M", help="momentum") + parser.add_argument("--wd","--weight-decay",default=1e-4,type=float,metavar="W",help="weight decay (default: 1e-4)",dest="weight_decay",) + parser.add_argument("--norm-weight-decay",default=None,type=float,help="weight decay for Normalization layers (default: None, same value as --wd)") + parser.add_argument("--lr-scheduler", default="multisteplr", type=str, help="name of lr scheduler (default: multisteplr)") + parser.add_argument("--lr-step-size", default=8, type=int, help="decrease lr every step-size epochs (multisteplr scheduler only)") + parser.add_argument("--lr-steps",default=[16, 22],nargs="+",type=int,help="decrease lr every step-size epochs (multisteplr scheduler only)") + parser.add_argument("--lr-gamma", default=0.1, type=float, help="decrease lr by a factor of lr-gamma (multisteplr scheduler only)") + parser.add_argument("--print-freq", default=1, type=int, help="print frequency") + parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") + parser.add_argument("--resume", default="", type=str, help="path of checkpoint") + parser.add_argument("--start_epoch", default=0, type=int, help="start epoch") + parser.add_argument("--aspect-ratio-group-factor", default=3, type=int) + parser.add_argument("--rpn-score-thresh", default=None, type=float, help="rpn score threshold for faster-rcnn") + parser.add_argument("--trainable-backbone-layers", default=None, type=int, help="number of trainable layers of backbone") + parser.add_argument("--data-augmentation", default="hflip", type=str, help="data augmentation policy (default: hflip)") + parser.add_argument("--sync-bn",dest="sync_bn",help="Use sync batch norm",action="store_true") + parser.add_argument("--test-only",dest="test_only",help="Only test the model",action="store_true") + parser.add_argument("--use-deterministic-algorithms", action="store_true", help="Forces the use of deterministic algorithms only.") + + # distributed training parameters + parser.add_argument("--world-size", default=1, type=int, help="number of distributed processes") + parser.add_argument("--dist-url", default="env://", type=str, help="url used to set up distributed training") + parser.add_argument("--weights", default=None, type=str, help="the weights enum name to load") + parser.add_argument("--weights-backbone", default=None, type=str, help="the backbone weights enum name to load") + + # Mixed precision training parameters + parser.add_argument("--amp", action="store_true", help="Use torch.cuda.amp for mixed precision training") + + # Use CopyPaste augmentation training parameter + parser.add_argument("--use-copypaste",action="store_true",help="Use CopyPaste data augmentation. Works only with data-augmentation='lsj'.",) + + parser.add_argument("--backend", default="PIL", type=str.lower, help="PIL or tensor - case insensitive") + parser.add_argument("--use-v2", action="store_true", help="Use V2 transforms") + + return parser + + +if __name__ == "__main__": + args = get_args_parser().parse_args() + main(args, timer) From 1ec3b1c23ac7995fb2eb6356fa1f83ef303e4901 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Wed, 13 Sep 2023 15:08:33 +1000 Subject: [PATCH 19/44] updateds --- monai/brats_mri_2d/loops.py | 5 +- monai/brats_mri_2d/train_cycling_diff.py | 86 ++++++++++++------------ monai/brats_mri_2d/train_cycling_gen.py | 9 ++- 3 files changed, 51 insertions(+), 49 deletions(-) diff --git a/monai/brats_mri_2d/loops.py b/monai/brats_mri_2d/loops.py index a8c7d23f..ad12efc6 100644 --- a/monai/brats_mri_2d/loops.py +++ b/monai/brats_mri_2d/loops.py @@ -23,7 +23,7 @@ def train_generator_one_epoch( discriminator.train() train_step = train_sampler.progress // train_loader.batch_size - total_steps = int((len(train_loader.dataset) / 11) / train_loader.batch_size) + total_steps = int(len(train_sampler) / train_loader.batch_size) print(f'\nTraining / resuming epoch {epoch} from training step {train_step}\n') for step, batch in enumerate(train_loader): @@ -99,11 +99,12 @@ def train_generator_one_epoch( recons_loss = epoch_loss / train_images_seen gen_loss = gen_epoch_loss / train_images_seen disc_loss = disc_epoch_loss / train_images_seen - print("Epoch [{}] Step [{}/{}] :: recons_loss: {:,.3f}, gen_loss: {:,.3f}, disc_loss: {:,.3f}".format(epoch, train_step, total_steps, recons_loss, gen_loss, disc_loss)) + print("Epoch [{}] Step [{}/{}] :: recons_loss: {:,.3f}, gen_loss: {:,.3f}, disc_loss: {:,.3f}".format(epoch, train_step+1, total_steps, recons_loss, gen_loss, disc_loss)) ## Checkpointing print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") train_sampler.advance(len(images)) + train_step = train_sampler.progress // train_loader.batch_size if utils.is_main_process() and train_step % 1 == 0: # Checkpointing every batch checkpoint = { # Universals diff --git a/monai/brats_mri_2d/train_cycling_diff.py b/monai/brats_mri_2d/train_cycling_diff.py index a79370a2..1ccd5b9b 100644 --- a/monai/brats_mri_2d/train_cycling_diff.py +++ b/monai/brats_mri_2d/train_cycling_diff.py @@ -43,33 +43,17 @@ def get_args_parser(add_help=True): # parser.add_argument("--epochs", default=30, type=int, metavar="N", help="number of total epochs to run") # parser.add_argument("--print-freq", default=1, type=int, help="print frequency") parser.add_argument("--dist-url", default="env://", type=str, help="url used to set up distributed training") + parser.add_argument("-j", "--workers", default=16, type=int, metavar="N", help="number of data loading workers (default: 16)") return parser -channel = 0 # 0 = Flair -assert channel in [0, 1, 2, 3], "Choose a valid channel" -train_transforms = transforms.Compose([ - transforms.LoadImaged(keys=["image", "label"]), - transforms.EnsureChannelFirstd(keys=["image", "label"]), - transforms.Lambdad(keys=["image"], func=lambda x: x[channel, None, :, :, :]), - transforms.EnsureTyped(keys=["image", "label"]), - transforms.Orientationd(keys=["image", "label"], axcodes="RAS"), - transforms.Spacingd(keys=["image", "label"], pixdim=(3.0, 3.0, 2.0), mode=("bilinear", "nearest")), - transforms.CenterSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 44)), - transforms.ScaleIntensityRangePercentilesd(keys="image", lower=0, upper=99.5, b_min=0, b_max=1), - transforms.RandSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 1), random_size=False), - transforms.Lambdad(keys=["image", "label"], func=lambda x: x.squeeze(-1)), - transforms.CopyItemsd(keys=["label"], times=1, names=["slice_label"]), - transforms.Lambdad(keys=["slice_label"], func=lambda x: 1.0 if x.sum() > 0 else 0.0), -]) - -timer.report('importing everything else and prepping transforms') +timer.report('importing everything else') def main(args, timer): - ## Distributed training prelims - if args.output_dir: - utils.mkdir(args.output_dir) + # ## Distributed training prelims + # if args.output_dir: + # utils.mkdir(args.output_dir) utils.init_distributed_mode(args) # Sets args.distributed among other things assert args.distributed # don't support cycling when not distributed for simplicity @@ -81,15 +65,36 @@ def main(args, timer): # Maybe this will work? set_determinism(42) + channel = 0 # 0 = Flair + assert channel in [0, 1, 2, 3], "Choose a valid channel" + train_transforms = transforms.Compose([ + transforms.LoadImaged(keys=["image", "label"], image_only=False), # image_only current default will change soon, so including explicitly + transforms.EnsureChannelFirstd(keys=["image", "label"]), + transforms.Lambdad(keys=["image"], func=lambda x: x[channel, None, :, :, :]), + transforms.EnsureTyped(keys=["image", "label"]), + transforms.Orientationd(keys=["image", "label"], axcodes="RAS"), + transforms.Spacingd(keys=["image", "label"], pixdim=(3.0, 3.0, 2.0), mode=("bilinear", "nearest")), + transforms.CenterSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 44)), + transforms.ScaleIntensityRangePercentilesd(keys="image", lower=0, upper=99.5, b_min=0, b_max=1), + transforms.RandSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 1), random_size=False), + transforms.Lambdad(keys=["image", "label"], func=lambda x: x.squeeze(-1)), + transforms.CopyItemsd(keys=["label"], times=1, names=["slice_label"]), + transforms.Lambdad(keys=["slice_label"], func=lambda x: 1.0 if x.sum() > 0 else 0.0), + ]) + train_ds = DecathlonDataset( - root_dir=args.data_path, task="Task01_BrainTumour", section="training", cache_rate=1.0, + root_dir=args.data_path, task="Task01_BrainTumour", section="training", cache_rate=0.0, num_workers=4, download=False, seed=0, transform=train_transforms, ) val_ds = DecathlonDataset( - root_dir=args.data_path, task="Task01_BrainTumour", section="validation", cache_rate=1.0, + root_dir=args.data_path, task="Task01_BrainTumour", section="validation", cache_rate=0.0, num_workers=4, download=False, seed=0, transform=train_transforms, ) + # ## SUBSET FOR TESTING + # train_ds = torch.utils.data.Subset(train_ds, torch.arange(2*9*3)) # batch_size x nodes x iterations + # val_ds = torch.utils.data.Subset(val_ds, torch.arange(1*9*2)) # batch_size x nodes x iterations + timer.report('build datasets') train_sampler = InterruptableDistributedSampler(train_ds) @@ -97,9 +102,9 @@ def main(args, timer): timer.report('build samplers') - train_loader = DataLoader(train_ds, batch_size=64, sampler=train_sampler, shuffle=True, num_workers=4, persistent_workers=True) - val_loader = DataLoader(val_ds, batch_size=64, sampler=val_sampler, shuffle=True, num_workers=4, persistent_workers=True) - check_data = first(train_loader) # Used later + train_loader = DataLoader(train_ds, batch_size=2, sampler=train_sampler, num_workers=args.workers) + val_loader = DataLoader(val_ds, batch_size=1, sampler=val_sampler, num_workers=args.workers) + # check_data = first(train_loader) # Used later timer.report('build dataloaders') @@ -154,7 +159,7 @@ def main(args, timer): unet_without_ddp = unet.module # perceptual_loss_without_ddp = perceptual_loss.module - timer.report('models prepped for distribution') + timer.report('unet prepped for distribution') # Optimizers # optimizer_g = torch.optim.Adam(generator_without_ddp.parameters(), lr=1e-4) @@ -170,26 +175,23 @@ def main(args, timer): timer.report('grad scalers') + # Init tracking metrics + # ??? + # RETRIEVE CHECKPOINT Path(args.resume).parent.mkdir(parents=True, exist_ok=True) if args.resume and os.path.isfile(args.resume): # If we're resuming... - checkpoint = torch.load(args.resume, map_location="cpu") - # generator_without_ddp.load_state_dict(checkpoint["generator"], strict=not args.test_only) - # discriminator_without_ddp.load_state_dict(checkpoint["discriminator"], strict=not args.test_only) - unet_without_ddp.load_state_dict(checkpoint["unet"], strict=not args.test_only) - args.start_epoch = checkpoint["epoch"] - - # optimizer.load_state_dict(checkpoint["optimizer"]) - # lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) - # train_sampler.load_state_dict(checkpoint["train_sampler"]) - # if args.amp: # Could align this syntactically... - # scaler.load_state_dict(checkpoint["scaler"]) - - # test_sampler.load_state_dict(checkpoint["test_sampler"]) - # confmat.mat = checkpoint["confmat"] - # confmat.temp_mat = checkpoint["confmat_temp"] + unet_without_ddp.load_state_dict(checkpoint["unet"], strict=not args.test_only) + optimizer_u.load_state_dict(checkpoint["optimizer_u"]) + scaler_u.load_state_dict(checkpoint["scaler_u"]) + train_sampler.load_state_dict(checkpoint["train_sampler"]) + val_sampler.load_state_dict(checkpoint["val_sampler"]) + train_images_seen = checkpoint["train_images_seen"] + val_images_seen = checkpoint["val_images_seen"] + # Metrics + # ??? timer.report('checkpoint retrieval') diff --git a/monai/brats_mri_2d/train_cycling_gen.py b/monai/brats_mri_2d/train_cycling_gen.py index 5dc5fcd9..8f663949 100644 --- a/monai/brats_mri_2d/train_cycling_gen.py +++ b/monai/brats_mri_2d/train_cycling_gen.py @@ -68,7 +68,7 @@ def main(args, timer): channel = 0 # 0 = Flair assert channel in [0, 1, 2, 3], "Choose a valid channel" train_transforms = transforms.Compose([ - transforms.LoadImaged(keys=["image", "label"]), + transforms.LoadImaged(keys=["image", "label"], image_only=False), # image_only current default will change soon, so including explicitly transforms.EnsureChannelFirstd(keys=["image", "label"]), transforms.Lambdad(keys=["image"], func=lambda x: x[channel, None, :, :, :]), transforms.EnsureTyped(keys=["image", "label"]), @@ -150,7 +150,6 @@ def main(args, timer): # unet_without_ddp = unet # perceptual_loss_without_ddp = perceptual_loss if args.distributed: - print('Confirm distributed training.') generator = torch.nn.parallel.DistributedDataParallel(generator, device_ids=[args.gpu]) discriminator = torch.nn.parallel.DistributedDataParallel(discriminator, device_ids=[args.gpu]) # unet = torch.nn.parallel.DistributedDataParallel(unet, device_ids=[args.gpu]) @@ -185,7 +184,6 @@ def main(args, timer): val_loss = 0 # RETRIEVE CHECKPOINT - Path(args.resume).parent.mkdir(parents=True, exist_ok=True) if args.resume and os.path.isfile(args.resume): # If we're resuming... checkpoint = torch.load(args.resume, map_location="cpu") @@ -200,6 +198,7 @@ def main(args, timer): val_sampler.load_state_dict(checkpoint["val_sampler"]) train_images_seen = checkpoint["train_images_seen"] val_images_seen = checkpoint["val_images_seen"] + # Metrics epoch_loss = checkpoint["epoch_loss"] gen_epoch_loss = checkpoint["gen_epoch_loss"] disc_epoch_loss = checkpoint["disc_epoch_loss"] @@ -215,7 +214,7 @@ def main(args, timer): for epoch in range(args.start_epoch, n_gen_epochs): print('\n') - print(f"EPOCH (gen) :: {epoch}") + print(f"EPOCH :: {epoch}") print('\n') with train_sampler.in_epoch(epoch): @@ -256,7 +255,7 @@ def main(args, timer): # for epoch in range(n_diff_epochs): # print('\n') - # print(f"EPOCH (diff) :: {epoch}") + # print(f"EPOCH :: {epoch}") # print('\n') # with train_sampler.in_epoch(epoch): From bd04c293d6e54c17f5fc6d5c57047faba62e379f Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Wed, 13 Sep 2023 15:17:22 +1000 Subject: [PATCH 20/44] updates --- monai/brats_mri_2d/loops.py | 60 +++++++++++++++----------------- monai/brats_mri_2d/models_DIS.py | 23 ------------ 2 files changed, 29 insertions(+), 54 deletions(-) delete mode 100644 monai/brats_mri_2d/models_DIS.py diff --git a/monai/brats_mri_2d/loops.py b/monai/brats_mri_2d/loops.py index ad12efc6..fbcf548b 100644 --- a/monai/brats_mri_2d/loops.py +++ b/monai/brats_mri_2d/loops.py @@ -199,50 +199,48 @@ def evaluate_generator( +## -- DIFFUSION MODEL - ## +def train_diffusion_one_epoch( + epoch, unet, generator, optimizer, inferer, scaler, train_loader, device + ): + unet.train() + generator.eval() -# ## -- DIFFUSION MODEL - ## - -# def train_diffusion_one_epoch(epoch, unet, generator, optimizer, inferer, scaler, train_loader, device): - -# unet.train() -# generator.eval() - -# epoch_losses = [] - -# epoch_loss = 0 -# train_images_seen = 0 + epoch_loss = 0 + train_images_seen = 0 -# progress_bar = tqdm(enumerate(train_loader), total=len(train_loader), ncols=70) -# progress_bar.set_description(f"Epoch (train diff) {epoch}") + train_step = train_sampler.progress // train_loader.batch_size + total_steps = int(len(train_sampler) / train_loader.batch_size) + print(f'\nTraining / resuming epoch {epoch} from training step {train_step}\n') -# for step, batch in progress_bar: + for step, batch in progress_bar: -# images = batch["image"].to(device) -# optimizer.zero_grad(set_to_none=True) + images = batch["image"].to(device) + optimizer.zero_grad(set_to_none=True) -# with autocast(enabled=True): + with autocast(enabled=True): -# z_mu, z_sigma = generator.encode(images) -# z = generator.sampling(z_mu, z_sigma) -# noise = torch.randn_like(z).to(device) -# timesteps = torch.randint(0, inferer.scheduler.num_train_timesteps, (z.shape[0],), device=z.device).long() -# noise_pred = inferer(inputs=images, diffusion_model=unet, noise=noise, timesteps=timesteps, autoencoder_model=generator) -# loss = F.mse_loss(noise_pred.float(), noise.float()) + z_mu, z_sigma = generator.encode(images) + z = generator.sampling(z_mu, z_sigma) + noise = torch.randn_like(z).to(device) + timesteps = torch.randint(0, inferer.scheduler.num_train_timesteps, (z.shape[0],), device=z.device).long() + noise_pred = inferer(inputs=images, diffusion_model=unet, noise=noise, timesteps=timesteps, autoencoder_model=generator) + loss = F.mse_loss(noise_pred.float(), noise.float()) -# scaler.scale(loss).backward() -# scaler.step(optimizer) -# scaler.update() + scaler.scale(loss).backward() + scaler.step(optimizer) + scaler.update() -# epoch_loss += loss.item() -# train_images_seen += len(images) + epoch_loss += loss.item() + train_images_seen += len(images) -# progress_bar.set_postfix({"loss": epoch_loss / train_images_seen}) + progress_bar.set_postfix({"loss": epoch_loss / train_images_seen}) -# epoch_losses.append(epoch_loss / train_images_seen) + epoch_losses.append(epoch_loss / train_images_seen) -# return unet, epoch_losses + return unet, epoch_losses # def evaluate_diffusion(epoch, unet, generator, inferer, val_loader, device): diff --git a/monai/brats_mri_2d/models_DIS.py b/monai/brats_mri_2d/models_DIS.py deleted file mode 100644 index c6f23c2d..00000000 --- a/monai/brats_mri_2d/models_DIS.py +++ /dev/null @@ -1,23 +0,0 @@ -# import os -# import pandas as pd -# from torchvision.io import read_image - -# class CustomImageDataset(Dataset): -# def __init__(self, annotations_file, img_dir, transform=None, target_transform=None): -# self.img_labels = pd.read_csv(annotations_file) -# self.img_dir = img_dir -# self.transform = transform -# self.target_transform = target_transform - -# def __len__(self): -# return len(self.img_labels) - -# def __getitem__(self, idx): -# img_path = os.path.join(self.img_dir, self.img_labels.iloc[idx, 0]) -# image = read_image(img_path) -# label = self.img_labels.iloc[idx, 1] -# if self.transform: -# image = self.transform(image) -# if self.target_transform: -# label = self.target_transform(label) -# return image, label \ No newline at end of file From fd82382e7d327d56422b7445581253fa0199146c Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Wed, 13 Sep 2023 16:08:31 +1000 Subject: [PATCH 21/44] updates --- monai/brats_mri_2d/loops.py | 154 ++++++++++++++++------- monai/brats_mri_2d/train_cycling_diff.py | 63 ++++++---- monai/brats_mri_2d/train_cycling_gen.py | 8 +- 3 files changed, 150 insertions(+), 75 deletions(-) diff --git a/monai/brats_mri_2d/loops.py b/monai/brats_mri_2d/loops.py index fbcf548b..7384f8cd 100644 --- a/monai/brats_mri_2d/loops.py +++ b/monai/brats_mri_2d/loops.py @@ -195,87 +195,145 @@ def evaluate_generator( val_loss /= val_images_seen print(f"Epoch {epoch} val loss: {val_loss:.4f}") - return timer # ?? + return timer ## -- DIFFUSION MODEL - ## def train_diffusion_one_epoch( - epoch, unet, generator, optimizer, inferer, scaler, train_loader, device + args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, + train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer ): unet.train() generator.eval() - epoch_loss = 0 - train_images_seen = 0 - train_step = train_sampler.progress // train_loader.batch_size total_steps = int(len(train_sampler) / train_loader.batch_size) print(f'\nTraining / resuming epoch {epoch} from training step {train_step}\n') - for step, batch in progress_bar: + for step, batch in enumerate(train_loader): images = batch["image"].to(device) - optimizer.zero_grad(set_to_none=True) + timer.report(f'train batch {train_step} to device') + + optimizer_u.zero_grad(set_to_none=True) with autocast(enabled=True): z_mu, z_sigma = generator.encode(images) + timer.report(f'train batch {train_step} generator encoded') z = generator.sampling(z_mu, z_sigma) + timer.report(f'train batch {train_step} generator sampling') noise = torch.randn_like(z).to(device) + timer.report(f'train batch {train_step} noise') timesteps = torch.randint(0, inferer.scheduler.num_train_timesteps, (z.shape[0],), device=z.device).long() + timer.report(f'train batch {train_step} timesteps') noise_pred = inferer(inputs=images, diffusion_model=unet, noise=noise, timesteps=timesteps, autoencoder_model=generator) + timer.report(f'train batch {train_step} noise_pred') loss = F.mse_loss(noise_pred.float(), noise.float()) + timer.report(f'train batch {train_step} loss') - scaler.scale(loss).backward() - scaler.step(optimizer) - scaler.update() + scaler_u.scale(loss).backward() + scaler_u.step(optimizer_u) + scaler_u.update() + timer.report(f'train batch {train_step} unet backward') epoch_loss += loss.item() train_images_seen += len(images) + recons_loss = epoch_loss / train_images_seen + print("Epoch [{}] Step [{}/{}] :: recons_loss: {:,.3f}".format(epoch, train_step+1, total_steps, recons_loss)) - progress_bar.set_postfix({"loss": epoch_loss / train_images_seen}) - - epoch_losses.append(epoch_loss / train_images_seen) - - return unet, epoch_losses - - -# def evaluate_diffusion(epoch, unet, generator, inferer, val_loader, device): - -# unet.eval() - -# val_losses = [] - -# val_loss = 0 -# val_images_seen = 0 - -# progress_bar = tqdm(enumerate(val_loader), total=len(val_loader), ncols=110) -# progress_bar.set_description(f"Epoch (eval diff) {epoch}") - -# with torch.no_grad(): -# for val_step, batch in enumerate(val_loader, start=1): -# images = batch["image"].to(device) - -# with autocast(enabled=True): -# z_mu, z_sigma = generator.encode(images) -# z = generator.sampling(z_mu, z_sigma) - -# noise = torch.randn_like(z).to(device) -# timesteps = torch.randint(0, inferer.scheduler.num_train_timesteps, (z.shape[0],), device=z.device).long() -# noise_pred = inferer(inputs=images,diffusion_model=unet,noise=noise,timesteps=timesteps,autoencoder_model=generator) + ## Checkpointing + print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") + train_sampler.advance(len(images)) + train_step = train_sampler.progress // train_loader.batch_size + if utils.is_main_process() and train_step % 1 == 0: # Checkpointing every batch + checkpoint = { + # Universals + "args": args, + "epoch": epoch, + + # State variables + "unet": unet.module.state_dict(), + "optimizer_u": optimizer_u.state_dict(), + "scaler_u": scaler_u.state_dict(), + "train_sampler": train_sampler.state_dict(), + "val_sampler": val_sampler.state_dict(), -# loss = F.mse_loss(noise_pred.float(), noise.float()) + # Evaluation metrics + "train_images_seen": train_images_seen, + "val_images_seen": val_images_seen, + "epoch_loss": epoch_loss, + "val_loss": val_loss, + } + timer = atomic_torch_save(checkpoint, args.resume, timer) -# val_loss += loss.item() -# val_images_seen += len(images) + return unet, timer -# val_loss /= val_images_seen -# val_losses.append(val_loss) -# print(f"Epoch {epoch} diff val loss: {val_loss:.4f}") -# progress_bar.close() +def evaluate_diffusion( + args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, + train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer + ): -# return val_losses # ?? + unet.eval() + + val_step = val_sampler.progress // val_loader.batch_size + print(f'\nEvaluating / resuming epoch {epoch} from training step {val_step}\n') + + with torch.no_grad(): + for step, batch in enumerate(val_loader): + + images = batch["image"].to(device) + timer.report(f'eval batch {val_step} to device') + + with autocast(enabled=True): + + z_mu, z_sigma = generator.encode(images) + timer.report(f'eval batch {val_step} generator encoded') + z = generator.sampling(z_mu, z_sigma) + timer.report(f'eval batch {val_step} generator sampling') + noise = torch.randn_like(z).to(device) + timer.report(f'eval batch {val_step} noise') + timesteps = torch.randint(0, inferer.scheduler.num_train_timesteps, (z.shape[0],), device=z.device).long() + timer.report(f'eval batch {val_step} timesteps') + noise_pred = inferer(inputs=images,diffusion_model=unet,noise=noise,timesteps=timesteps,autoencoder_model=generator) + timer.report(f'eval batch {val_step} noise_pred') + loss = F.mse_loss(noise_pred.float(), noise.float()) + timer.report(f'eval batch {val_step} loss') + + val_loss += loss.item() + val_images_seen += len(images) + timer.report(f'eval batch {val_step} metrics update') + + ## Checkpointing + print(f"Saving checkpoint at epoch {epoch} val batch {val_step}") + val_sampler.advance(len(images)) + if utils.is_main_process() and val_step % 1 == 0: # Checkpointing every batch + print(f"Saving checkpoint at epoch {epoch} train batch {val_step}") + checkpoint = { + # Universals + "args": args, + "epoch": epoch, + + # State variables + "unet": unet.module.state_dict(), + "optimizer_u": optimizer_u.state_dict(), + "scaler_u": scaler_u.state_dict(), + "train_sampler": train_sampler.state_dict(), + "val_sampler": val_sampler.state_dict(), + + # Evaluation metrics + "train_images_seen": train_images_seen, + "val_images_seen": val_images_seen, + "epoch_loss": epoch_loss, + "val_loss": val_loss, + } + timer = atomic_torch_save(checkpoint, args.resume, timer) + + val_loss /= val_images_seen + print(f"Epoch {epoch} diff val loss: {val_loss:.4f}") + + return timer diff --git a/monai/brats_mri_2d/train_cycling_diff.py b/monai/brats_mri_2d/train_cycling_diff.py index 1ccd5b9b..d1c67757 100644 --- a/monai/brats_mri_2d/train_cycling_diff.py +++ b/monai/brats_mri_2d/train_cycling_diff.py @@ -108,15 +108,15 @@ def main(args, timer): timer.report('build dataloaders') - # # Auto-encoder definition - # generator = AutoencoderKL( - # spatial_dims=2, in_channels=1, out_channels=1, num_channels=(128, 128, 256), latent_channels=3, - # num_res_blocks=2, attention_levels=(False, False, False),with_encoder_nonlocal_attn=False, - # with_decoder_nonlocal_attn=False, - # ) - # generator = generator.to(device) + # Auto-encoder definition + generator = AutoencoderKL( + spatial_dims=2, in_channels=1, out_channels=1, num_channels=(128, 128, 256), latent_channels=3, + num_res_blocks=2, attention_levels=(False, False, False),with_encoder_nonlocal_attn=False, + with_decoder_nonlocal_attn=False, + ) + generator = generator.to(device) - # timer.report('generator to device') + timer.report('generator to device') # # Discriminator definition # discriminator = PatchDiscriminator( @@ -176,8 +176,13 @@ def main(args, timer): timer.report('grad scalers') # Init tracking metrics - # ??? + train_images_seen = 0 + val_images_seen = 0 + epoch_loss = 0 + val_loss = 0 + # RETRIEVE GENERATOR CHECKPOINT FROM PREVIOUS JOB + # RETRIEVE CHECKPOINT Path(args.resume).parent.mkdir(parents=True, exist_ok=True) if args.resume and os.path.isfile(args.resume): # If we're resuming... @@ -191,7 +196,10 @@ def main(args, timer): train_images_seen = checkpoint["train_images_seen"] val_images_seen = checkpoint["val_images_seen"] # Metrics - # ??? + train_images_seen = checkpoint["train_images_seen"] + val_images_seen = checkpoint["val_images_seen"] + epoch_loss = checkpoint["epoch_loss"] + val_loss = checkpoint["val_loss"] timer.report('checkpoint retrieval') @@ -200,24 +208,29 @@ def main(args, timer): # n_gen_epochs = 100 # gen_val_interval = 1 - # for epoch in range(n_gen_epochs): + # for epoch in range(args.start_epoch, n_gen_epochs): # print('\n') - # print(f"EPOCH (gen) :: {epoch}") + # print(f"EPOCH :: {epoch}") # print('\n') # with train_sampler.in_epoch(epoch): - # timer = Timer() - # generator, timer, _, _, _ = train_generator_one_epoch( - # epoch, generator, discriminator, optimizer_g, optimizer_d, - # scaler_g, scaler_d, train_loader, perceptual_loss, adv_loss, device, timer + # timer = Timer("Start training") + # generator, timer = train_generator_one_epoch( + # args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, + # scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, + # train_images_seen, val_images_seen, epoch_loss, gen_epoch_loss, disc_epoch_loss, val_loss # ) # timer.report(f'training generator for epoch {epoch}') # if epoch % gen_val_interval == 0: # Eval every epoch # with val_sampler.in_epoch(epoch): - # timer = Timer() - # timer, _ = evaluate_generator(epoch, generator, val_loader, device, timer) + # timer = Timer("Start evaluation") + # timer = evaluate_generator( + # args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, + # scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, + # train_images_seen, val_images_seen, epoch_loss, gen_epoch_loss, disc_epoch_loss, val_loss + # ) # timer.report(f'evaluating generator for epoch {epoch}') @@ -236,24 +249,28 @@ def main(args, timer): timer.report('building inferer') - for epoch in range(n_diff_epochs): + for epoch in range(args.start_epoch, n_diff_epochs): print('\n') print(f"EPOCH (diff) :: {epoch}") print('\n') with train_sampler.in_epoch(epoch): - timer = Timer() + timer = Timer("Start training") unet, timer, _ = train_diffusion_one_epoch( - epoch, unet, generator, optimizer_u, - inferer, scaler_u, train_loader, device, timer + args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, + train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer ) timer.report(f'training unet for epoch {epoch}') if epoch % diff_val_interval == 0: with val_sampler.in_epoch(epoch): + timer = Timer("Start evaluation") timer = Timer() - _ = evaluate_diffusion(epoch, unet, generator, inferer, val_loader, device) + _ = evaluate_diffusion( + args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, + train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer + ) timer.report(f'evaluating unet for epoch {epoch}') diff --git a/monai/brats_mri_2d/train_cycling_gen.py b/monai/brats_mri_2d/train_cycling_gen.py index 8f663949..95d81b3b 100644 --- a/monai/brats_mri_2d/train_cycling_gen.py +++ b/monai/brats_mri_2d/train_cycling_gen.py @@ -91,9 +91,9 @@ def main(args, timer): num_workers=4, download=False, seed=0, transform=train_transforms, ) - # ## SUBSET FOR TESTING - # train_ds = torch.utils.data.Subset(train_ds, torch.arange(2*9*3)) # batch_size x nodes x iterations - # val_ds = torch.utils.data.Subset(val_ds, torch.arange(1*9*2)) # batch_size x nodes x iterations + ## SUBSET FOR TESTING + train_ds = torch.utils.data.Subset(train_ds, torch.arange(2*9*3)) # batch_size x nodes x iterations + val_ds = torch.utils.data.Subset(val_ds, torch.arange(1*9*2)) # batch_size x nodes x iterations timer.report('build datasets') @@ -196,9 +196,9 @@ def main(args, timer): scaler_d.load_state_dict(checkpoint["scaler_d"]) train_sampler.load_state_dict(checkpoint["train_sampler"]) val_sampler.load_state_dict(checkpoint["val_sampler"]) + # Metrics train_images_seen = checkpoint["train_images_seen"] val_images_seen = checkpoint["val_images_seen"] - # Metrics epoch_loss = checkpoint["epoch_loss"] gen_epoch_loss = checkpoint["gen_epoch_loss"] disc_epoch_loss = checkpoint["disc_epoch_loss"] From 7259776e49f49d4a597dfc3f942d0ddcd6aebbaf Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Wed, 13 Sep 2023 17:31:20 +1000 Subject: [PATCH 22/44] potential fix for cudnnbatchnorm error --- cycling_utils/cycling_utils/timer.py | 2 +- monai/brats_mri_2d/loops.py | 4 ++-- monai/brats_mri_2d/train_cycling_diff.py | 2 +- monai/brats_mri_2d/train_cycling_gen.py | 12 +++++++++--- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/cycling_utils/cycling_utils/timer.py b/cycling_utils/cycling_utils/timer.py index 788abd86..47d68b84 100644 --- a/cycling_utils/cycling_utils/timer.py +++ b/cycling_utils/cycling_utils/timer.py @@ -7,7 +7,7 @@ def __init__(self, report=None, start_time=None, running=0): self.running = running if str(os.environ["RANK"]) == "0": report = report if report else "Start" - print("{:<70}{:>12} ms, {:>12} s total".format(report, f'{0.0:,.3f}', f'{0.0:,.2f}')) + print("{:<80}{:>12} ms, {:>12} s total".format(report, f'{0.0:,.3f}', f'{0.0:,.2f}')) def report(self, annot): if str(os.environ["RANK"]) == "0": now = time.time() diff --git a/monai/brats_mri_2d/loops.py b/monai/brats_mri_2d/loops.py index 7384f8cd..139e0ee0 100644 --- a/monai/brats_mri_2d/loops.py +++ b/monai/brats_mri_2d/loops.py @@ -67,9 +67,9 @@ def train_generator_one_epoch( if epoch > generator_warm_up_n_epochs: # Train generator for n epochs before starting discriminator training - with autocast(enabled=True): + optimizer_d.zero_grad(set_to_none=True) - optimizer_d.zero_grad(set_to_none=True) + with autocast(enabled=True): logits_fake = discriminator(reconstruction.contiguous().detach())[-1] timer.report(f'train batch {train_step} discriminator forward (fake)') diff --git a/monai/brats_mri_2d/train_cycling_diff.py b/monai/brats_mri_2d/train_cycling_diff.py index d1c67757..d9f9d81e 100644 --- a/monai/brats_mri_2d/train_cycling_diff.py +++ b/monai/brats_mri_2d/train_cycling_diff.py @@ -104,7 +104,7 @@ def main(args, timer): train_loader = DataLoader(train_ds, batch_size=2, sampler=train_sampler, num_workers=args.workers) val_loader = DataLoader(val_ds, batch_size=1, sampler=val_sampler, num_workers=args.workers) - # check_data = first(train_loader) # Used later + check_data = first(train_loader) # Used later timer.report('build dataloaders') diff --git a/monai/brats_mri_2d/train_cycling_gen.py b/monai/brats_mri_2d/train_cycling_gen.py index 95d81b3b..e6a58309 100644 --- a/monai/brats_mri_2d/train_cycling_gen.py +++ b/monai/brats_mri_2d/train_cycling_gen.py @@ -91,9 +91,9 @@ def main(args, timer): num_workers=4, download=False, seed=0, transform=train_transforms, ) - ## SUBSET FOR TESTING - train_ds = torch.utils.data.Subset(train_ds, torch.arange(2*9*3)) # batch_size x nodes x iterations - val_ds = torch.utils.data.Subset(val_ds, torch.arange(1*9*2)) # batch_size x nodes x iterations + # ## SUBSET FOR TESTING + # train_ds = torch.utils.data.Subset(train_ds, torch.arange(2*9*3)) # batch_size x nodes x iterations + # val_ds = torch.utils.data.Subset(val_ds, torch.arange(1*9*2)) # batch_size x nodes x iterations timer.report('build datasets') @@ -145,6 +145,10 @@ def main(args, timer): timer.report('loss functions') # Prepare for distributed training + generator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) + discriminator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) + # unet = = torch.nn.SyncBatchNorm.convert_sync_batchnorm(unet) + generator_without_ddp = generator discriminator_without_ddp = discriminator # unet_without_ddp = unet @@ -206,6 +210,8 @@ def main(args, timer): timer.report('checkpoint retrieval') + torch.autograd.set_detect_anomaly(mode=True, check_nan=False) + ## -- TRAINING THE AUTO-ENCODER - ## n_gen_epochs = 100 From 91fc3143ca6a9db67b3c67526ec8ee9de3a5d183 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Wed, 13 Sep 2023 18:08:51 +1000 Subject: [PATCH 23/44] minor update --- monai/brats_mri_2d/train_cycling_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/brats_mri_2d/train_cycling_gen.py b/monai/brats_mri_2d/train_cycling_gen.py index e6a58309..d0180fd1 100644 --- a/monai/brats_mri_2d/train_cycling_gen.py +++ b/monai/brats_mri_2d/train_cycling_gen.py @@ -16,7 +16,7 @@ from monai.utils import first, set_determinism from torch.cuda.amp import GradScaler# , autocast from pathlib import Path -from tqdm import tqdm +# from tqdm import tqdm # from generative.inferers import LatentDiffusionInferer from generative.losses.adversarial_loss import PatchAdversarialLoss From 70cba9d913070f630cd0ad28c5f22b5f31a497b2 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Wed, 13 Sep 2023 22:07:40 +1000 Subject: [PATCH 24/44] still failing when discriminator included in loss --- monai/brats_mri_2d/train_cycling_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/brats_mri_2d/train_cycling_gen.py b/monai/brats_mri_2d/train_cycling_gen.py index d0180fd1..3c4f77da 100644 --- a/monai/brats_mri_2d/train_cycling_gen.py +++ b/monai/brats_mri_2d/train_cycling_gen.py @@ -210,7 +210,7 @@ def main(args, timer): timer.report('checkpoint retrieval') - torch.autograd.set_detect_anomaly(mode=True, check_nan=False) + # torch.autograd.set_detect_anomaly(mode=True, check_nan=False) ## -- TRAINING THE AUTO-ENCODER - ## From e2863d1986e33c5713361889449681574d9778a4 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Thu, 14 Sep 2023 15:31:21 +1000 Subject: [PATCH 25/44] autoencoder issue fixed --- hello_world/hello_world.isc | 7 +++++++ hello_world/hello_world.py | 4 ++++ monai/brats_mri_2d/loops.py | 2 +- monai/brats_mri_2d/prep.py | 15 ++++++++++++--- monai/brats_mri_2d/train_cycling_gen.py | 16 ++++++---------- tv-detection/maskrcnn_resnet50_fpn_small.isc | 2 +- tv-detection/train_cycling_small.py | 2 +- tv-segmentation/fcn_resnet101.isc | 2 +- 8 files changed, 33 insertions(+), 17 deletions(-) create mode 100644 hello_world/hello_world.isc create mode 100644 hello_world/hello_world.py diff --git a/hello_world/hello_world.isc b/hello_world/hello_world.isc new file mode 100644 index 00000000..c956c7af --- /dev/null +++ b/hello_world/hello_world.isc @@ -0,0 +1,7 @@ + +experiment_name="hello_world" +gpu_type="24GB VRAM GPU" +nnodes = 11 +venv_path = "~/.venv/bin/activate" +output_path = "~/output_hello_world" +command="hello_world.py --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/hello_world/hello_world.py b/hello_world/hello_world.py new file mode 100644 index 00000000..0b0eab52 --- /dev/null +++ b/hello_world/hello_world.py @@ -0,0 +1,4 @@ +import socket + +print("Hello world!") +print(socket.gethostname()) \ No newline at end of file diff --git a/monai/brats_mri_2d/loops.py b/monai/brats_mri_2d/loops.py index 139e0ee0..106dc3d9 100644 --- a/monai/brats_mri_2d/loops.py +++ b/monai/brats_mri_2d/loops.py @@ -15,7 +15,7 @@ def train_generator_one_epoch( # Maybe pull these out into args later kl_weight = 1e-6 - generator_warm_up_n_epochs = 10 + generator_warm_up_n_epochs = 3 perceptual_weight = 0.001 adv_weight = 0.01 diff --git a/monai/brats_mri_2d/prep.py b/monai/brats_mri_2d/prep.py index 17766722..3667da0d 100644 --- a/monai/brats_mri_2d/prep.py +++ b/monai/brats_mri_2d/prep.py @@ -1,4 +1,13 @@ -# Obtain the dataset -from monai.apps import DecathlonDataset +# # Download the toy dataset from MONAI +# print("Downloadning BraTS2016/17") +# from monai.apps import DecathlonDataset +# _ = DecathlonDataset(root_dir="/mnt/Datasets/Open-Datasets/MONAI", task="Task01_BrainTumour", section="training", download=True) -_ = DecathlonDataset(root_dir="/mnt/Datasets/Open-Datasets/MONAI", task="Task01_BrainTumour", section="training", download=True) \ No newline at end of file +# # Download the bigger dataset from Synapse +# print("Downloadning BraTS2023") +# import synapseclient +# syn = synapseclient.Synapse() +# syn.login('adam_peaston','AXXXXXXXXX2') +# syn51514132 = syn.get(entity='syn51514132', downloadFile=True, downloadLocation="/mnt/Datasets/strongcompute_adam/MONAI", ifcollision="overwrite.local") +# filepath = syn51514132.path +# print(f"BraTS2023-GLI downloaded to {filepath}") \ No newline at end of file diff --git a/monai/brats_mri_2d/train_cycling_gen.py b/monai/brats_mri_2d/train_cycling_gen.py index 3c4f77da..ef9eee81 100644 --- a/monai/brats_mri_2d/train_cycling_gen.py +++ b/monai/brats_mri_2d/train_cycling_gen.py @@ -84,17 +84,13 @@ def main(args, timer): train_ds = DecathlonDataset( root_dir=args.data_path, task="Task01_BrainTumour", section="training", cache_rate=0.0, - num_workers=4, download=False, seed=0, transform=train_transforms, + num_workers=1, download=False, seed=0, transform=train_transforms, ) val_ds = DecathlonDataset( root_dir=args.data_path, task="Task01_BrainTumour", section="validation", cache_rate=0.0, - num_workers=4, download=False, seed=0, transform=train_transforms, + num_workers=1, download=False, seed=0, transform=train_transforms, ) - # ## SUBSET FOR TESTING - # train_ds = torch.utils.data.Subset(train_ds, torch.arange(2*9*3)) # batch_size x nodes x iterations - # val_ds = torch.utils.data.Subset(val_ds, torch.arange(1*9*2)) # batch_size x nodes x iterations - timer.report('build datasets') train_sampler = InterruptableDistributedSampler(train_ds) @@ -102,8 +98,8 @@ def main(args, timer): timer.report('build samplers') - train_loader = DataLoader(train_ds, batch_size=2, sampler=train_sampler, num_workers=args.workers) - val_loader = DataLoader(val_ds, batch_size=1, sampler=val_sampler, num_workers=args.workers) + train_loader = DataLoader(train_ds, batch_size=2, sampler=train_sampler, num_workers=1) + val_loader = DataLoader(val_ds, batch_size=1, sampler=val_sampler, num_workers=1) # check_data = first(train_loader) # Used later timer.report('build dataloaders') @@ -154,8 +150,8 @@ def main(args, timer): # unet_without_ddp = unet # perceptual_loss_without_ddp = perceptual_loss if args.distributed: - generator = torch.nn.parallel.DistributedDataParallel(generator, device_ids=[args.gpu]) - discriminator = torch.nn.parallel.DistributedDataParallel(discriminator, device_ids=[args.gpu]) + generator = torch.nn.parallel.DistributedDataParallel(generator, device_ids=[args.gpu], find_unused_parameters=True) # find_unused_parameters necessary for monai training + discriminator = torch.nn.parallel.DistributedDataParallel(discriminator, device_ids=[args.gpu], find_unused_parameters=True) # find_unused_parameters necessary for monai training # unet = torch.nn.parallel.DistributedDataParallel(unet, device_ids=[args.gpu]) # perceptual_loss = torch.nn.parallel.DistributedDataParallel(perceptual_loss, device_ids=[args.gpu]) generator_without_ddp = generator.module diff --git a/tv-detection/maskrcnn_resnet50_fpn_small.isc b/tv-detection/maskrcnn_resnet50_fpn_small.isc index 32d8d5d2..1e3448d1 100644 --- a/tv-detection/maskrcnn_resnet50_fpn_small.isc +++ b/tv-detection/maskrcnn_resnet50_fpn_small.isc @@ -3,4 +3,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_maskrcnn_resnet50_fpn" -command="train_cycling_small.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/Datasets/Open-Datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file +command="train_cycling_small.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/tv-detection/train_cycling_small.py b/tv-detection/train_cycling_small.py index 10c7622d..4fb6f148 100644 --- a/tv-detection/train_cycling_small.py +++ b/tv-detection/train_cycling_small.py @@ -38,7 +38,7 @@ import torchvision.models.detection import torchvision.models.detection.mask_rcnn from engine import evaluate, train_one_epoch -from group_by_aspect_ratio_SC import create_aspect_ratio_groups, GroupedBatchSampler +from group_by_aspect_ratio import create_aspect_ratio_groups, GroupedBatchSampler from torchvision.transforms import InterpolationMode from transforms import SimpleCopyPaste diff --git a/tv-segmentation/fcn_resnet101.isc b/tv-segmentation/fcn_resnet101.isc index 1b8d72d8..13786ca1 100644 --- a/tv-segmentation/fcn_resnet101.isc +++ b/tv-segmentation/fcn_resnet101.isc @@ -4,4 +4,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_fcn_resnet101" -command="train_cycling.py --lr 0.0002 --dataset coco -b 2 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/mnt/Datasets/Open-Datasets/coco --epochs=1 --resume $OUTPUT_PATH/checkpoint.isc" +command="train_cycling.py --lr 0.0002 --dataset coco --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/mnt/Datasets/Open-Datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file From 43b3bd2dd6176e60322fea530447a34fcf6acb6e Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Fri, 15 Sep 2023 17:28:10 +1000 Subject: [PATCH 26/44] diffusion model training, finished 3 epochs --- cycling_utils/cycling_utils/saving.py | 3 +- monai/brats_mri_2d/brats_mri_2d_diff.isc | 2 +- monai/brats_mri_2d/brats_mri_2d_gen.isc | 2 +- monai/brats_mri_2d/prep.py | 8 +++-- monai/brats_mri_2d/train_cycling_diff.py | 30 +++++++++++-------- monai/brats_mri_2d/train_cycling_gen.py | 12 ++++---- tv-detection/maskrcnn_resnet50_fpn.isc | 2 +- tv-detection/retinanet_resnet50_fpn.isc | 2 +- tv-detection/train_cycling.py | 10 +++++-- .../deeplabv3_mobilenet_v3_large.isc | 2 +- tv-segmentation/fcn_resnet101.isc | 2 +- tv-segmentation/train_cycling.py | 10 ++++++- 12 files changed, 55 insertions(+), 30 deletions(-) diff --git a/cycling_utils/cycling_utils/saving.py b/cycling_utils/cycling_utils/saving.py index 0ca69743..d72112fa 100644 --- a/cycling_utils/cycling_utils/saving.py +++ b/cycling_utils/cycling_utils/saving.py @@ -13,4 +13,5 @@ def atomic_torch_save(obj, f: str | Path, timer=None, **kwargs): timer.report(f'replacing temp checkpoint with checkpoint') return timer else: - return \ No newline at end of file + return + diff --git a/monai/brats_mri_2d/brats_mri_2d_diff.isc b/monai/brats_mri_2d/brats_mri_2d_diff.isc index daad1731..da3a923e 100644 --- a/monai/brats_mri_2d/brats_mri_2d_diff.isc +++ b/monai/brats_mri_2d/brats_mri_2d_diff.isc @@ -3,4 +3,4 @@ gpu_type="24GB VRAM GPU" nnodes = 9 venv_path = "~/.venv/bin/activate" output_path = "~/output_brats_mri_2d_diff" -command="train_cycling_diff.py --data-path=/mnt/Datasets/Open-Datasets/MONAI --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file +command="train_cycling_diff.py --data-path=/workspace/datasets/MONAI --resume $OUTPUT_PATH/checkpoint.isc, --gen-load-path ~/output_brats_mri_2d_gen/exp_1645/checkpoint.isc" \ No newline at end of file diff --git a/monai/brats_mri_2d/brats_mri_2d_gen.isc b/monai/brats_mri_2d/brats_mri_2d_gen.isc index c57c5e55..386846ce 100644 --- a/monai/brats_mri_2d/brats_mri_2d_gen.isc +++ b/monai/brats_mri_2d/brats_mri_2d_gen.isc @@ -3,4 +3,4 @@ gpu_type="24GB VRAM GPU" nnodes = 9 venv_path = "~/.venv/bin/activate" output_path = "~/output_brats_mri_2d_gen" -command="train_cycling_gen.py --data-path=/mnt/Datasets/Open-Datasets/MONAI --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file +command="train_cycling_gen.py --data-path=/workspace/datasets/MONAI --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/monai/brats_mri_2d/prep.py b/monai/brats_mri_2d/prep.py index 3667da0d..84e0c1c1 100644 --- a/monai/brats_mri_2d/prep.py +++ b/monai/brats_mri_2d/prep.py @@ -1,7 +1,9 @@ -# # Download the toy dataset from MONAI -# print("Downloadning BraTS2016/17") -# from monai.apps import DecathlonDataset +# Download the toy dataset from MONAI +print("Downloadning BraTS2016/17") +from monai.apps import DecathlonDataset + # _ = DecathlonDataset(root_dir="/mnt/Datasets/Open-Datasets/MONAI", task="Task01_BrainTumour", section="training", download=True) +_ = DecathlonDataset(root_dir="/workspace/datasets/MONAI", task="Task01_BrainTumour", section="training", download=True) # # Download the bigger dataset from Synapse # print("Downloadning BraTS2023") diff --git a/monai/brats_mri_2d/train_cycling_diff.py b/monai/brats_mri_2d/train_cycling_diff.py index d9f9d81e..d4e1ab14 100644 --- a/monai/brats_mri_2d/train_cycling_diff.py +++ b/monai/brats_mri_2d/train_cycling_diff.py @@ -35,6 +35,8 @@ def get_args_parser(add_help=True): parser = argparse.ArgumentParser(description="Latent Diffusion Model Training", add_help=add_help) parser.add_argument("--resume", type=str, help="path of checkpoint", required=True) # for checkpointing + parser.add_argument("--gen-load-path", type=str, help="path of checkpoint", dest="gen_load_path") # for checkpointing + parser.add_argument("--prev-resume", default=None, help="path of previous job checkpoint for strong fail resume", dest="prev_resume") # for checkpointing # parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") parser.add_argument("--data-path", default="/mnt/Datasets/Open-Datasets/MONAI", type=str, help="dataset path", dest="data_path") parser.add_argument("--device", default="cuda", type=str, help="device (Use cuda or cpu Default: cuda)") @@ -91,10 +93,6 @@ def main(args, timer): num_workers=4, download=False, seed=0, transform=train_transforms, ) - # ## SUBSET FOR TESTING - # train_ds = torch.utils.data.Subset(train_ds, torch.arange(2*9*3)) # batch_size x nodes x iterations - # val_ds = torch.utils.data.Subset(val_ds, torch.arange(1*9*2)) # batch_size x nodes x iterations - timer.report('build datasets') train_sampler = InterruptableDistributedSampler(train_ds) @@ -114,6 +112,9 @@ def main(args, timer): num_res_blocks=2, attention_levels=(False, False, False),with_encoder_nonlocal_attn=False, with_decoder_nonlocal_attn=False, ) + # saved_generator_checkpoint = torch.load("/output_brats_mri_2d_gen/exp_1645/checkpoint.isc", map_location="cpu") + saved_generator_checkpoint = torch.load(args.gen_load_path, map_location="cpu") + generator.load_state_dict(saved_generator_checkpoint["generator"]) generator = generator.to(device) timer.report('generator to device') @@ -145,19 +146,20 @@ def main(args, timer): # timer.report('loss functions') # Prepare for distributed training + # generator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) + # discriminator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) + unet = torch.nn.SyncBatchNorm.convert_sync_batchnorm(unet) + # generator_without_ddp = generator # discriminator_without_ddp = discriminator unet_without_ddp = unet - # perceptual_loss_without_ddp = perceptual_loss if args.distributed: - # generator = torch.nn.parallel.DistributedDataParallel(generator, device_ids=[args.gpu]) - # discriminator = torch.nn.parallel.DistributedDataParallel(discriminator, device_ids=[args.gpu]) - unet = torch.nn.parallel.DistributedDataParallel(unet, device_ids=[args.gpu]) - # perceptual_loss = torch.nn.parallel.DistributedDataParallel(perceptual_loss, device_ids=[args.gpu]) + # generator = torch.nn.parallel.DistributedDataParallel(generator, device_ids=[args.gpu], find_unused_parameters=True) + # discriminator = torch.nn.parallel.DistributedDataParallel(discriminator, device_ids=[args.gpu], find_unused_parameters=True) + unet = torch.nn.parallel.DistributedDataParallel(unet, device_ids=[args.gpu], find_unused_parameters=True) # generator_without_ddp = generator.module # discriminator_without_ddp = discriminator.module unet_without_ddp = unet.module - # perceptual_loss_without_ddp = perceptual_loss.module timer.report('unet prepped for distribution') @@ -185,8 +187,12 @@ def main(args, timer): # RETRIEVE CHECKPOINT Path(args.resume).parent.mkdir(parents=True, exist_ok=True) + checkpoint = None if args.resume and os.path.isfile(args.resume): # If we're resuming... checkpoint = torch.load(args.resume, map_location="cpu") + elif args.prev_resume and os.path.isfile(args.prev_resume): + checkpoint = torch.load(args.prev_resume, map_location="cpu") + if checkpoint is not None: args.start_epoch = checkpoint["epoch"] unet_without_ddp.load_state_dict(checkpoint["unet"], strict=not args.test_only) optimizer_u.load_state_dict(checkpoint["optimizer_u"]) @@ -257,7 +263,7 @@ def main(args, timer): with train_sampler.in_epoch(epoch): timer = Timer("Start training") - unet, timer, _ = train_diffusion_one_epoch( + unet, timer = train_diffusion_one_epoch( args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer ) @@ -267,7 +273,7 @@ def main(args, timer): with val_sampler.in_epoch(epoch): timer = Timer("Start evaluation") timer = Timer() - _ = evaluate_diffusion( + timer = evaluate_diffusion( args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer ) diff --git a/monai/brats_mri_2d/train_cycling_gen.py b/monai/brats_mri_2d/train_cycling_gen.py index ef9eee81..9b3b7c0c 100644 --- a/monai/brats_mri_2d/train_cycling_gen.py +++ b/monai/brats_mri_2d/train_cycling_gen.py @@ -35,6 +35,7 @@ def get_args_parser(add_help=True): parser = argparse.ArgumentParser(description="Latent Diffusion Model Training", add_help=add_help) parser.add_argument("--resume", type=str, help="path of checkpoint", required=True) # for checkpointing + parser.add_argument("--prev-resume", default=None, help="path of previous job checkpoint for strong fail resume", dest="prev_resume") # for checkpointing # parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") parser.add_argument("--data-path", default="/mnt/Datasets/Open-Datasets/MONAI", type=str, help="dataset path", dest="data_path") parser.add_argument("--device", default="cuda", type=str, help="device (Use cuda or cpu Default: cuda)") @@ -143,21 +144,18 @@ def main(args, timer): # Prepare for distributed training generator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) discriminator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) - # unet = = torch.nn.SyncBatchNorm.convert_sync_batchnorm(unet) + # unet = torch.nn.SyncBatchNorm.convert_sync_batchnorm(unet) generator_without_ddp = generator discriminator_without_ddp = discriminator # unet_without_ddp = unet - # perceptual_loss_without_ddp = perceptual_loss if args.distributed: generator = torch.nn.parallel.DistributedDataParallel(generator, device_ids=[args.gpu], find_unused_parameters=True) # find_unused_parameters necessary for monai training discriminator = torch.nn.parallel.DistributedDataParallel(discriminator, device_ids=[args.gpu], find_unused_parameters=True) # find_unused_parameters necessary for monai training - # unet = torch.nn.parallel.DistributedDataParallel(unet, device_ids=[args.gpu]) - # perceptual_loss = torch.nn.parallel.DistributedDataParallel(perceptual_loss, device_ids=[args.gpu]) + # unet = torch.nn.parallel.DistributedDataParallel(unet, device_ids=[args.gpu], find_unused_parameters=True) generator_without_ddp = generator.module discriminator_without_ddp = discriminator.module # unet_without_ddp = unet.module - # perceptual_loss_without_ddp = perceptual_loss.module timer.report('models prepped for distribution') @@ -185,8 +183,12 @@ def main(args, timer): # RETRIEVE CHECKPOINT Path(args.resume).parent.mkdir(parents=True, exist_ok=True) + checkpoint = None if args.resume and os.path.isfile(args.resume): # If we're resuming... checkpoint = torch.load(args.resume, map_location="cpu") + elif args.prev_resume and os.path.isfile(args.prev_resume): + checkpoint = torch.load(args.prev_resume, map_location="cpu") + if checkpoint is not None: args.start_epoch = checkpoint["epoch"] generator_without_ddp.load_state_dict(checkpoint["generator"]) discriminator_without_ddp.load_state_dict(checkpoint["discriminator"]) diff --git a/tv-detection/maskrcnn_resnet50_fpn.isc b/tv-detection/maskrcnn_resnet50_fpn.isc index 11a52700..0b65440d 100644 --- a/tv-detection/maskrcnn_resnet50_fpn.isc +++ b/tv-detection/maskrcnn_resnet50_fpn.isc @@ -3,4 +3,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_maskrcnn_resnet50_fpn" -command="train_cycling.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/Datasets/Open-Datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file +command="train_cycling.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/tv-detection/retinanet_resnet50_fpn.isc b/tv-detection/retinanet_resnet50_fpn.isc index 6fcda85b..61b4667a 100644 --- a/tv-detection/retinanet_resnet50_fpn.isc +++ b/tv-detection/retinanet_resnet50_fpn.isc @@ -3,5 +3,5 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_retinanet_resnet50_fpn" -command="train_cycling.py --dataset coco --model retinanet_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --lr 0.01 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/Datasets/Open-Datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" +command="train_cycling.py --dataset coco --model retinanet_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --lr 0.01 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index a05c1ea2..c99cdb6e 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -235,10 +235,15 @@ def main(args, timer): timer.report('init coco evaluator') + # RETRIEVE CHECKPOINT Path(args.resume).parent.mkdir(parents=True, exist_ok=True) - if args.resume and os.path.isfile(args.resume): - + checkpoint = None + if args.resume and os.path.isfile(args.resume): # If we're resuming... checkpoint = torch.load(args.resume, map_location="cpu") + elif args.prev_resume and os.path.isfile(args.prev_resume): + checkpoint = torch.load(args.prev_resume, map_location="cpu") + if checkpoint is not None: + model_without_ddp.load_state_dict(checkpoint["model"]) args.start_epoch = checkpoint["epoch"] @@ -303,6 +308,7 @@ def get_args_parser(add_help=True): parser.add_argument("--print-freq", default=1, type=int, help="print frequency") parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") parser.add_argument("--resume", default="", type=str, help="path of checkpoint") + parser.add_argument("--prev-resume", default=None, help="path of previous job checkpoint for strong fail resume", dest="prev_resume") # for checkpointing parser.add_argument("--start_epoch", default=0, type=int, help="start epoch") parser.add_argument("--aspect-ratio-group-factor", default=3, type=int) parser.add_argument("--rpn-score-thresh", default=None, type=float, help="rpn score threshold for faster-rcnn") diff --git a/tv-segmentation/deeplabv3_mobilenet_v3_large.isc b/tv-segmentation/deeplabv3_mobilenet_v3_large.isc index 7af6132a..4ea27bd0 100644 --- a/tv-segmentation/deeplabv3_mobilenet_v3_large.isc +++ b/tv-segmentation/deeplabv3_mobilenet_v3_large.isc @@ -4,4 +4,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_deeplabv3_mobilenet_v3_large" -command="train_cycling.py --dataset coco -b 2 --model deeplabv3_mobilenet_v3_large --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 --data-path=/mnt/Datasets/Open-Datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" +command="train_cycling.py --dataset coco -b 2 --model deeplabv3_mobilenet_v3_large --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc --prev-resume ~/output_deeplabv3_mobilenet_v3_large/exp_1640/checkpoint.isc" diff --git a/tv-segmentation/fcn_resnet101.isc b/tv-segmentation/fcn_resnet101.isc index 13786ca1..20f2833b 100644 --- a/tv-segmentation/fcn_resnet101.isc +++ b/tv-segmentation/fcn_resnet101.isc @@ -4,4 +4,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_fcn_resnet101" -command="train_cycling.py --lr 0.0002 --dataset coco --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/mnt/Datasets/Open-Datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file +command="train_cycling.py --lr 0.02 --dataset coco -b 4 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index 1adda6d8..3d198b1d 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -314,10 +314,17 @@ def main(args, timer): confmat = utils.ConfusionMatrix(num_classes) timer.report('init confmat') + # RETRIEVE CHECKPOINT Path(args.resume).parent.mkdir(parents=True, exist_ok=True) + checkpoint = None if args.resume and os.path.isfile(args.resume): # If we're resuming... - checkpoint = torch.load(args.resume, map_location="cpu") + print("RESUMING FROM CURRENT JOB") + elif args.prev_resume and os.path.isfile(args.prev_resume): + print(f"RESUMING FROM PREVIOUS JOB {args.prev_resume}") + checkpoint = torch.load(args.prev_resume, map_location="cpu") + if checkpoint is not None: + model_without_ddp.load_state_dict(checkpoint["model"], strict=not args.test_only) args.start_epoch = checkpoint["epoch"] @@ -389,6 +396,7 @@ def get_args_parser(add_help=True): parser.add_argument("--print-freq", default=1, type=int, help="print frequency") parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") parser.add_argument("--resume", type=str, help="path of checkpoint", required=True) + parser.add_argument("--prev-resume", default=None, help="path of previous job checkpoint for strong fail resume", dest="prev_resume") # for checkpointing parser.add_argument("--start-epoch", default=0, type=int, metavar="N", help="start epoch") parser.add_argument( "--test-only", From 29c6bfbbc2dc608e664a92c9b0a73b0d33fe0c3d Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Fri, 15 Sep 2023 18:02:57 +1000 Subject: [PATCH 27/44] updates to args to fix issue resuming --- monai/brats_mri_2d/train_cycling_diff.py | 4 ++-- tv-segmentation/deeplabv3_mobilenet_v3_large.isc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/monai/brats_mri_2d/train_cycling_diff.py b/monai/brats_mri_2d/train_cycling_diff.py index d4e1ab14..31d3953d 100644 --- a/monai/brats_mri_2d/train_cycling_diff.py +++ b/monai/brats_mri_2d/train_cycling_diff.py @@ -194,7 +194,7 @@ def main(args, timer): checkpoint = torch.load(args.prev_resume, map_location="cpu") if checkpoint is not None: args.start_epoch = checkpoint["epoch"] - unet_without_ddp.load_state_dict(checkpoint["unet"], strict=not args.test_only) + unet_without_ddp.load_state_dict(checkpoint["unet"]) optimizer_u.load_state_dict(checkpoint["optimizer_u"]) scaler_u.load_state_dict(checkpoint["scaler_u"]) train_sampler.load_state_dict(checkpoint["train_sampler"]) @@ -258,7 +258,7 @@ def main(args, timer): for epoch in range(args.start_epoch, n_diff_epochs): print('\n') - print(f"EPOCH (diff) :: {epoch}") + print(f"EPOCH :: {epoch}") print('\n') with train_sampler.in_epoch(epoch): diff --git a/tv-segmentation/deeplabv3_mobilenet_v3_large.isc b/tv-segmentation/deeplabv3_mobilenet_v3_large.isc index 4ea27bd0..2895d54c 100644 --- a/tv-segmentation/deeplabv3_mobilenet_v3_large.isc +++ b/tv-segmentation/deeplabv3_mobilenet_v3_large.isc @@ -4,4 +4,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_deeplabv3_mobilenet_v3_large" -command="train_cycling.py --dataset coco -b 2 --model deeplabv3_mobilenet_v3_large --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc --prev-resume ~/output_deeplabv3_mobilenet_v3_large/exp_1640/checkpoint.isc" +command="train_cycling.py --dataset coco -b 2 --model deeplabv3_mobilenet_v3_large --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc --prev-resume ~/output_deeplabv3_mobilenet_v3_large/exp_1654/checkpoint.isc" From 288f0a9bdd8135524dac8dde7ba419ca59600283 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Mon, 18 Sep 2023 17:49:39 +1000 Subject: [PATCH 28/44] Latest updates --- cycling_utils/cycling_utils/saving.py | 85 +++++ monai/brats_mri_2d/brats_mri_2d_diff.isc | 2 +- monai/brats_mri_2d/brats_mri_2d_gen.isc | 2 +- monai/brats_mri_2d/prep.py | 6 +- monai/brats_mri_2d/train_cycling_diff.py | 4 +- monai/brats_mri_2d/train_cycling_gen.py | 9 +- monai/brats_mri_2d_0/brats_mri_2d_diff.isc | 6 + monai/brats_mri_2d_0/brats_mri_2d_gen_0.isc | 6 + monai/brats_mri_2d_0/loops_0.py | 351 ++++++++++++++++++ monai/brats_mri_2d_0/prep.py | 20 + monai/brats_mri_2d_0/train_cycling_diff.py | 285 ++++++++++++++ monai/brats_mri_2d_0/train_cycling_gen_0.py | 275 ++++++++++++++ monai/brats_mri_2d_0/utils.py | 72 ++++ ...nts.out.tfevents.1695014380.sc10.1361389.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc10.1361390.0 | Bin 0 -> 415 bytes ...nts.out.tfevents.1695014380.sc10.1361391.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc10.1361392.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc10.1361393.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc10.1361394.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc13.1453448.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc13.1453449.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc13.1453450.0 | Bin 0 -> 415 bytes ...nts.out.tfevents.1695014380.sc13.1453451.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc13.1453452.0 | Bin 0 -> 415 bytes ...nts.out.tfevents.1695014380.sc13.1453453.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc14.1487328.0 | Bin 0 -> 508 bytes ...nts.out.tfevents.1695014380.sc14.1487329.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc14.1487330.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc14.1487331.0 | Bin 0 -> 508 bytes ...nts.out.tfevents.1695014380.sc14.1487332.0 | Bin 0 -> 508 bytes ...nts.out.tfevents.1695014380.sc14.1487333.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc15.1433694.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc15.1433695.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc15.1433696.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc15.1433697.0 | Bin 0 -> 415 bytes ...nts.out.tfevents.1695014380.sc15.1433698.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc15.1433699.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc16.1326408.0 | Bin 0 -> 415 bytes ...nts.out.tfevents.1695014380.sc16.1326409.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc16.1326410.0 | Bin 0 -> 508 bytes ...nts.out.tfevents.1695014380.sc16.1326411.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc16.1326412.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc16.1326413.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc18.1798304.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc18.1798305.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc18.1798306.0 | Bin 0 -> 415 bytes ...nts.out.tfevents.1695014380.sc18.1798307.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc18.1798308.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc18.1798309.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc19.1242796.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc19.1242797.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc19.1242798.0 | Bin 0 -> 508 bytes ...nts.out.tfevents.1695014380.sc19.1242799.0 | Bin 0 -> 415 bytes ...nts.out.tfevents.1695014380.sc19.1242800.0 | Bin 0 -> 508 bytes ...nts.out.tfevents.1695014380.sc19.1242801.0 | Bin 0 -> 508 bytes ...nts.out.tfevents.1695014380.sc20.1423153.0 | Bin 0 -> 508 bytes ...nts.out.tfevents.1695014380.sc20.1423154.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc20.1423155.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc20.1423156.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc20.1423157.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014380.sc20.1423158.0 | Bin 0 -> 461 bytes ...ents.out.tfevents.1695014380.sc9.1402602.0 | Bin 0 -> 461 bytes ...ents.out.tfevents.1695014380.sc9.1402603.0 | Bin 0 -> 461 bytes ...ents.out.tfevents.1695014380.sc9.1402604.0 | Bin 0 -> 415 bytes ...ents.out.tfevents.1695014380.sc9.1402605.0 | Bin 0 -> 461 bytes ...ents.out.tfevents.1695014380.sc9.1402606.0 | Bin 0 -> 461 bytes ...ents.out.tfevents.1695014380.sc9.1402607.0 | Bin 0 -> 461 bytes ...nts.out.tfevents.1695014637.sc10.1364265.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc10.1364266.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc10.1364267.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc10.1364268.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc10.1364269.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc10.1364270.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc11.1659942.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc11.1659943.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc11.1659944.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc11.1659945.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc11.1659946.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc11.1659947.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc12.2917688.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc12.2917689.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc12.2917690.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc12.2917691.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc12.2917692.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc12.2917693.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc14.1490226.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc14.1490227.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc14.1490228.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc14.1490229.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc14.1490230.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc14.1490231.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc15.1436569.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc15.1436570.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc15.1436571.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc15.1436572.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc15.1436573.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc15.1436574.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc16.1329253.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc16.1329254.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc16.1329255.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc16.1329256.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc16.1329257.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc16.1329258.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc18.1800064.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc18.1800065.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc18.1800066.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc18.1800067.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc18.1800068.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc18.1800069.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc19.1244600.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc19.1244601.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc19.1244602.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc19.1244603.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc19.1244604.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc19.1244605.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc20.1425998.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc20.1425999.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc20.1426000.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc20.1426001.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc20.1426002.0 | Bin 0 -> 2669 bytes ...nts.out.tfevents.1695014637.sc20.1426003.0 | Bin 0 -> 2669 bytes tv-detection/maskrcnn_resnet50_fpn.isc | 2 +- tv-detection/maskrcnn_resnet50_fpn_small.isc | 2 +- tv-detection/retinanet_resnet50_fpn.isc | 2 +- tv-detection/train_cycling.py | 2 + .../deeplabv3_mobilenet_v3_large.isc | 2 +- tv-segmentation/fcn_resnet101.isc | 2 +- 127 files changed, 1118 insertions(+), 17 deletions(-) create mode 100644 monai/brats_mri_2d_0/brats_mri_2d_diff.isc create mode 100644 monai/brats_mri_2d_0/brats_mri_2d_gen_0.isc create mode 100644 monai/brats_mri_2d_0/loops_0.py create mode 100644 monai/brats_mri_2d_0/prep.py create mode 100644 monai/brats_mri_2d_0/train_cycling_diff.py create mode 100644 monai/brats_mri_2d_0/train_cycling_gen_0.py create mode 100644 monai/brats_mri_2d_0/utils.py create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361389.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361390.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361391.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361392.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361393.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361394.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc13.1453448.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc13.1453449.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc13.1453450.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc13.1453451.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc13.1453452.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc13.1453453.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487328.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487329.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487330.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487331.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487332.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487333.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433694.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433695.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433696.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433697.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433698.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433699.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326408.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326409.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326410.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326411.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326412.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326413.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798304.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798305.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798306.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798307.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798308.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798309.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242796.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242797.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242798.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242799.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242800.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242801.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc20.1423153.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc20.1423154.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc20.1423155.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc20.1423156.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc20.1423157.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc20.1423158.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc9.1402602.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc9.1402603.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc9.1402604.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc9.1402605.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc9.1402606.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc9.1402607.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc10.1364265.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc10.1364266.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc10.1364267.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc10.1364268.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc10.1364269.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc10.1364270.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659942.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659943.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659944.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659945.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659946.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659947.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917688.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917689.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917690.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917691.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917692.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917693.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490226.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490227.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490228.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490229.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490230.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490231.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436569.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436570.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436571.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436572.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436573.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436574.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329253.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329254.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329255.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329256.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329257.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329258.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800064.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800065.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800066.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800067.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800068.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800069.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244600.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244601.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244602.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244603.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244604.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244605.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1425998.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1425999.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426000.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426001.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426002.0 create mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426003.0 diff --git a/cycling_utils/cycling_utils/saving.py b/cycling_utils/cycling_utils/saving.py index d72112fa..0331b71d 100644 --- a/cycling_utils/cycling_utils/saving.py +++ b/cycling_utils/cycling_utils/saving.py @@ -15,3 +15,88 @@ def atomic_torch_save(obj, f: str | Path, timer=None, **kwargs): else: return +# ## ENABLING ACTIVE PROGRESS TRACKING + +# from sqlalchemy.orm import Session, sessionmaker +# from sqlmodel import SQLModel, create_engine +# from strenum import StrEnum + +# class Experiment(SQLModel, table=True): +# __tablename__ = "experiments" + +# id: int = Field(primary_key=True, index=True) +# org_id: str = Field(unique=False) +# user_id: int = Field(foreign_key="users.id", unique=False) +# user: User = Relationship(back_populates="experiments") +# runtime: int | None = Field(unique=False, nullable=True) +# name: str = Field(unique=False) +# output_path_used: str | None = Field(unique=False, nullable=True) +# output_path: str = Field(unique=False) +# ips: dict[str, int] | None = Field(sa_column=Column(JSON, nullable=True)) +# status: str = Field(nullable=False) +# gpu_type: str = Field(unique=False) +# nnodes: int = Field(unique=False) +# venv_path: str = Field(unique=False) +# command: str = Field(unique=False) +# work_dir: str = Field(unique=False) +# framework: str | None = Field(unique=False, nullable=True) +# created_at: datetime = Field(default_factory=timestamp_factory, nullable=False) +# usage_wall_time: int = Field(default=0, nullable=False) +# last_ran_at: datetime | None = Field(default=None, nullable=True) +# # started_at: datetime | None = Field(nullable=True) +# # finished_at: datetime | None = Field(nullable=True) +# progress: int = Field(default=0, nullable = False) # perhaps + + +# SQLALCHEMY_DATABASE_URL = "postgresql://postgres:postgres@localhost:5432/cluster_server" + +# def get_db(database_url: str | None = None) -> Session: +# if database_url is None: +# database_url = SQLALCHEMY_DATABASE_URL +# """Returns a session to the database""" +# engine = create_engine(database_url, isolation_level="AUTOCOMMIT") +# SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) +# return SessionLocal() + +# # SessionLocal: Callable[[], Session] = get_db +# SessionLocal = get_db + +# class AtomicTorchSave: +# def __init__(self): +# self.progress = 0 +# self.experiment_id = os.environ["STRONG_EXPERIMENT_ID"] + +# def commit_progress(self): +# db = SessionLocal() +# db_experiment = db.query(Experiment).filter(Experiment.id == self.experiment_id).first() +# assert db_experiment is not None +# db_experiment.progress = self.progress +# db.commit() +# db.refresh(db_experiment) +# return db_experiment + +# def save(self, obj, f: str | Path, timer=None, **kwargs): +# f = str(f) +# temp_f = f + ".temp" +# torch.save(obj, temp_f, **kwargs) +# if timer is not None: +# timer.report(f'saving temp checkpoint') +# os.replace(temp_f, f) +# timer.report(f'replacing temp checkpoint with checkpoint') + +# if self.experiment_id is not None: +# try: +# self.commit_progress() +# except: +# print("Progress commit failed.") +# else: +# print("Experiment id not set.") +# timer.report(f'committing progress to database') + +# if timer is not None: +# return timer +# else: +# return + + + diff --git a/monai/brats_mri_2d/brats_mri_2d_diff.isc b/monai/brats_mri_2d/brats_mri_2d_diff.isc index da3a923e..9409c35c 100644 --- a/monai/brats_mri_2d/brats_mri_2d_diff.isc +++ b/monai/brats_mri_2d/brats_mri_2d_diff.isc @@ -3,4 +3,4 @@ gpu_type="24GB VRAM GPU" nnodes = 9 venv_path = "~/.venv/bin/activate" output_path = "~/output_brats_mri_2d_diff" -command="train_cycling_diff.py --data-path=/workspace/datasets/MONAI --resume $OUTPUT_PATH/checkpoint.isc, --gen-load-path ~/output_brats_mri_2d_gen/exp_1645/checkpoint.isc" \ No newline at end of file +command="train_cycling_diff.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc, --gen-load-path ~/output_brats_mri_2d_gen/exp_1645/checkpoint.isc" \ No newline at end of file diff --git a/monai/brats_mri_2d/brats_mri_2d_gen.isc b/monai/brats_mri_2d/brats_mri_2d_gen.isc index 386846ce..ff29a6c2 100644 --- a/monai/brats_mri_2d/brats_mri_2d_gen.isc +++ b/monai/brats_mri_2d/brats_mri_2d_gen.isc @@ -3,4 +3,4 @@ gpu_type="24GB VRAM GPU" nnodes = 9 venv_path = "~/.venv/bin/activate" output_path = "~/output_brats_mri_2d_gen" -command="train_cycling_gen.py --data-path=/workspace/datasets/MONAI --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file +command="train_cycling_gen.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/monai/brats_mri_2d/prep.py b/monai/brats_mri_2d/prep.py index 84e0c1c1..a3a43134 100644 --- a/monai/brats_mri_2d/prep.py +++ b/monai/brats_mri_2d/prep.py @@ -3,7 +3,11 @@ from monai.apps import DecathlonDataset # _ = DecathlonDataset(root_dir="/mnt/Datasets/Open-Datasets/MONAI", task="Task01_BrainTumour", section="training", download=True) -_ = DecathlonDataset(root_dir="/workspace/datasets/MONAI", task="Task01_BrainTumour", section="training", download=True) +_ = DecathlonDataset(root_dir="/mnt/.node1/Open-Datsets/MONAI", task="Task01_BrainTumour", section="training", download=True) + +perceptual_loss = PerceptualLoss( + spatial_dims=2, network_type="resnet50", pretrained=True, #ImageNet pretrained weights used +) # # Download the bigger dataset from Synapse # print("Downloadning BraTS2023") diff --git a/monai/brats_mri_2d/train_cycling_diff.py b/monai/brats_mri_2d/train_cycling_diff.py index 31d3953d..c1665320 100644 --- a/monai/brats_mri_2d/train_cycling_diff.py +++ b/monai/brats_mri_2d/train_cycling_diff.py @@ -69,6 +69,7 @@ def main(args, timer): channel = 0 # 0 = Flair assert channel in [0, 1, 2, 3], "Choose a valid channel" + ## NEED TO COME BACK AND ALIGN WITH BRATS CONFIG train_transforms = transforms.Compose([ transforms.LoadImaged(keys=["image", "label"], image_only=False), # image_only current default will change soon, so including explicitly transforms.EnsureChannelFirstd(keys=["image", "label"]), @@ -78,7 +79,7 @@ def main(args, timer): transforms.Spacingd(keys=["image", "label"], pixdim=(3.0, 3.0, 2.0), mode=("bilinear", "nearest")), transforms.CenterSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 44)), transforms.ScaleIntensityRangePercentilesd(keys="image", lower=0, upper=99.5, b_min=0, b_max=1), - transforms.RandSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 1), random_size=False), + transforms.RandSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 1), random_size=False, random_centre=True), # Eeach of the 44 slices will be randomly sampled. transforms.Lambdad(keys=["image", "label"], func=lambda x: x.squeeze(-1)), transforms.CopyItemsd(keys=["label"], times=1, names=["slice_label"]), transforms.Lambdad(keys=["slice_label"], func=lambda x: 1.0 if x.sum() > 0 else 0.0), @@ -272,7 +273,6 @@ def main(args, timer): if epoch % diff_val_interval == 0: with val_sampler.in_epoch(epoch): timer = Timer("Start evaluation") - timer = Timer() timer = evaluate_diffusion( args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer diff --git a/monai/brats_mri_2d/train_cycling_gen.py b/monai/brats_mri_2d/train_cycling_gen.py index 9b3b7c0c..72e93c29 100644 --- a/monai/brats_mri_2d/train_cycling_gen.py +++ b/monai/brats_mri_2d/train_cycling_gen.py @@ -52,10 +52,6 @@ def get_args_parser(add_help=True): def main(args, timer): - # ## Distributed training prelims - # if args.output_dir: - # utils.mkdir(args.output_dir) - utils.init_distributed_mode(args) # Sets args.distributed among other things assert args.distributed # don't support cycling when not distributed for simplicity @@ -68,6 +64,7 @@ def main(args, timer): channel = 0 # 0 = Flair assert channel in [0, 1, 2, 3], "Choose a valid channel" + ## NEED TO COME BACK AND ALIGN WITH BRATS CONFIG train_transforms = transforms.Compose([ transforms.LoadImaged(keys=["image", "label"], image_only=False), # image_only current default will change soon, so including explicitly transforms.EnsureChannelFirstd(keys=["image", "label"]), @@ -77,7 +74,7 @@ def main(args, timer): transforms.Spacingd(keys=["image", "label"], pixdim=(3.0, 3.0, 2.0), mode=("bilinear", "nearest")), transforms.CenterSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 44)), transforms.ScaleIntensityRangePercentilesd(keys="image", lower=0, upper=99.5, b_min=0, b_max=1), - transforms.RandSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 1), random_size=False), + transforms.RandSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 1), random_size=False), # Each of the 44 slices will be randomly sampled. transforms.Lambdad(keys=["image", "label"], func=lambda x: x.squeeze(-1)), transforms.CopyItemsd(keys=["label"], times=1, names=["slice_label"]), transforms.Lambdad(keys=["slice_label"], func=lambda x: 1.0 if x.sum() > 0 else 0.0), @@ -208,8 +205,6 @@ def main(args, timer): timer.report('checkpoint retrieval') - # torch.autograd.set_detect_anomaly(mode=True, check_nan=False) - ## -- TRAINING THE AUTO-ENCODER - ## n_gen_epochs = 100 diff --git a/monai/brats_mri_2d_0/brats_mri_2d_diff.isc b/monai/brats_mri_2d_0/brats_mri_2d_diff.isc new file mode 100644 index 00000000..9409c35c --- /dev/null +++ b/monai/brats_mri_2d_0/brats_mri_2d_diff.isc @@ -0,0 +1,6 @@ +experiment_name="brats_mri_2d_diff" +gpu_type="24GB VRAM GPU" +nnodes = 9 +venv_path = "~/.venv/bin/activate" +output_path = "~/output_brats_mri_2d_diff" +command="train_cycling_diff.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc, --gen-load-path ~/output_brats_mri_2d_gen/exp_1645/checkpoint.isc" \ No newline at end of file diff --git a/monai/brats_mri_2d_0/brats_mri_2d_gen_0.isc b/monai/brats_mri_2d_0/brats_mri_2d_gen_0.isc new file mode 100644 index 00000000..6829fad6 --- /dev/null +++ b/monai/brats_mri_2d_0/brats_mri_2d_gen_0.isc @@ -0,0 +1,6 @@ +experiment_name="brats_mri_2d_gen" +gpu_type="24GB VRAM GPU" +nnodes = 9 +venv_path = "~/.venv/bin/activate" +output_path = "~/output_brats_mri_2d_gen" +command="train_cycling_gen_0.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/monai/brats_mri_2d_0/loops_0.py b/monai/brats_mri_2d_0/loops_0.py new file mode 100644 index 00000000..c9c8d804 --- /dev/null +++ b/monai/brats_mri_2d_0/loops_0.py @@ -0,0 +1,351 @@ +from tqdm import tqdm +import torch +from torch.cuda.amp import GradScaler, autocast +import torch.nn.functional as F +import torch.distributed as dist +import utils +from cycling_utils import atomic_torch_save +from torch.utils.tensorboard import SummaryWriter +tb_path = "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/output_brats_mri_2d_gen/tb" + +## -- AUTO-ENCODER - ## + +def train_generator_one_epoch( + args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, + scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, + metrics + ): + + # Maybe pull these out into args later + kl_weight = 1e-6 + generator_warm_up_n_epochs = 3 + perceptual_weight = 0.001 + adv_weight = 0.01 + + generator.train() + discriminator.train() + + train_step = train_sampler.progress // train_loader.batch_size + total_steps = int(len(train_sampler) / train_loader.batch_size) + print(f'\nTraining / resuming epoch {epoch} from training step {train_step}\n') + + for step, batch in enumerate(train_loader): + + images = batch["image"].to(device) + timer.report(f'train batch {train_step} to device') + + # TRAIN GENERATOR + + optimizer_g.zero_grad(set_to_none=True) + + with autocast(enabled=True): + + reconstruction, z_mu, z_sigma = generator(images) + timer.report(f'train batch {train_step} generator forward') + recons_loss = F.l1_loss(reconstruction.float(), images.float()) + timer.report(f'train batch {train_step} recons_loss') + p_loss = perceptual_loss(reconstruction.float(), images.float()) + timer.report(f'train batch {train_step} p_loss') + kl_loss = 0.5 * torch.sum(z_mu.pow(2) + z_sigma.pow(2) - torch.log(z_sigma.pow(2)) - 1, dim=[1, 2, 3]) + kl_loss = torch.sum(kl_loss) / kl_loss.shape[0] + timer.report(f'train batch {train_step} kl_loss') + loss_g = recons_loss + (kl_weight * kl_loss) + (perceptual_weight * p_loss) + timer.report(f'train batch {train_step} loss_g (1)') + + if epoch > generator_warm_up_n_epochs: # Train generator for n epochs on reconstruction, KL, and perceptual loss before introducing discriminator loss + + logits_fake = discriminator(reconstruction.contiguous().float())[-1] + timer.report(f'train batch {train_step} logits_fake from discriminator') + generator_loss = adv_loss(logits_fake, target_is_real=True, for_discriminator=False) + timer.report(f'train batch {train_step} generator_loss (adv_loss)') + loss_g += adv_weight * generator_loss + timer.report(f'train batch {train_step} loss_g (2)') + + scaler_g.scale(loss_g).backward() + scaler_g.step(optimizer_g) + scaler_g.update() + timer.report(f'train batch {train_step} generator backward') + + # TRAIN DISCRIMINATOR + + if epoch > generator_warm_up_n_epochs: # Train generator for n epochs before starting discriminator training + + optimizer_d.zero_grad(set_to_none=True) + + with autocast(enabled=True): + + logits_fake = discriminator(reconstruction.contiguous().detach())[-1] + timer.report(f'train batch {train_step} discriminator forward (fake)') + loss_d_fake = adv_loss(logits_fake, target_is_real=False, for_discriminator=True) + timer.report(f'train batch {train_step} loss_d_fake') + logits_real = discriminator(images.contiguous().detach())[-1] + timer.report(f'train batch {train_step} discriminator forward (real)') + loss_d_real = adv_loss(logits_real, target_is_real=True, for_discriminator=True) + timer.report(f'train batch {train_step} loss_d_real') + discriminator_loss = (loss_d_fake + loss_d_real) * 0.5 + timer.report(f'train batch {train_step} discriminator_loss') + loss_d = adv_weight * discriminator_loss + timer.report(f'train batch {train_step} loss_d') + + scaler_d.scale(loss_d).backward() + scaler_d.step(optimizer_d) + scaler_d.update() + timer.report(f'train batch {train_step} discriminator backward') + + # Reduce metrics accross nodes + metrics["train"].update({"train_images_seen":len(images), "epoch_loss":recons_loss.item()}) + if epoch > generator_warm_up_n_epochs: + metrics["train"].update({"gen_epoch_loss":generator_loss.item(), "disc_epoch_loss":discriminator_loss.item()}) + metrics["train"].reduce_and_reset_local() + + timer.report(f'train batch {train_step} metrics update') + + recons_loss = metrics["train"].agg[metrics["train"].map["epoch_loss"]] / metrics["train"].agg[metrics["train"].map["train_images_seen"]] + gen_loss = metrics["train"].agg[metrics["train"].map["gen_epoch_loss"]] / metrics["train"].agg[metrics["train"].map["train_images_seen"]] + disc_loss = metrics["train"].agg[metrics["train"].map["disc_epoch_loss"]] / metrics["train"].agg[metrics["train"].map["train_images_seen"]] + print("Epoch [{}] Step [{}/{}] :: recons_loss: {:,.3f}, gen_loss: {:,.3f}, disc_loss: {:,.3f}".format(epoch, train_step+1, total_steps, recons_loss, gen_loss, disc_loss)) + + ## Checkpointing + print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") + train_sampler.advance(len(images)) + train_step = train_sampler.progress // train_loader.batch_size + + if train_step == total_steps: + metrics["train"].end_epoch() + + if utils.is_main_process() and train_step % 1 == 0: # Checkpointing every batch + writer = SummaryWriter(log_dir=tb_path) + writer.add_scalar("recons_loss", recons_loss, step) + writer.add_scalar("gen_loss", recons_loss, step) + writer.add_scalar("disc_loss", recons_loss, step) + writer.flush() + writer.close() + checkpoint = { + # Universals + "args": args, + "epoch": epoch, + # State variables + "generator": generator.module.state_dict(), + "discriminator": discriminator.module.state_dict(), + "optimizer_g": optimizer_g.state_dict(), + "optimizer_d": optimizer_d.state_dict(), + "scaler_g": scaler_g.state_dict(), + "scaler_d": scaler_d.state_dict(), + "train_sampler": train_sampler.state_dict(), + "val_sampler": val_sampler.state_dict(), + # Metrics + "metrics": metrics, + } + timer = atomic_torch_save(checkpoint, args.resume, timer) + + return generator, timer, metrics + + +def evaluate_generator( + args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, + scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, + metrics + ): + + generator.eval() + + val_step = val_sampler.progress // val_loader.batch_size + total_steps = int(len(val_sampler) / val_loader.batch_size) + print(f'\nEvaluating / resuming epoch {epoch} from eval step {val_step}\n') + + with torch.no_grad(): + for batch in val_loader: + + images = batch["image"].to(device) + timer.report(f'eval batch {val_step} to device') + + with autocast(enabled=True): + + reconstruction, _, _ = generator(images) + timer.report(f'eval batch {val_step} forward') + recons_loss = F.l1_loss(images.float(), reconstruction.float()) + timer.report(f'eval batch {val_step} recons_loss') + + metrics["val"].update({"val_images_seen": len(images), "val_loss": recons_loss.item()}) + metrics["val"].reduce_and_reset_local() + + timer.report(f'eval batch {val_step} metrics update') + + ## Checkpointing + print(f"Saving checkpoint at epoch {epoch} val batch {val_step}") + val_sampler.advance(len(images)) + val_step = val_sampler.progress // val_loader.batch_size + + if val_step == total_steps: + metrics["val"].end_epoch() + + if utils.is_main_process() and val_step % 1 == 0: # Checkpointing every batch + print(f"Saving checkpoint at epoch {epoch} train batch {val_step}") + checkpoint = { + # Universals + "args": args, + "epoch": epoch, + # State variables + "generator": generator.module.state_dict(), + "discriminator": discriminator.module.state_dict(), + "optimizer_g": optimizer_g.state_dict(), + "optimizer_d": optimizer_d.state_dict(), + "scaler_g": scaler_g.state_dict(), + "scaler_d": scaler_d.state_dict(), + "train_sampler": train_sampler.state_dict(), + "val_sampler": val_sampler.state_dict(), + # Metrics + "metrics": metrics, + } + timer = atomic_torch_save(checkpoint, args.resume, timer) + + val_loss = metrics["val"].agg[metrics["val"].map["val_loss"]] / metrics["val"].agg[metrics["val"].map["val_images_seen"]] + if utils.is_main_process(): + writer = SummaryWriter(log_dir=tb_path) + writer.add_scalar("val", val_loss, epoch) + writer.flush() + writer.close() + print(f"Epoch {epoch} val loss: {val_loss:.4f}") + + return timer, metrics + + + +## -- DIFFUSION MODEL - ## + +def train_diffusion_one_epoch( + args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, + train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer + ): + + unet.train() + generator.eval() + + train_step = train_sampler.progress // train_loader.batch_size + total_steps = int(len(train_sampler) / train_loader.batch_size) + print(f'\nTraining / resuming epoch {epoch} from training step {train_step}\n') + + for step, batch in enumerate(train_loader): + + images = batch["image"].to(device) + timer.report(f'train batch {train_step} to device') + + optimizer_u.zero_grad(set_to_none=True) + + with autocast(enabled=True): + + z_mu, z_sigma = generator.encode(images) + timer.report(f'train batch {train_step} generator encoded') + z = generator.sampling(z_mu, z_sigma) + timer.report(f'train batch {train_step} generator sampling') + noise = torch.randn_like(z).to(device) + timer.report(f'train batch {train_step} noise') + timesteps = torch.randint(0, inferer.scheduler.num_train_timesteps, (z.shape[0],), device=z.device).long() + timer.report(f'train batch {train_step} timesteps') + noise_pred = inferer(inputs=images, diffusion_model=unet, noise=noise, timesteps=timesteps, autoencoder_model=generator) + timer.report(f'train batch {train_step} noise_pred') + loss = F.mse_loss(noise_pred.float(), noise.float()) + timer.report(f'train batch {train_step} loss') + + scaler_u.scale(loss).backward() + scaler_u.step(optimizer_u) + scaler_u.update() + timer.report(f'train batch {train_step} unet backward') + + epoch_loss += loss.item() + train_images_seen += len(images) + recons_loss = epoch_loss / train_images_seen + print("Epoch [{}] Step [{}/{}] :: recons_loss: {:,.3f}".format(epoch, train_step+1, total_steps, recons_loss)) + + ## Checkpointing + print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") + train_sampler.advance(len(images)) + train_step = train_sampler.progress // train_loader.batch_size + if utils.is_main_process() and train_step % 1 == 0: # Checkpointing every batch + checkpoint = { + # Universals + "args": args, + "epoch": epoch, + + # State variables + "unet": unet.module.state_dict(), + "optimizer_u": optimizer_u.state_dict(), + "scaler_u": scaler_u.state_dict(), + "train_sampler": train_sampler.state_dict(), + "val_sampler": val_sampler.state_dict(), + + # Evaluation metrics + "train_images_seen": train_images_seen, + "val_images_seen": val_images_seen, + "epoch_loss": epoch_loss, + "val_loss": val_loss, + } + timer = atomic_torch_save(checkpoint, args.resume, timer) + + return unet, timer + + +def evaluate_diffusion( + args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, + train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer + ): + + unet.eval() + + val_step = val_sampler.progress // val_loader.batch_size + print(f'\nEvaluating / resuming epoch {epoch} from training step {val_step}\n') + + with torch.no_grad(): + for step, batch in enumerate(val_loader): + + images = batch["image"].to(device) + timer.report(f'eval batch {val_step} to device') + + with autocast(enabled=True): + + z_mu, z_sigma = generator.encode(images) + timer.report(f'eval batch {val_step} generator encoded') + z = generator.sampling(z_mu, z_sigma) + timer.report(f'eval batch {val_step} generator sampling') + noise = torch.randn_like(z).to(device) + timer.report(f'eval batch {val_step} noise') + timesteps = torch.randint(0, inferer.scheduler.num_train_timesteps, (z.shape[0],), device=z.device).long() + timer.report(f'eval batch {val_step} timesteps') + noise_pred = inferer(inputs=images,diffusion_model=unet,noise=noise,timesteps=timesteps,autoencoder_model=generator) + timer.report(f'eval batch {val_step} noise_pred') + loss = F.mse_loss(noise_pred.float(), noise.float()) + timer.report(f'eval batch {val_step} loss') + + val_loss += loss.item() + val_images_seen += len(images) + timer.report(f'eval batch {val_step} metrics update') + + ## Checkpointing + print(f"Saving checkpoint at epoch {epoch} val batch {val_step}") + val_sampler.advance(len(images)) + if utils.is_main_process() and val_step % 1 == 0: # Checkpointing every batch + print(f"Saving checkpoint at epoch {epoch} train batch {val_step}") + checkpoint = { + # Universals + "args": args, + "epoch": epoch, + + # State variables + "unet": unet.module.state_dict(), + "optimizer_u": optimizer_u.state_dict(), + "scaler_u": scaler_u.state_dict(), + "train_sampler": train_sampler.state_dict(), + "val_sampler": val_sampler.state_dict(), + + # Evaluation metrics + "train_images_seen": train_images_seen, + "val_images_seen": val_images_seen, + "epoch_loss": epoch_loss, + "val_loss": val_loss, + } + timer = atomic_torch_save(checkpoint, args.resume, timer) + + val_loss /= val_images_seen + print(f"Epoch {epoch} diff val loss: {val_loss:.4f}") + + return timer diff --git a/monai/brats_mri_2d_0/prep.py b/monai/brats_mri_2d_0/prep.py new file mode 100644 index 00000000..80c5c480 --- /dev/null +++ b/monai/brats_mri_2d_0/prep.py @@ -0,0 +1,20 @@ +# Download the toy dataset from MONAI +print("Downloadning BraTS2016/17") +from monai.apps import DecathlonDataset +from generative.losses.perceptual import PerceptualLoss + +# _ = DecathlonDataset(root_dir="/mnt/Datasets/Open-Datasets/MONAI", task="Task01_BrainTumour", section="training", download=True) +_ = DecathlonDataset(root_dir="/mnt/.node1/Open-Datsets/MONAI", task="Task01_BrainTumour", section="training", download=True) + +perceptual_loss = PerceptualLoss( + spatial_dims=2, network_type="resnet50", pretrained=True, #ImageNet pretrained weights used +) + +# # Download the bigger dataset from Synapse +# print("Downloadning BraTS2023") +# import synapseclient +# syn = synapseclient.Synapse() +# syn.login('adam_peaston','AXXXXXXXXX2') +# syn51514132 = syn.get(entity='syn51514132', downloadFile=True, downloadLocation="/mnt/Datasets/strongcompute_adam/MONAI", ifcollision="overwrite.local") +# filepath = syn51514132.path +# print(f"BraTS2023-GLI downloaded to {filepath}") \ No newline at end of file diff --git a/monai/brats_mri_2d_0/train_cycling_diff.py b/monai/brats_mri_2d_0/train_cycling_diff.py new file mode 100644 index 00000000..c1665320 --- /dev/null +++ b/monai/brats_mri_2d_0/train_cycling_diff.py @@ -0,0 +1,285 @@ +from cycling_utils import Timer + +timer = Timer() +timer.report('importing Timer') + +import os + +import matplotlib.pyplot as plt +import numpy as np +import torch +import torch.nn.functional as F +from monai import transforms +from monai.apps import DecathlonDataset +from monai.config import print_config +from monai.data import DataLoader, Dataset +from monai.utils import first, set_determinism +from torch.cuda.amp import GradScaler, autocast +from pathlib import Path +from tqdm import tqdm + +from generative.inferers import LatentDiffusionInferer +from generative.losses.adversarial_loss import PatchAdversarialLoss +from generative.losses.perceptual import PerceptualLoss +from generative.networks.nets import AutoencoderKL, DiffusionModelUNet, PatchDiscriminator +from generative.networks.schedulers import DDPMScheduler + +from cycling_utils import InterruptableDistributedSampler, Timer +# from loops import train_generator_one_epoch, evaluate_generator +from loops import train_diffusion_one_epoch, evaluate_diffusion +import utils + +def get_args_parser(add_help=True): + import argparse + + parser = argparse.ArgumentParser(description="Latent Diffusion Model Training", add_help=add_help) + + parser.add_argument("--resume", type=str, help="path of checkpoint", required=True) # for checkpointing + parser.add_argument("--gen-load-path", type=str, help="path of checkpoint", dest="gen_load_path") # for checkpointing + parser.add_argument("--prev-resume", default=None, help="path of previous job checkpoint for strong fail resume", dest="prev_resume") # for checkpointing + # parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") + parser.add_argument("--data-path", default="/mnt/Datasets/Open-Datasets/MONAI", type=str, help="dataset path", dest="data_path") + parser.add_argument("--device", default="cuda", type=str, help="device (Use cuda or cpu Default: cuda)") + parser.add_argument("--start-epoch", default=0, type=int, metavar="N", help="start epoch") + # parser.add_argument("-b", "--batch-size", default=8, type=int, help="images per gpu, the total batch size is $NGPU x batch_size", dest="batch_size") + # parser.add_argument("--epochs", default=30, type=int, metavar="N", help="number of total epochs to run") + # parser.add_argument("--print-freq", default=1, type=int, help="print frequency") + parser.add_argument("--dist-url", default="env://", type=str, help="url used to set up distributed training") + parser.add_argument("-j", "--workers", default=16, type=int, metavar="N", help="number of data loading workers (default: 16)") + + return parser + +timer.report('importing everything else') + +def main(args, timer): + + # ## Distributed training prelims + # if args.output_dir: + # utils.mkdir(args.output_dir) + + utils.init_distributed_mode(args) # Sets args.distributed among other things + assert args.distributed # don't support cycling when not distributed for simplicity + + device = torch.device(args.device) + + timer.report('preliminaries') + + # Maybe this will work? + set_determinism(42) + + channel = 0 # 0 = Flair + assert channel in [0, 1, 2, 3], "Choose a valid channel" + ## NEED TO COME BACK AND ALIGN WITH BRATS CONFIG + train_transforms = transforms.Compose([ + transforms.LoadImaged(keys=["image", "label"], image_only=False), # image_only current default will change soon, so including explicitly + transforms.EnsureChannelFirstd(keys=["image", "label"]), + transforms.Lambdad(keys=["image"], func=lambda x: x[channel, None, :, :, :]), + transforms.EnsureTyped(keys=["image", "label"]), + transforms.Orientationd(keys=["image", "label"], axcodes="RAS"), + transforms.Spacingd(keys=["image", "label"], pixdim=(3.0, 3.0, 2.0), mode=("bilinear", "nearest")), + transforms.CenterSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 44)), + transforms.ScaleIntensityRangePercentilesd(keys="image", lower=0, upper=99.5, b_min=0, b_max=1), + transforms.RandSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 1), random_size=False, random_centre=True), # Eeach of the 44 slices will be randomly sampled. + transforms.Lambdad(keys=["image", "label"], func=lambda x: x.squeeze(-1)), + transforms.CopyItemsd(keys=["label"], times=1, names=["slice_label"]), + transforms.Lambdad(keys=["slice_label"], func=lambda x: 1.0 if x.sum() > 0 else 0.0), + ]) + + train_ds = DecathlonDataset( + root_dir=args.data_path, task="Task01_BrainTumour", section="training", cache_rate=0.0, + num_workers=4, download=False, seed=0, transform=train_transforms, + ) + val_ds = DecathlonDataset( + root_dir=args.data_path, task="Task01_BrainTumour", section="validation", cache_rate=0.0, + num_workers=4, download=False, seed=0, transform=train_transforms, + ) + + timer.report('build datasets') + + train_sampler = InterruptableDistributedSampler(train_ds) + val_sampler = InterruptableDistributedSampler(val_ds) + + timer.report('build samplers') + + train_loader = DataLoader(train_ds, batch_size=2, sampler=train_sampler, num_workers=args.workers) + val_loader = DataLoader(val_ds, batch_size=1, sampler=val_sampler, num_workers=args.workers) + check_data = first(train_loader) # Used later + + timer.report('build dataloaders') + + # Auto-encoder definition + generator = AutoencoderKL( + spatial_dims=2, in_channels=1, out_channels=1, num_channels=(128, 128, 256), latent_channels=3, + num_res_blocks=2, attention_levels=(False, False, False),with_encoder_nonlocal_attn=False, + with_decoder_nonlocal_attn=False, + ) + # saved_generator_checkpoint = torch.load("/output_brats_mri_2d_gen/exp_1645/checkpoint.isc", map_location="cpu") + saved_generator_checkpoint = torch.load(args.gen_load_path, map_location="cpu") + generator.load_state_dict(saved_generator_checkpoint["generator"]) + generator = generator.to(device) + + timer.report('generator to device') + + # # Discriminator definition + # discriminator = PatchDiscriminator( + # spatial_dims=2, num_layers_d=3, num_channels=64, + # in_channels=1, out_channels=1 + # ) + # discriminator = discriminator.to(device) + + # timer.report('discriminator to device') + + # Diffusion model (unet) + unet = DiffusionModelUNet( + spatial_dims=2, in_channels=3, out_channels=3, num_res_blocks=2, + num_channels=(128, 256, 512),attention_levels=(False, True, True), + num_head_channels=(0, 256, 512), + ) + unet = unet.to(device) + + timer.report('unet to device') + + # # Autoencoder loss functions + # adv_loss = PatchAdversarialLoss(criterion="least_squares") + # perceptual_loss = PerceptualLoss(spatial_dims=2, network_type="alex") + # perceptual_loss.to(device) + + # timer.report('loss functions') + + # Prepare for distributed training + # generator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) + # discriminator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) + unet = torch.nn.SyncBatchNorm.convert_sync_batchnorm(unet) + + # generator_without_ddp = generator + # discriminator_without_ddp = discriminator + unet_without_ddp = unet + if args.distributed: + # generator = torch.nn.parallel.DistributedDataParallel(generator, device_ids=[args.gpu], find_unused_parameters=True) + # discriminator = torch.nn.parallel.DistributedDataParallel(discriminator, device_ids=[args.gpu], find_unused_parameters=True) + unet = torch.nn.parallel.DistributedDataParallel(unet, device_ids=[args.gpu], find_unused_parameters=True) + # generator_without_ddp = generator.module + # discriminator_without_ddp = discriminator.module + unet_without_ddp = unet.module + + timer.report('unet prepped for distribution') + + # Optimizers + # optimizer_g = torch.optim.Adam(generator_without_ddp.parameters(), lr=1e-4) + # optimizer_d = torch.optim.Adam(discriminator_without_ddp.parameters(), lr=5e-4) + optimizer_u = torch.optim.Adam(unet_without_ddp.parameters(), lr=1e-4) + + timer.report('optimizers') + + # For mixed precision training + # scaler_g = GradScaler() + # scaler_d = GradScaler() + scaler_u = GradScaler() + + timer.report('grad scalers') + + # Init tracking metrics + train_images_seen = 0 + val_images_seen = 0 + epoch_loss = 0 + val_loss = 0 + + # RETRIEVE GENERATOR CHECKPOINT FROM PREVIOUS JOB + + # RETRIEVE CHECKPOINT + Path(args.resume).parent.mkdir(parents=True, exist_ok=True) + checkpoint = None + if args.resume and os.path.isfile(args.resume): # If we're resuming... + checkpoint = torch.load(args.resume, map_location="cpu") + elif args.prev_resume and os.path.isfile(args.prev_resume): + checkpoint = torch.load(args.prev_resume, map_location="cpu") + if checkpoint is not None: + args.start_epoch = checkpoint["epoch"] + unet_without_ddp.load_state_dict(checkpoint["unet"]) + optimizer_u.load_state_dict(checkpoint["optimizer_u"]) + scaler_u.load_state_dict(checkpoint["scaler_u"]) + train_sampler.load_state_dict(checkpoint["train_sampler"]) + val_sampler.load_state_dict(checkpoint["val_sampler"]) + train_images_seen = checkpoint["train_images_seen"] + val_images_seen = checkpoint["val_images_seen"] + # Metrics + train_images_seen = checkpoint["train_images_seen"] + val_images_seen = checkpoint["val_images_seen"] + epoch_loss = checkpoint["epoch_loss"] + val_loss = checkpoint["val_loss"] + + timer.report('checkpoint retrieval') + + # ## -- TRAINING THE AUTO-ENCODER - ## + + # n_gen_epochs = 100 + # gen_val_interval = 1 + + # for epoch in range(args.start_epoch, n_gen_epochs): + + # print('\n') + # print(f"EPOCH :: {epoch}") + # print('\n') + + # with train_sampler.in_epoch(epoch): + # timer = Timer("Start training") + # generator, timer = train_generator_one_epoch( + # args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, + # scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, + # train_images_seen, val_images_seen, epoch_loss, gen_epoch_loss, disc_epoch_loss, val_loss + # ) + # timer.report(f'training generator for epoch {epoch}') + + # if epoch % gen_val_interval == 0: # Eval every epoch + # with val_sampler.in_epoch(epoch): + # timer = Timer("Start evaluation") + # timer = evaluate_generator( + # args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, + # scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, + # train_images_seen, val_images_seen, epoch_loss, gen_epoch_loss, disc_epoch_loss, val_loss + # ) + # timer.report(f'evaluating generator for epoch {epoch}') + + + ## -- TRAINING THE DIFFUSION MODEL - ## + + n_diff_epochs = 200 + diff_val_interval = 1 + + # Prepare LatentDiffusionInferer + scheduler = DDPMScheduler(num_train_timesteps=1000, schedule="linear_beta", beta_start=0.0015, beta_end=0.0195) + with torch.no_grad(): + with autocast(enabled=True): + z = generator.encode_stage_2_inputs(check_data["image"].to(device)) + scale_factor = 1 / torch.std(z) + inferer = LatentDiffusionInferer(scheduler, scale_factor=scale_factor) + + timer.report('building inferer') + + for epoch in range(args.start_epoch, n_diff_epochs): + + print('\n') + print(f"EPOCH :: {epoch}") + print('\n') + + with train_sampler.in_epoch(epoch): + timer = Timer("Start training") + unet, timer = train_diffusion_one_epoch( + args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, + train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer + ) + timer.report(f'training unet for epoch {epoch}') + + if epoch % diff_val_interval == 0: + with val_sampler.in_epoch(epoch): + timer = Timer("Start evaluation") + timer = evaluate_diffusion( + args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, + train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer + ) + timer.report(f'evaluating unet for epoch {epoch}') + + +if __name__ == "__main__": + args = get_args_parser().parse_args() + main(args, timer) diff --git a/monai/brats_mri_2d_0/train_cycling_gen_0.py b/monai/brats_mri_2d_0/train_cycling_gen_0.py new file mode 100644 index 00000000..d8db2b91 --- /dev/null +++ b/monai/brats_mri_2d_0/train_cycling_gen_0.py @@ -0,0 +1,275 @@ +from cycling_utils import Timer + +timer = Timer() +timer.report('importing Timer') + +import os + +# import matplotlib.pyplot as plt +# import numpy as np +import torch +import torch.distributed as dist +# import torch.nn.functional as F +from monai import transforms +from monai.apps import DecathlonDataset +# from monai.config import print_config +from monai.data import DataLoader# , Dataset +from monai.utils import first, set_determinism +from torch.cuda.amp import GradScaler# , autocast +from pathlib import Path +# from tqdm import tqdm + +# from generative.inferers import LatentDiffusionInferer +from generative.losses.adversarial_loss import PatchAdversarialLoss +from generative.losses.perceptual import PerceptualLoss +from generative.networks.nets import AutoencoderKL, PatchDiscriminator # , DiffusionModelUNet +# from generative.networks.schedulers import DDPMScheduler + +from cycling_utils import InterruptableDistributedSampler, Timer +from loops_0 import train_generator_one_epoch, evaluate_generator +# from loops import train_diffusion_one_epoch, evaluate_diffusion +import utils + +def get_args_parser(add_help=True): + import argparse + + parser = argparse.ArgumentParser(description="Latent Diffusion Model Training", add_help=add_help) + + parser.add_argument("--resume", type=str, help="path of checkpoint", required=True) # for checkpointing + parser.add_argument("--prev-resume", default=None, help="path of previous job checkpoint for strong fail resume", dest="prev_resume") # for checkpointing + # parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") + parser.add_argument("--data-path", default="/mnt/Datasets/Open-Datasets/MONAI", type=str, help="dataset path", dest="data_path") + parser.add_argument("--device", default="cuda", type=str, help="device (Use cuda or cpu Default: cuda)") + parser.add_argument("--start-epoch", default=0, type=int, metavar="N", help="start epoch") + # parser.add_argument("-b", "--batch-size", default=8, type=int, help="images per gpu, the total batch size is $NGPU x batch_size", dest="batch_size") + # parser.add_argument("--epochs", default=30, type=int, metavar="N", help="number of total epochs to run") + # parser.add_argument("--print-freq", default=1, type=int, help="print frequency") + parser.add_argument("--dist-url", default="env://", type=str, help="url used to set up distributed training") + parser.add_argument("-j", "--workers", default=16, type=int, metavar="N", help="number of data loading workers (default: 16)") + + return parser + +timer.report('importing everything else') + +class MetricsTracker: + def __init__(self, metric_names): + self.map = {n:i for i,n in enumerate(metric_names)} + self.local = torch.zeros(len(metric_names), dtype=torch.float16, requires_grad=False, device='cuda') + self.agg = torch.zeros(len(metric_names), dtype=torch.float16, requires_grad=False, device='cuda') + self.epoch_reports = [] + + def update(self, metrics: dict): + for n,v in metrics.items(): + self.local[self.map[n]] += v + + def reduce_and_reset_local(self): + # Reduce over all nodes, add that to local store, and reset local + dist.all_reduce(self.local, op=dist.ReduceOp.SUM) + self.agg += self.local + self.local = torch.zeros(len(self.local), dtype=torch.float16, requires_grad=False, device='cuda') + + def end_epoch(self): + self.epoch_reports.append(self.agg) + self.local = torch.zeros(len(self.local), dtype=torch.float16, requires_grad=False, device='cuda') + self.agg = torch.zeros(len(self.local), dtype=torch.float16, requires_grad=False, device='cuda') + + def to(self, device): + self.local = self.local.to(device) + self.agg = self.agg.to(device) + + +def main(args, timer): + + # ## Distributed training prelims + # if args.output_dir: + # utils.mkdir(args.output_dir) + + utils.init_distributed_mode(args) # Sets args.distributed among other things + assert args.distributed # don't support cycling when not distributed for simplicity + + device = torch.device(args.device) + + timer.report('preliminaries') + + # Maybe this will work? + set_determinism(42) + + channel = 0 # 0 = Flair + assert channel in [0, 1, 2, 3], "Choose a valid channel" + preprocessing_transform = transforms.Compose([ + transforms.LoadImaged(keys="image", image_only=False), # image_only current default will change soon, so including explicitly + transforms.EnsureChannelFirstd(keys="image"), + transforms.Lambdad(keys="image", func=lambda x: x[channel, :, :, :]), + transforms.AddChanneld(keys="image"), + transforms.EnsureTyped(keys="image"), + transforms.Orientationd(keys="image", axcodes="RAS"), + transforms.CenterSpatialCropd(keys="image", roi_size=(240, 240, 100)), + transforms.ScaleIntensityRangePercentilesd(keys="image", lower=0, upper=100, b_min=0, b_max=1), + ]) + + crop_transform = transforms.Compose([ + transforms.DivisiblePadd(keys="image", k=[4,4,1]), + # transforms.RandSpatialCropSamplesd(keys="image", roi_size=(240, 240, 1), random_size=False, num_samples=26), + transforms.RandSpatialCropd(keys="image", roi_size=(240, 240, 1), random_size=False), # Each of the 100 slices will be randomly sampled. + transforms.SqueezeDimd(keys="image", dim=3), + transforms.RandFlipd(keys="image", prob=0.5, spatial_axis=0), + transforms.RandFlipd(keys="image", prob=0.5, spatial_axis=1), + ]) + + preprocessing = transforms.Compose([preprocessing_transform, crop_transform]) + + train_ds = DecathlonDataset( + root_dir=args.data_path, task="Task01_BrainTumour", section="training", cache_rate=0.0, + num_workers=8, download=False, seed=0, transform=preprocessing, + ) + val_ds = DecathlonDataset( + root_dir=args.data_path, task="Task01_BrainTumour", section="validation", cache_rate=0.0, + num_workers=8, download=False, seed=0, transform=preprocessing, + ) + + timer.report('build datasets') + + train_sampler = InterruptableDistributedSampler(train_ds) + val_sampler = InterruptableDistributedSampler(val_ds) + + timer.report('build samplers') + + # Original trainer had batch size = 26. Using 9 nodes x batch size 3 = eff batch size = 27 + train_loader = DataLoader(train_ds, batch_size=3, sampler=train_sampler, num_workers=1) + val_loader = DataLoader(val_ds, batch_size=1, sampler=val_sampler, num_workers=1) + # check_data = first(train_loader) # Used later + + timer.report('build dataloaders') + + # Auto-encoder definition + generator = AutoencoderKL( + spatial_dims=2, in_channels=1, out_channels=1, num_channels=(64, 128, 256), + latent_channels=1, num_res_blocks=2, norm_num_groups=32, norm_eps=1e-06, + attention_levels=(False, False, False), with_encoder_nonlocal_attn=True, + with_decoder_nonlocal_attn=True, + ) + generator = generator.to(device) + + timer.report('generator to device') + + # Discriminator definition + discriminator = PatchDiscriminator( + spatial_dims=2, num_layers_d=3, num_channels=32, + in_channels=1, out_channels=1, norm="INSTANCE" + ) + discriminator = discriminator.to(device) + + timer.report('discriminator to device') + + # # Diffusion model (unet) + # unet = DiffusionModelUNet( + # spatial_dims=2, in_channels=3, out_channels=3, num_res_blocks=2, + # num_channels=(128, 256, 512),attention_levels=(False, True, True), + # num_head_channels=(0, 256, 512), + # ) + # unet = unet.to(device) + + # timer.report('unet to device') + + # Autoencoder loss functions + adv_loss = PatchAdversarialLoss(criterion="least_squares") + perceptual_loss = PerceptualLoss( + spatial_dims=2, network_type="resnet50", pretrained=True, #ImageNet pretrained weights used + ) + perceptual_loss.to(device) + + timer.report('loss functions') + + # Prepare for distributed training + generator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) + discriminator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) + # unet = torch.nn.SyncBatchNorm.convert_sync_batchnorm(unet) + + generator_without_ddp = generator + discriminator_without_ddp = discriminator + # unet_without_ddp = unet + if args.distributed: + generator = torch.nn.parallel.DistributedDataParallel(generator, device_ids=[args.gpu], find_unused_parameters=True) # find_unused_parameters necessary for monai training + discriminator = torch.nn.parallel.DistributedDataParallel(discriminator, device_ids=[args.gpu], find_unused_parameters=True) # find_unused_parameters necessary for monai training + # unet = torch.nn.parallel.DistributedDataParallel(unet, device_ids=[args.gpu], find_unused_parameters=True) + generator_without_ddp = generator.module + discriminator_without_ddp = discriminator.module + # unet_without_ddp = unet.module + + timer.report('models prepped for distribution') + + # Optimizers + optimizer_g = torch.optim.Adam(generator_without_ddp.parameters(), lr=5e-5) + optimizer_d = torch.optim.Adam(discriminator_without_ddp.parameters(), lr=5e-5) + # optimizer_u = torch.optim.Adam(unet_without_ddp.parameters(), lr=1e-4) + + timer.report('optimizers') + + # For mixed precision training + scaler_g = GradScaler() + scaler_d = GradScaler() + # scaler_u = GradScaler() + + timer.report('grad scalers') + + # Init metric tracker + train_metrics = MetricsTracker(["train_images_seen", "epoch_loss", "gen_epoch_loss", "disc_epoch_loss"]) + val_metrics = MetricsTracker(["val_images_seen", "val_loss"]) + metrics = {'train': train_metrics, 'val': val_metrics} + + # RETRIEVE CHECKPOINT + Path(args.resume).parent.mkdir(parents=True, exist_ok=True) + checkpoint = None + if args.resume and os.path.isfile(args.resume): # If we're resuming... + checkpoint = torch.load(args.resume, map_location="cpu") + elif args.prev_resume and os.path.isfile(args.prev_resume): + checkpoint = torch.load(args.prev_resume, map_location="cpu") + if checkpoint is not None: + args.start_epoch = checkpoint["epoch"] + generator_without_ddp.load_state_dict(checkpoint["generator"]) + discriminator_without_ddp.load_state_dict(checkpoint["discriminator"]) + optimizer_g.load_state_dict(checkpoint["optimizer_g"]) + optimizer_d.load_state_dict(checkpoint["optimizer_d"]) + scaler_g.load_state_dict(checkpoint["scaler_g"]) + scaler_d.load_state_dict(checkpoint["scaler_d"]) + train_sampler.load_state_dict(checkpoint["train_sampler"]) + val_sampler.load_state_dict(checkpoint["val_sampler"]) + # Metrics + metrics = checkpoint["metrics"] + metrics["train"].to(device) + metrics["val"].to(device) + + timer.report('checkpoint retrieval') + + ## -- TRAINING THE AUTO-ENCODER - ## + + n_gen_epochs = 200 + gen_val_interval = 1 + + for epoch in range(args.start_epoch, n_gen_epochs): + + print('\n') + print(f"EPOCH :: {epoch}") + print('\n') + + with train_sampler.in_epoch(epoch): + timer = Timer("Start training") + generator, timer, metrics = train_generator_one_epoch( + args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, + scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, metrics + ) + timer.report(f'training generator for epoch {epoch}') + + if epoch % gen_val_interval == 0: # Eval every epoch + with val_sampler.in_epoch(epoch): + timer = Timer("Start evaluation") + timer, metrics = evaluate_generator( + args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, + scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, metrics + ) + timer.report(f'evaluating generator for epoch {epoch}') + + +if __name__ == "__main__": + args = get_args_parser().parse_args() + main(args, timer) diff --git a/monai/brats_mri_2d_0/utils.py b/monai/brats_mri_2d_0/utils.py new file mode 100644 index 00000000..1b268b82 --- /dev/null +++ b/monai/brats_mri_2d_0/utils.py @@ -0,0 +1,72 @@ +import torch, os, errno +import torch.distributed as dist + +def mkdir(path): + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + +def setup_for_distributed(is_master): + """ + This function disables printing when not in master process + """ + import builtins as __builtin__ + + builtin_print = __builtin__.print + + def print(*args, **kwargs): + force = kwargs.pop("force", False) + if is_master or force: + builtin_print(*args, **kwargs) + + __builtin__.print = print + +def init_distributed_mode(args): + if "RANK" in os.environ and "WORLD_SIZE" in os.environ: + args.rank = int(os.environ["RANK"]) + args.world_size = int(os.environ["WORLD_SIZE"]) + args.gpu = int(os.environ["LOCAL_RANK"]) + elif "SLURM_PROCID" in os.environ: + args.rank = int(os.environ["SLURM_PROCID"]) + args.gpu = args.rank % torch.cuda.device_count() + else: + print("Not using distributed mode") + args.distributed = False + return + + args.distributed = True + + torch.cuda.set_device(args.gpu) + args.dist_backend = "nccl" + print(f"| distributed init (rank {args.rank}): {args.dist_url}", flush=True) + torch.distributed.init_process_group( + backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank + ) + torch.distributed.barrier() + setup_for_distributed(args.rank == 0) + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True + + +def get_world_size(): + if not is_dist_avail_and_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + if not is_dist_avail_and_initialized(): + return 0 + return dist.get_rank() + + +def is_main_process(): + return get_rank() == 0 \ No newline at end of file diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361389.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361389.0 new file mode 100644 index 0000000000000000000000000000000000000000..c66dc3c9667008bf810abbbbf31a309c30b79af2 GIT binary patch literal 461 zcmeZZfPjCKJmzv?s(HtiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@~75bMjmQzptp+zC)fY2H6U}fM7V^wxQkMg z^Ye=1bMlLeMHv?G+0{*|`X>Wbe&XO%S9_rW!wxdo{DXCY@1iZ@RqRHM5FsC;44{SXw_2wyt_aSam|*ElS(d*O0Q3&l0d QZ@@L6xdy0OvWH0p0E3T<;{X5v literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361390.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361390.0 new file mode 100644 index 0000000000000000000000000000000000000000..cb8d4131a461d18fa23c586f8ebd65de58cbd918 GIT binary patch literal 415 zcmeZZfPjCKJmzvn*z;C@cf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZSrn9VT^?#~ptp+zr{CYLH6U}fM7V^wxQkMg z^Ye=1bMlLeMHv?G*{u*-`c4L_{KUbjvYc-iw?UK(a`AINnpwAMEt11-V0j5sOO_tn7@Yn@x~|YPfkv z$cQs!FI)qfTY##yFD;Hi@uucJxN3CQ0F^u1xxPnnjl{Jh5Z5qaagDNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZiL!AFmWP@f=M-d>8W`Lwe1HPf@Ps<6ZG#Z zbMAww72*=$;!MdbPDUvHy!@SsB2+PB^_y>;A9s4yg4`j%h{YudR(3U6Pg&)l8gAYZ zGU7Df2iJh+7NF|fpyWOjZ~jE6Mt2QR`PuV_T2WjhcI^nnHB4AsgKu?XY literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361393.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361393.0 new file mode 100644 index 0000000000000000000000000000000000000000..67675700629e71807b09e52c3871f3885678799d GIT binary patch literal 461 zcmeZZfPjCKJmzv**z;C@cf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z5k7ymN*-!%ptp+zXZYW(H6U}fM7V^wxQkMg z^Ye=1bMlLeMHv?G*;QS8wnYZ2{KUbjvYcUz+aSsXx%jy_(o^#gYW3P!+yYe18xs2*#hdF-z*VEW2B>`6M%&dWu2H;p1mYSdEUs}_Vi)Im>?ewA Rl5W5?pt%O9+Kfx$CIFVkjO73T literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361394.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361394.0 new file mode 100644 index 0000000000000000000000000000000000000000..7f0ad2699af738ec7093d63ab389a55e2a19be3a GIT binary patch literal 461 zcmeZZfPjCKJmzxd+w)d`cf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zk(RJ6lZToc=~7t2iIagUKXGuXEayk&Z4l*xT>M-d>8W`Lwa;%)+$IZEo1lMR znR7Y|RILz~02gOUW^pn?vB@>&97U*N#_BiUIPG?M)q>n1z=*{q308JXI2~`xK{edG zBV@$6&^GLpm=ljLAYvk*8r8oSob|YCz^{iEs&VaTlc~ z=jRp2=j0a`i!vNnpw#gF;ag4`j%h{YudR(6FGdn)9h8gAYZ zGU8-C3)g_=7NF|QnTrKbyjgkNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z(LPi9R32(>ptp+zXEn!;8j!hKB3wdT+(oI$ z`FX|hIr+uKq6`c8>?R2XXURa7pEx*GmNSYUs$7tZpNk_sH4mXybV=tCS*Y3s{rk$C z3Ib5ILRb$q2=asryY8p^6!+-+beYJ?c{na)$sT7MCPg+07Jx6(NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z5f|=zC=WF^(A!0Vb0gP|8j!hKB3wdT+(oI$ z`FX|hIr+uKq6`c8>;fKa5SD=|KXGuXEN8nQRJkA*KNm-OY92zZ>XYsmS*Y3s{rk$C zfkIHVLRb$q2>m3BM;PLKQPszxl@La@405oK+kz{lH45vrK6`pq{^uA@G+Aa@8bVsS}=l^wrpgS;G6!_7NF zMx1Al!!@9}1*rPF{+qif-mE?WSB>r(pmJyD+kGgmne^}GJoJw3fYCz^{iEs&VaTlc~ z=jRp2=j0a`i!v BfU*Dp literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc13.1453453.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc13.1453453.0 new file mode 100644 index 0000000000000000000000000000000000000000..690d7f733f166bd3cfd929fb69da8549d839fabb GIT binary patch literal 461 zcmeZZfPjCKJmzw)@Zhih?s(HtiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!k_W@fIQUPKyMcb&Q9(fH6U}fM7V^wxQkMg z^Ye=1bMlLeMHv?G*)`=GyUIY7pEx*Gma|zHs$7tZpNk_sH4mY7^Mc4yS*Y3s{rk$C zMj}wPLRb$q2va1Cf~0jl<2p|}IZn>nZ9s?l8oR9^HeY7UBPavmOoxP}RfYaEu?&9)NmL2(V| Q6SxL6*8o+|k9I2u0Ko~24FCWD literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487328.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487328.0 new file mode 100644 index 0000000000000000000000000000000000000000..2f9a2aa7aec0632f4f5fde7165f2f6c9c7d01652 GIT binary patch literal 508 zcmeZZfPjCKJmzxFbmp!8?s(HtiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!hOTUNgir$ptp+zryaw#8j!hKB3wdT+(oI$ z`FX|hIr+uKq6`c8>=rkru9bl*KXGuXET;r3RJkA*KNm-OY92zZB_q##S*Y3s{rk$C zcVKFTxCFR3Q!ar5sel%{xLy zoV)hHHK4f#sG4u%Cm$4VChv!vptp+zrxoM28j!hKB3wdT+(oI$ z`FX|hIr+uKq6`c8?CK7rn9D$wpEx*GmeZ9Ts$7tZpNk_sH4mZI&wJ-%S*Y3s{rk$C z9vo1$LRb$q2={uCt~qLKQPszxl@bX^&Se$Q=TVSX`1|Ww(9D8E-kLhMRYU zj5s$Pglj-^3s7}R=IT8t-fTDoSB>r(pmOG!x96j{hV#Y|h-;XzxW-|L-I1uBpHN&A ReG9Gu%{4&PPYO2Y0RZ48j;{a! literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487330.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487330.0 new file mode 100644 index 0000000000000000000000000000000000000000..c7dc4277a3d646ae82144a3f9d4feffa68907c52 GIT binary patch literal 461 zcmeZZfPjCKJmzw8I`dY4cf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIXbaBQyyw=ptp+z=QD;J$t-rLGBP>#Nv_!D?4|yKQ?ku4L9!y z8F3yu1lNG(7NF{DcD|M<-aK^_t{UAnK;_%kZQqFEnqSwCKwQIw#WfB~?9MftpF(ks f#Vxo7G}i!CyRXrEi{hGV2-WDW0V+S!%H{_E$#tG! literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487332.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487332.0 new file mode 100644 index 0000000000000000000000000000000000000000..2156c28f55f02d45f03aecedfcd5800c40429a71 GIT binary patch literal 508 zcmeZZfPjCKJmzvTx$sthcf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZdH*wYzdY33KyMcb&PNQ}YCz^{iEs&VaTlc~ z=jRp2=j0a`i!v@4L9!y z8F5-4fNMZ=3s7~pipxNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z>9crIAP+S+(A!0VGl*eZ4ai(A5iTJv?xNJ> z{Ji4$oc!WqQHBM4cH7Tf|1JYne&XO%SNnpw&GvZJg4`j%h{YudR(AK#UJH?fYPfkv z$cR(>09*r_TR^I3CV8WH^9@2Zx@&;S6TVBcp}6MT^&=41Fkx|x!xB5+e#gfsuJOGE O*MR04py~;MH@g5kn2@gk literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433694.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433694.0 new file mode 100644 index 0000000000000000000000000000000000000000..0a53c6468459c6702ae8888ea0ca4cc26c6ac636 GIT binary patch literal 461 zcmeZZfPjCKJmzxxd-7I)cf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIkzKat31@)KyMcb&MJ;=H6U}fM7V^wxQkMg z^Ye=1bMlLeMHv?G*-fc%nj`~Ne&XO%Sr(pz<$FpI4!{X6gMS5Z5qaagD`CeHQc-- zWW@RAI9vmoTY#$HzyGI;;?4ah;HuGG162Mj<#!s2YZlx;0&xu!7S}i|v0JHi;vkA^ Rcpkwupt%O9x^2ysV*owhlY{^O literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433696.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433696.0 new file mode 100644 index 0000000000000000000000000000000000000000..1b277af681ed19677a1c1e3fb77158430f378474 GIT binary patch literal 461 zcmeZZfPjCKJmzxV_u#Gm?s(HtiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@}*TjR32(>ptp+z=Sq%kH6U}fM7V^wxQkMg z^Ye=1bMlLeMHv?G*-bY3o*@HOe&XO%S|Q00PL{9GLAsd)&s`SukivQV`N`uCMN zoA{t=g}4N`I8!oNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIi&X6Lmp~wptp+z=TDApH6U}fM7V^wxQkMg z^Ye=1bMlLeMHv?G*}ZskuS^E2{KUbjvYZF_pvncg__;XJQ}YmNALPAyD+^VdpnqSP zvxpz6R)|Z0i!&v&I2ob%^NZkkMW|xN>Nnpw1CDsrg4`j%h{YudR(5_q0#R~M4L9!y z8F2=kglj-^3sCjPei0uOZ}Ojlt44PXP`P=F!UPo8%)fsG;uPt8N9ef)IS4q2$$1pWKU zoV)p*bT97*g7_qn{!OG6m`S)=-sD_(& zgp4>PkHR&exdo`&IFE}B#hVKes?l8oRGu7A--zOxx%ZDiT*HLLH4aPcUX_%*KygjX QBe(`M*8o+&_z=zq0Q};Sb^rhX literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433699.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433699.0 new file mode 100644 index 0000000000000000000000000000000000000000..d5adfef7425e8808c77854238bf5957e577e06af GIT binary patch literal 461 zcmeZZfPjCKJmzw)@Zhcf?s(HtiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!n1GhetD?5f!;0>oTi-HYCz^{iEs&VaTlc~ z=jRp2=j0a`i!vK%z`H6#5WjUYnL6r+~@pEycr{*ElYX2(fl7*^G(7&(D z*~br6E5s$h#hH>>oQzPMQX?j-2vy8j{pK5Iz!9%nkUInzvA86`%FfS6AW9CZ;pQD7 zBhJ|;;2O}}0#wa1FZDZ$H=R$yRinEGsGMnrs62{mX5BvmaSam|*ElS(J9TyHVHDRC RJcMgNa}7}S*4Ia;0Ra4yj)4FG literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326408.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326408.0 new file mode 100644 index 0000000000000000000000000000000000000000..a26724a61f88f0ccf27718ef513426950f921810 GIT binary patch literal 415 zcmeZZfPjCKJmzxBTJctYcf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZnRPAHMjmQzptp+z=ZarjYe42|iEs&VaTlc~ z=jRp2=j0a`i!vs<`$r8lY%KqDBk>vP>t>ypmLVkIp!#?S$**c#5GJNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zd6`qWSRQI_ptp+z=iwh)Ye42|iEs&VaTlc~ z=jRp2=j0a`i!vj(SRju6rqY4tKWR%)Y|G*3v!15BNmq=SlQYAE?pr9)o}BU zkP&C@Hn;{fw*XZy)bdP1@n-W5xN3CQfDG?@RD$A~ITw#WT*HLLH4aPc#LPAZqqyeO QWw-`3*8o+oVxQ^;0EtYHVE_OC literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326410.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326410.0 new file mode 100644 index 0000000000000000000000000000000000000000..02efdbf3f8e3d5d4cebf1ef340262203f5448428 GIT binary patch literal 508 zcmeZZfPjCKJmzvnTk%$Zcf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zd2jnoMjmQzptp+z=aHXVYe42|iEs&VaTlc~ z=jRp2=j0a`i!v>oQzQXZ)50HMW|xN>Nnpwxwd-Mg4`j%h{YudR(2aqw8gAYZ zGU6272G@Y*7NF{eHs&f6Z`SO9t44PXP`TShZ&?)Athjgt;uNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zi84Q%BM&t<(A!0V)AHxm8j!hKB3wdT+(oI$ z`FX|hIr+uKq6`c8?2d#qo|J(qKXGuXENAy$sB%FrelCvm)I5aR2bmFevQV`N`uCMN zXZ?e!72*=$;!MdbPDUvH;hMig5vrK6`pq{^t*u_QAa@8bVsS}=m7U%1(iL)04L9!y z8FBh-higD{3sAMll9+QS-n@uVjqVzt@}Gf9MJTSBbMXkoHB4AsNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z`S(1*Tpns}ptp+z=gA*iYe42|iEs&VaTlc~ z=jRp2=j0a`i!v(3F8HKXGuXEa%(5Q00PL{9GLAsd)&sAG;_1l!dBI(7&(D zdGQ}qtq_+07iUUlaWX=2;KBe_MW|xN>NnpwZMJ&Vg4`j%h{YudR(1h>E|cV-8gAYZ zGUB|s9j*b*EkMAg*D;;u?n~c4fWax=>tG QbOo*f%{4&PZ%W;(086=&JOBUy literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326413.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326413.0 new file mode 100644 index 0000000000000000000000000000000000000000..78fec76651295e8ba46367dac24ab10fb2dd3a07 GIT binary patch literal 461 zcmeZZfPjCKJmzv{S@BkXcf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIn;OjhCI~VKyMcb&UZhy)_}~_65$f!;x0-} z&d)23&&e+?7G+q#XBT&7r?CuF`H6#5WjRm&fhrf|;^*Q>Pt8N9wc6oPDGODbpnqSP zv+6HYtq_+07iUUlaWX>j!^c0CD?$}BR=@ei>9EzS7UT{AMl3E#u(H$Mm*6A^)o}BU zkP+vi?QjifZUL&k;uX-3;?1xfaMkFp0V>z4t)7MAng_L+hdk8WKyMcbPNw!9H6U}fM7V^wxQkMg z^Ye=1bMlLeMHv?G+0}7&GRQ!cpEx*GmUDS8RJkA*KNm-OY92!Eq&M>p%R<#A=-*f7 zyxj*?E5s$h#hH>>oQzOh9sKQ~B2+PB^_y>;SDbunLGBP>#Nv_!D?4ir;TdvJ4L9!y z8F6~J!8M?{1*qEUyHGcZH#ywls?l8oRDM_Jsw9eQuE!mRxP}RfYaEu?eXe}555+Y{ Q6W|)qTmw`cUpjj#02dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWS_>zBl1vl1HD}&I2qe^)PT&@65$f!;x0-} z&d)23&&e+?7G+q#XO~p|?}H3f`H6#5WjPObLzN41@pEycr{*ElmZ!H_%0ks9=-*f7 z?Cyc872*=$;!MdbPDUt>l&_qq2vy8j{pK6zWM|)6kUInzvA86`%FaRdR;?UV!_7NF zMx41Ga1Cf~0jdtZAs&R{O)gKkYIN5Cm75uspt|Ns+;NC&n6S9UVTs+0+f}S6uHjFF OYd~`iP<7X&TfYHJxQ#>r literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798306.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798306.0 new file mode 100644 index 0000000000000000000000000000000000000000..883221b8b4a42c17bfb527964fe92878b89555f4 GIT binary patch literal 415 zcmeZZfPjCKJmzwyAKWt4#`KXGuXEN5;vRJkA*KNm-OY92zZ;jIrpWua;l^zSQk z*7iWv3ULW=ai(MzCnFRGpDT;pQD7 zBTj2~xCS)009BW@{GWp2O<51PYIN5Cm48^@c?ZQcKJmvPu3^IB8iyrz%O2@>0RZp- Bf8YQB literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798307.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798307.0 new file mode 100644 index 0000000000000000000000000000000000000000..fd153772ccc935436ea59f98e1ec039947c00393 GIT binary patch literal 461 zcmeZZfPjCKJmzvvKf+)A-SMWQ6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v>oQzO>>7`1KB2+PB^_y>;ZqB~7Aa@8bVsS}=m7Ury`Hyl?4L9!y z8FB7(higD{3sAMltwT{L-pu!at44PXP`UUlp$HV${Ej;gaSam|*ElS(`**oH3&k}F QiEs^Qt^ulE@qM2v0B@v_CjbBd literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798308.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798308.0 new file mode 100644 index 0000000000000000000000000000000000000000..fb9ae9465c75aa87b445cec61917c9d4ca9f2530 GIT binary patch literal 461 zcmeZZfPjCKJmzxB9^$Y5?s(HtiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-qAto-A`dk;(A!0V^K;vd8j!hKB3wdT+(oI$ z`FX|hIr+uKq6`c8?27)cl#_ufKXGuXEGK6lRJkA*KNm-OY92yuan|M8vQV`N`uCMN z&%o3QaS3p7reqc;BNT7xn-ir7Rm@oZ<{RgFC*N9-I|LZ9xFo^KuHnbWW;v*an|Fka zIA^%SHK4f#sCt*gnL8-nwDf?hMt2QRdG#M(KNQ#8i8~H)4HFjEI4rRX*vEee#Wg?U P;Tq6f15~Z>e(78Q5rdD7 literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798309.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798309.0 new file mode 100644 index 0000000000000000000000000000000000000000..a775d97eee2e7d78477260508fb91124646e9866 GIT binary patch literal 461 zcmeZZfPjCKJmzvvKgeJG-SMWQ6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vS;1iPt8N9RZjcwCJR-YpnqSP zQ?3uHR)|Z0i!&v&I2ob1$5lvJ5vrK6`pq}aDktAskUInzvA86`$}a7;y0{!v!_7NF zMx2eVa1Cf~0jl2E5>k%h&6#d+)#$DPDwpn^zXioL7V*a+u3^IB8iyrz8jfj_D6ZL? P2-kq-8ldW>p}X_}H3W;> literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242796.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242796.0 new file mode 100644 index 0000000000000000000000000000000000000000..8e24d704a6519265930ca150b3791bf91577a5fa GIT binary patch literal 461 zcmeZZfPjCKJmzxFFyXKM?s(HtiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^5CmcfjrdQKyMcb&JSO<*MQ8`65$f!;x0-} z&d)23&&e+?7G+q#XD3waS1$uqe&XO%Sx&ECQ00PL{9GLAsd)&slXQK5$U@a7=-*f7 zRR0ZCE5s$h#hH>>oQzQHFsnOR5vrK6`pq}aRU3S2LGBP>#Nv_!E4%(F-)_o5HQc-- zWW@PuD_jGbTY#!RXxH?hc=OOUxN3CQ0F|@t`1J|JHKu2dL0rRx#WfB~?2NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z>C5ChE)O*~(A!0VQ}pZh8j!hKB3wdT+(oI$ z`FX|hIr+uKq6`c8>|9D7-;seTKXGuXEa!sXQ00PL{9GLAsd)&sGtNZy%0ks9=-*f7 zobv~&R)|Z0i!&v&I2ob%^q-D(icrOj)o;FW&feft3v!15BNmq=SlOND4p=J()o}BU zkP&Cs7Ptm9w*XaZr0lwY;!UTmaMkFp0V@CaP9YY>HQ8s5L0rRx#WfB~>|)mD{X}t1 R#5uSIG}i!Cvuv?f0RXG9ksts7 literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242798.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242798.0 new file mode 100644 index 0000000000000000000000000000000000000000..344552e22fe27d895d1e4229a77ef7ee0b189308 GIT binary patch literal 508 zcmeZZfPjCKJmzv9H|DSY?s(HtiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(xJHEl|0nkKyMcbPV2ARYe42|iEs&VaTlc~ z=jRp2=j0a`i!v>oQzPs@nZ8#MW|xN>Nnpww{7sL1-V0j5sOO_tnA`74l2q)HQc-- zWW@Pk3tR)5TY##$x@NCO@#em*aMkFp0V+>YKV*jDn%pzTAg*D;;u?n~c8o_9%}`uZ gbq=lp%{4&Pr=%;tqqxT6JX|%pYkNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z@s+vqMjmQzptp+z=Z7!bYe42|iEs&VaTlc~ z=jRp2=j0a`i!v>oQzN`)3!-W5vrK6`pq}aOB;M@LGBP>#Nv_!E4wE7oyl@g4L9!y z8FBh-g=;`_3sAMqV(w}bZ^~?gt44PXP&wz(U#%#vNjh^3;u>oQzO>P$$|$5vrK6`pq}a#T$HTLGBP>#Nv_!D?9af7E*Fh4L9!y z8F3!o0@r}%7NBb5k~tqxycxR{t{UAnK;RNj!55;uNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZxmkLkSsrR`ptp+zr|{S9H6U}fM7V^wxQkMg z^Ye=1bMlLeMHv?G*%`<7-;jYSKXGuXEa${uQ00PL{9GLAsd)&s(ZUh8Wua;l^zSQk zCjEx072*=$;!MdbPDUt}Tz7q+B2+PB^_y>;vp4wEg4`j%h{YudR(9vP1J=qxHQc-- zWW*V?1+D?jEkM<(w!f=Uys5est{UAnK;^&egv3x>Q+VbW#5GJb$q2<078dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq$PXlKY6IRf!;0>oYgC~)_}~_65$f!;x0-} z&d)23&&e+?7G+q#XZQL>(+L@<@)HNA%5vteg(?^1;^*Q>Pt8N9UAOY7vn*6?g8qGF zPN#KHwL)A1T%0MH#mNZ8k&c|x6`_h5tKWR%luY)j1-V0j5sOO_tn3V*N1T^~YPfkv z$cU3O4Xy#rEkM(X3UfuMV#ew>-#FEiy=p=35MadOk_0O|mGyDva!?I7?+6)j zu1ke$KywRFb*`zNG>SJ<(%`DmT?16^@2dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vr0Gvyygbz0KyMcb&WIITYe42|iEs&VaTlc~ z=jRp2=j0a`i!v#XC|EDKeepnqSP zbMIQHS|Kh0F3yz9;$(zk9@R%WicrOj)o;FWN+f&Lg4`j%h{YudR(5X+kLk-nHQc-- zWW-sN3fF+<7NBbJb$QJw-n2-At44PXPdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWC6SJVtJ^!f!;0>oTpZ7tpS;bi_ZUL&^V|RNqiZ_>}!&RfZ2B`eV;^?&~uBop-0&xu!7S}i|u`8ZaS&HJC RT@7#zXs!XO4qzw`0|0o$kpTbz literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc20.1423158.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc20.1423158.0 new file mode 100644 index 0000000000000000000000000000000000000000..641f99beff96f88c01e5e2d2d4dc1070b753fd98 GIT binary patch literal 461 zcmeZZfPjCKJmzw)dC61#-SMWQ6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vB-m``DS4>5f!;0>oGvT2)_}~_65$f!;x0-} z&d)23&&e+?7G+q#XUE@tYk~|^`H6#5WjSruLX``0@pEycr{*ElK9BO$kcFyE(7&(D zDYOo%R)|Z0i!&v&I2obXFqTzF5vrK6`pq}awq&nbkUInzvA86`%I-|_BO^JehMRYU zj5wLn;Tq7~0#t1`^|1_!H)kPKqq_#EeACxI(ewMf!;0>oL-{aYCz^{iEs&VaTlc~ z=jRp2=j0a`i!v>oQzPMAE9(d5vrK6`pq}a+zVc{Aa@8bVsS}=m0gF1gSi}3!_7NF zMw}*>;Tq7~0#v<%xAh!~H<_-$RinEGs9g4H5HpHvj=nhpaSam|*ElS(%hc#Nfa03G QcW@18t^ul!2@7Nb0JlMlo&W#< literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc9.1402603.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc9.1402603.0 new file mode 100644 index 0000000000000000000000000000000000000000..7a6dc43e22f684012af988061996116f5a075ed3 GIT binary patch literal 461 zcmeZZfPjCKJmzx7hwxT^cf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZnR+aXLmp~wptp+zC%@>n8j!hKB3wdT+(oI$ z`FX|hIr+uKq6`c8>^Afk1j;~_pEx*GmQza-s$7tZpNk_sH4mY7tLIz?S*Y3s{rk$C z7h!6JxCFR3Q!$lb9RiG4T#{gAH&0OGryNwn%{xLy zock`rHK4f#sCo{o;|~;X=3IfRMt2QR`IM~>?xMKn^_wFQ*Dzsmjl&W<{u$@BQC!pa Q9;xfi@@LGBP>#Nv_!E4vO22Xi^7hMRYU zj5zJCz%`(`1*qCF*<26Bn}S#2s?l8oRK9Wj=QtGC9C&jC;uoZm&a)qu>^65$f!;x0-} z&d)23&&e+?7G+q#XP2qet|bFie&XO%SLRb$q2<9Cko#yLKQPszxl>_;euB!$Q=TVSX`1|Whe09dzBni!_7NF zMw}ln!!@9}1*m#?d$t{lH`}hjRinEGsJvmp)p8Wqe1CHU;uNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIb7BrD-Sg{(A!0V(?x1q4ai(A5iTJv?xNJ> z{Ji4$oc!WqQHBM4c1Nea@0WopKXGuXET_6WRJkA*KNm-OY92!EC&k@MWT9#k^zSQk zDk?(N3ULW=ai(MzCnFT^nz%4U5vrK6`pq}a@C#nGAa@8bVsS}=m0kAbTwgh;hMRYU zj5rG~!!@9}1*m$HX8dMUZ(f0`Mt2QRx#Q(CAt>oQzO>|I4pNnpwqb_*Wg4`j%h{YudR(7Vde)7pdHQc-- zWW-r^5v~EvEkM<~`4f6jy!r7mTs69DfXZ`Ln|7nP=JlH+5Z5qaagDNS(3Th zyk$dcla~_CTWzc+u{7^m$V(~n_H_5`xqYAR*Pre4efIf&Kl@dT#QF30K+FkFCZARD zbK0|Ms{RHc#bnV%XBjOS5n=Ond`O1TVoete=_x|0&YEd9rwG})HG(O_DCmsclzzD> z&8T<&rfhoDXn*o<+?*&Mu7A1YdiT3p1uy5Nf^k{8#i~zBw_1ltiUy#nc$vg9EiN{&-m~71u*qXy25^_I6IRX-q+w@XR2A;i;hjEk6(*ec>)BAs|7z)cB;AJLG~MtMqWJ6 zduFAi4Ushe%e9}bt^=?7Xdm4aNg0`hW)+ML1u3N=1ZRblkoW4iV35)rM%W&@NeCtA z3&HUkzExCqd$>(0F(B{Exy9gkx7+Ce77^Pi4t^hWAA3DI;V0uaD zyU)Nzp`mo6o_ccuyt}hWeOElD4;R!3W4<9cqm273k}g~Y zV*0-&EGvv*W&8Sb;Cvh6TO1y4zs$;Hl$uw%4V>?L0v%vnFnuwB~sd?KY;8;l=n z>+}US;a4*4qA@FN$ZJ*3V_-A4j1I6iHl*2nL=9}r<&^GWV?(hCTl#{Srig%h4jWJS zWtxtaAn)s=UcFy4lpj54l0P!f;Br_G;4$LgRhUb0GlkoGN+BZKeI*osQUHD z8eo&{PX`zqOqWbiv;iBLhSIDJ#^-JCEr6J#8kRqF+@?2-8FscdAG|xcGw+c|Oi3Kt zg$yPTv+*~<8D-4IH+`^xn3;bF%L-$-MSUU=oKNGuu|_mz(R_3atzHby*P)~Xj0>g% Ub!R?-HS<)IW^FLuI>GerKjNbwga7~l literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc10.1364266.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc10.1364266.0 new file mode 100644 index 0000000000000000000000000000000000000000..e0dc66ce7acc4644551c27cf7061b4f03b596777 GIT binary patch literal 2669 zcmZ{mX-pGA7(iQ5eqYn9~Ogjun^Tu`~}TGzQ_2k;8Mw_sReHi^k;W>NH#Sv=6=M@eW1!JBFi z_;FvZ5xgJo$@fY%n4GLh`_kq|SZ&C&;aqzaVHJE|o=Y)`OPuhwfbvZN1eRyM`^H&M zWZOOZ%3YXfBazTAX|F#SXl+x3ac@r10mcQ>%YSQi;F`s!Db3noyma2h%fMz}@E?aw zHy1WBMQBE-{2{Q}9YO~f8%%q>o~-~jBSI<7+F(3mZ*GSVxoXGV@BZ9<&&DcyBX^$X z-PY*RykyV+ZdsJ;I{=(|FtE8=B4vyQmDWcoLCR;r1ZRacC~Mi-lOW|L6=DC=)u8=i zd;(bB`_C0MT^=q|%Je9p^@18KZ_-yfz_?&qo^v=HT+`zlrCA${kJ(_E32Y*B<7(|T zu2K#yMz5-3jsu&ReRP1a!F1*XqZ_cPcham4#y9>csUApD=6vckhHHQL#bF~K3t~1D zTARSRI+D=$j7{m7_7q4V?|*~XfS6&UMKd^en$^ESI%fXn*lZBft)1YEGVXKq z%(*fU)B81HSz!$8!lMp@^_47mYxi*ZWY%br=VNsNSYN!F4lpj5?#wDo0oR5MK`eRUB9=N7@C#6{%j1SJgY6Uhc{mL9Ru3od2 zL&*Q`qC8;pPDuwC8%)Q0Z(Rs%s)tgVwZXVC|KLp!v!r$DZToJz!u^Xarl~Da)vwxnFfQ8kW}PRyl4%!rWYtqhMfU? zKu|aQgNYeSiqFI+51op|2b9M)A0bNAi~z&97$=S)YGgaK$J6|dz5Vm%`<;B}_vPLu z%bdUQ?~@O6hvZ3@UM}lh7U-ihW$O*<#6rQ4uZdZu=0D9B45HDbGiIAMsKtUUTe3~t z)Mk@DUofeK2`V#JpC@EGpA`RczBH7ao4Pz`+nXhw>OsbAk>qE|@O(I=LEL^Nx#V zZ7?3W)zScLCP%$>+IYILiM67D%S{}xITTF?7#mD04512O^Tm8hvo;t%YdL<=pIr6v zZ(|{x;M?BfX!HTk^InD7d496K|8{&8P^kx}{`pDwRf&|b8Aw-JKOLkTiXu2GoPpZB z#akd{g_^MM>1Lpi72yM5dBZO**(N;Prj+SWsxa^aSl;P>=>X$`Y5%ePE5S93Mk&qO zV7zQW<{eAJC}TEf(i}U8IlP;&tT2Yd zpCycd_3doGe_b-BO^5ufmXlz8dpqd>NEW&u>mp9H2pjP&b|34q)j?z zRY7k$h#B2aa7G#TS#jNS9K`H@$Fjm0R`Z1>u)cF?e>ptdzL~ZxH0|EH!(e^$wRC`S z!E|D!MGLO!wT{xP4aO}!lV1UwtcHSi$(S}PIybjH57?A7(E--Rih}Od76BV&Go|0N zv7*_vi+%?&b2Slmhm99}GneYu4}o*L!awhjh}oBp%ma-_Kuo!c;EXb6H<`=fXV&(P zWrZ;uqkM7uDSeUO0zZ?*9zL|Uw~Ok*}dcZx~_u`WJlqcN_fg<`7M>Mf{q(N&`H z6*u3;WoW~S8WyuS6(RV*2O>~4pjkA*Ld4hRmWAO~k)S)YXQ%lcd;4ee{Z78~`*PZR zF7SD~Kfi*X6qQ%+oAq#3LZn$VS#4_VI*Y9|d+H*!@KLG7=CF%qyGblkJIYqCHi=)U z*NfIti>S5?QpRFyiNzTBW+3PEU&-X$g}U5$p1Z(Z8a!7OBglo}qNUJoa~MnP4o58K z`Vck$&~kPh!EaqXl*kXeNbr$Dln}APVhgg}ljSXmge}PIOyev5XXQe)z!zB^g+bV* z^!BC_SoZZF|MDN4+~W6$mW41W$VM?s6UQczfrlOPsk~542N)Mj$KSQe!8JWWnzh0B zjIG)tU^9F2K)_~52%8)?>gZcD3E0g2m<}*Dm_GbVbq}yfokD5W2IIz>ivBoq)#rC! zC-GCFn=bnsjSvJOY@Mkjmg{==O>X;{4dB!`mFJ2?${Yg{(;I6*$__QbSz!ZmX+Dnu zDUW3m_B~w#s&ooB!ScHPu5TFh2$_;IBjdqozTrlnZzAOb?^I9vVSsRQi7T0zH8#LN{Lo%kb3e7vP zz7E*b#M1%B2Ggw>O?qIX8$)T<2IDJV^mstb@W@e(#0JFdOcvY0xx6>&rgTitr%_qp zWv+Nka7G#TsXJAh2Vy376P6Xmu-!545m?{GCBOSULcYv~0+jokZX#G;+A=!8xM2E> z&lCx+c_K)&HWCF-`yOyK5&eMG$=1K*@8D-4AvBiVtWRF9Xz>e&`p0xGLkYtlF97Sy+C~Q$7fg$F1sZV8wjj;gVB9g&J_Oi^iRS}0 zp?+pvN6~q1(|llaAejy@HkjTgi`)ThtSOXcZ7?qP3{`@dg%65b{ksW=F<-XroC(g= z`*wOHVsZ;n%6H-LW!`^Ha7G!kU*7wy9mGr@AS^44;mUqjJy>5v+~(VoFQ%%W*02zkdzhJ1?$I5L+5Oh)!OQtm&`79p$kD5 zQL;=jT3Lb--N_2@i{tE*6f(&Nd8emL)!sWW-l6r1g~jIbKjt58l+L)ZX*1kbHDSTa1YMsc7%gup)e zpS!vCGuvGrBYeD=XlA48(o(aW479xnh~aLWqXUc!riB9&V!<^vm6T>}Fh1>p;{>qr zkLq>X4Dezz*NP77^O}LpS`{5&Y%tw=wJ09g42-5UYlHD84bp8v$zU3R0d>5%xdb zR8(EF!3mZ(w##Pk_wbrhqD7VG_nrjHo77DQ7#B=S{lZ|%kYAK$Z7`m&rM4Z|Y&jTz z%Vpy&W$0?RC{onl*+~vnzZX2Hn z5YwTa*9Oil$nB{Ui781%@(O(xh^fvaIHQbNUQuKOF{?KdmKDbEQdLhBSl^sRXT4}l zyA}-+O3#7yIh*JJ0U~+HW+W4_o)Qf{D=V^zVK5MPf=4 zQLg6C5AZTK_Y#~@#%$5`6(t~MLmy#TVGPSPY1_g2f`hiUh{h~mj!MvmB(T2yl_sLY_yJOZ=V?e&%CZQuTs-}CT$pC5A(68q=xv50K0$u0bD zM`&HBr>n-K)*0ksTeSwWB6yXYpK8_`1fxk~RGT)+g`~tpwP~9?*`zaTO>%876_=pX zYvb&Hc^z|Lq9572a>=5x-0WMD`@MUEM)ER#s7V`dGzf8eqaciu%5` z;ltg#Pw?UVFn-7;t)ZW_c-&U{5LPw)(*$na1H#I9cb?m*6XN^f)P}6N{seZ~(bCSf zT-amtaCdTGqEetHX;iH*8K|q51#|WJbbxWe^klD2GkE6J%amqqFkYpQP69U4Y29`k zX9qS5t>|dNwn||0b~+tkY%uNmbEyj0+?+va)&}FzI}6r~Ay57CqSKeFaW8*hYb51) z-es#=KT6Wjvpe8r@ei;yJFx1ZNXmsuRQ2urVUTjLoZzgm5WTocM!8Zo#2czW}A|NCW4r+_7Rp9#&F2+fHH8t1?9D6qA^P} zXpre_J~*FmB^_W~Fdca(dJ=f%hkly1!Fc@a(kx&TFkBsf|WnKA6Jhvf3pZX8>kF5CSOJIi`9|kyS z%`U6QBnLYtDdFhs;;vcVWT36xHJD3ipaYBxrh8U)#er)&-=;KcgYn|T5FcRUDF0=% z8D+;N(v0?2zs>|U8>8p|V}t4RtGf$ncZ~D+GqmL z^P@K_vOGkV;oQ|>S|4y~o~)%tAZ26{QWQTHfs{o{2+j&8p~}*^_d!a}D8l}yn}lLw z>pQ^m4E&AS5f8g5MM{)CId~>mUdIGFz_?&~-dMM0aLr0FrCA${TlBKiz~*p;yv}N4 zFXi+K)HG(_S^J?gAU*IZCrO7$5K`8kkB_=6`XV&H48B+iV;HLCk=h zaXH}J)rW@a1!9VlP~yR(K_F&b0l^t%%sQLrTmUi877~^f#<0Q1^AcEJZpVuT!I-s5 zl)F8130R-Hn+`B8m_9w2kOr<9`G(T04aWUS4#=m2Ab z>6n4NkAThm`IKgDFg~~@dk%=%GsEpRu>mowo`hZj=WhM#eMdOv_h+0Jh#5aba7G#T zi3nBXgO~|F2+InOd`hmhEduM?rtY|BM&M4pN$#$`8K`)F>_-5 znyogY;gh-6AiFzO3VPfN{a}5u@cDxTd6((yR@}6P_>M2Vy=7zGAbn_cdEy zf=-m~UIT1$L+AiwgXuBl?JmH^GnCS-4aTeIj#>p`rhH7hZ{5up81vJix2M3lUE+f+ z0x?B#sMlk^KZx1mMsP+MvsEqe1t8{icfzv57;YaPdJ5Jjm+X8X7_&ScZMoDg0qavN aqyvl#rgKu&@UOwLFiNvF81M5M7XJYS{wDDN literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659943.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659943.0 new file mode 100644 index 0000000000000000000000000000000000000000..daf6b320f7158aedc904b61798e8e6dbaf2737ab GIT binary patch literal 2669 zcmZ{lYe-X39Kh#%uzRzmGkc+y7-7?9S!Nj?H52VYSyWKT=BC#&bfeulE6G;&GU+M3 zjLK}%i=YH6dTr~eX=R_%=&c}GeXyc3Dm%{8k^gD;ejLC5;dlN&?rw&x|M{~&;~2Lh zF8%uVl(v*ngAHbVzEP8A6^xdI*hLyX+9DW5li6U>n{zc{QDLFpyiQYW&bJ6=jnGHw z^70D=9r-2l{F>3^+=X*zjo^A)WHz{`V4b^=nER2Ra?K-&vdEGOQk1B?r%Z*I5|53V`+4y9QejKBTR-~cv%X8rEB zk$bR7wxezGw?<%NO{W8l4W@EWwI8PU!Uy@Qa+nWa8_80HhfA@fRuTug#Axfi<+uj zbzpgO`5VqY505Ej26TA&wFI!dN`E@QxM12fdCDVj&5*&AW^FKjq)a&$*woafx!g9M zQjV!b#lI}j=ErF|z}R5g_rd}Nu<<%eY1Rhg`;wD7Mv{~j-HJ%=Zr7K78!t78IX-KP zA2>I4ch7B!m@+Nml}kH7%z1kW&M0H{gl%glh&lEkVOe1eFR4=7!TL78YP};F(`i74 zjv5ZcT>Xv?FfN#0V#_@auBmRPG;4$LWTmbW*z`mj?n=hA)uML4u8Y7%9zzEh8%!@w z3*QB7b_~$04aQq0IZlF@2P1;+5gQQmoS2pl&eeSlyDuHHHAWT)VxH+CIHQdFc+Z&r z6~uh;ld!BXhU0CzJz#y-!Y;Rm$2ZfNjjT?~DX_jTMmoT_V7fPJ-EVNsIuoT?8;q-; zUorukoQFjZBx5@5XjfCB7T9Q;=>ThEM>D4AeSl5p0R5kh9WCftc>u%=$&6@l+jzq_ zvtespF*x^K{Io`im~XOBc5rtXh^b2?IHQc&(zPS@gP0=`VOe1eb3PT3V145cjeF=8 z_VmTHEk{Y4%A&yfo>$QU#s$+EE%hhCHP0TUG;4!#M_F?buz4I?-*4mTZ?^gf`dL|V z4cL^#(E-K=)1ths2H1?6N@>;x<5!~OnIPt#PkBx5-T1(mwtzSoGhDf|St6!vAsXga zz6`{CuOc|3jM>KSAOXZ|4I(TnjA655y$h_bJ*xDPWX$SC=t*E`J6K=hL^{B@V0wLN T#7nTg36m(z+F*S2m@?O2V(K4z literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659944.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659944.0 new file mode 100644 index 0000000000000000000000000000000000000000..2e5fc78c65f4f2b5304433febd779d289cf5e3ee GIT binary patch literal 2669 zcmZ{leN4<@9Ke59Out`J>Jp)}2+`Ftvf`m=7xETDv%Bs%?Wl{p>uAl(jZ8EeBY9i1 zYW(G8Xy{T(lrYB9yhV9g6nV*c2~T&A=k|Tv@BVC`?{}Z)``L8~oc;5AU&1k-pDe%f zEw(w<-$SieX?3!AlSXHZj!cpXw!g>o{0>4UYSm}VnYFZu!5m=J*zMGd_ zHQPMABu-3HqfuGhi;w^^(EP+Zk{?t<2N)Mj=iZ+*4?J^JEu~o-j1O#3>;yJ$@jZ4M z7biB;OVEz;r)z=DQaK%9Y%o18e}6w<)9s*H8;om)J`VRIPknR88o<9AeWTLWsJ|cx zt|nEwFW2&COPyhV8Q8i!w(+V+%IOMpddc~|AZ6eTg0sR3)bGy$SCCSW6ZSt{1s3%*|yiODI@HTPT&#B3}iIHQc&&e3BdK}_=@!m`2`PORMN0?v2o*`pfKm{n?2 z5vMu=&X@C&4lpj5PHu|Y1)e#zmC~#Y#@9!;E(SJ3!ql~*F}IeY4#k6Qz$QDK4lp*D z{(3rXBe3y`pfqcP@#LQNY7ldKu+Md317gM`>N~;S@-G2(;xX?nZ4Lo3J-!p1QO13Q zXUcORX45~G6~?e>^9Ln3-qS>V4d|7oR3>a2N)Mjhnl+N;F%T&&DvnR z-SeOk*vxInx*-~~ssz1DG2I0=NsV-XwJAZ-1!tcFn^p(?pG^tM-4k&P#5^%S_@>Rq z6@D|X{FvYe_DZ8B*Nen#S%Lh8p0^<8rx=1W%9xeBm-v8~mMMf~g)uzMKW!*D-^jv{ z2Ai<6U(Bs3$XpTl37oI3hz>9=m_C@l;0}0Z?O{r@HW(Lt$HoAgyO9^{HqQRd78fD8 zC1?e(xjc~$FgBQW@1B?gY>bmA&Dvo6_w02=Af~co^)1_N`oNe+6WdH+Z{>hpjUq9* zg=l2G`6r0k;X`mn8MDjweOwG;K9mxc6~=HyVU`~_U)#9Nw?$(XC!r4!zv{sG)`Zgm Y#s$-!oz2(5Gqn+vW^FLeXQ#XT1*MZFTL1t6 literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659945.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659945.0 new file mode 100644 index 0000000000000000000000000000000000000000..ef79dc1b783089ad781111791db0157fd5740b7b GIT binary patch literal 2669 zcmZ{mSxi$w6hMpm!Fz}zYH*`qP*j9g2}C72S`i_l5yT~KrL7dKLW?b88Wkn2QE@>K zH;95siSXf%3PA;_btwXFi5tcRB?vM2p(27C4(&M29sBy(oI5#lPu^?eM9$Bj!nm`X zF(UT*cUg;UoVQA^)MzC!*=lW8RQL)BADX4s8gzPMPb zVtzYq2D!0WU_sf@#X-O(FqRH5HkkIR^REOpRu|3MU_83-Tecs$YR7|KF;_jse#6md zD9`hQvz6(iMXmjN>X$7)0#419HP;KITp&l!x3%Vglp_`soE4TM#r#!aAm#gi?0>p) zw6^3`KUm%tzScV6;Wnj6g}mk@hJocx^QHrg3#QwQjl01$?Jk)%U*Id_0Y1RhgT`MDPz-DZi>b79az2)eW{D}$JOq0+7 z#s<@hnTI97=8crntPRHJw*54Lm_~^re8Z8}QTrh2rg`Wf0T)mCb ztPRFX4>mLc8*SruyI@SK1tk?$1^}DPW;(#ySWr=HM;@^0anb+TSkOUpjU2=rlMvM4 zuo(=WOur@mb>LiX*6e!%FzRELnfN{a}%hx(PxaO>4O0zZ?4>fOX0ydArtDH9OUbAIq zkg~d~3fR1hpaYBzraL}Cjz`DF9`esJ#OVR_90 zF-3`}_WWb`_s8x_a7G!k%PLHTAf|03VOe1edoDuz!TRz-bM6bqEL(wom4-!v^>s_> Z0ONw`-o9NbaLuY2lxA%(UUDtr&|koPEs_8L literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659946.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659946.0 new file mode 100644 index 0000000000000000000000000000000000000000..1660384d92ac387aa8bd00089a6d81a7db023590 GIT binary patch literal 2669 zcmZ{ldq`7J9Dui8bZ?rEvNFmvwQST(rKfSq7nAIzrGL1ZZaQVCTW#ukP}*Ayi|mbs zCbI`o6s%IL+{(;MqbQ<>n$ZiatSB?1TF32lzdxn1$GLA| zvf7@Arx9aaReGgHD~-ulYjfv@CP?|IxoWLJr&sBe`gExwFDFN-Une!{HMwfNRNY6V zWN5P0DUK&Cmba%TkaOeXqQ`Jy4@B4c&XtVhJ^5jJb(&6VNXgb23|^v&;pjqFR_Zu{ zH(C#l=O#TSxCigX56x6-``IV!mV6jb*wo+`{@j~?tS9fzbLkpGT0i`y-jwb~VDW|z z-?@Og&2|rWu@jTjxu{xhYV;ukPg^}hxg-l6U|cZ$Ox`UA*9^2$nzg~W$Hs|uz-C0u zZ->nQCpHUAD9PVg1Z-x@=m2AbX|DUX6455jwV-qckhP}cYpHj>-I)2 zJkJlzS7v*OI&1}V0yVAR)cFz3bpk0DD3Jb_xe27~i6%HJtUz;Yi)0|BIfk(R=_*iQ z-I5Zpye$5zrO(4@N|6dB`95y}%RA&k2N)MjTR$7~!8I4UQku2F_zaJPNMN(ED#mKJ zahB4z3h6H2i3B$0Q*?l_!So-^lyYFR_%x+i8;mcnkeR(n%H3av25~bxzdCHh5)gC8 zrD8vL?w;No0x?AjRII&d12O0CCpe>w*_R*9=^$qO0m8Dv7`|T0Z3XKqXn%53Fs4O? z3f5;uf%PT7q63TzrqdFoqrf#!zNR#5gK@=~40m7?JWW+E7_+bnl@#qw1~zfi=>TJc z>6=RpJAuur5K6N)7_Y3~t^+ZR0V8e^8xV8b#HMg?uH>6fgK$jL>FNn$%6kdUDC0h3 zy4u=7%#fdiWrZ>Pt2}TYSYKXFr`^Npn`uc!zEStKf%Pd?(*ec>)1fJ!y}>mL)=-+Y z!T7u_4Yz=evMKMjU`&e%4H0{n0h_{RI>6fCPqHoRIk1Utq4a+?CiF%ervWinEc0)) z+YE$n=FDN?1HidQX3f4M5VKw*&oNtHi4Lx^9aidWB6h7X$`=}#*%%fL0O`)Sq&WnCF}?YkKSV|rQLLcqD@F1wlq zVv6EXebk8n5c8cI!5L-DzL?Ni2x6XhCoC(B;ftlqE`aqJr*3Kyj9HO@UY_gd0qe`0 aP6rqlOm|lYOTabdA(UoqFkadg9sU>fhbch- literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659947.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659947.0 new file mode 100644 index 0000000000000000000000000000000000000000..93a91c305b23de1449f619fc7c25b28087e1a630 GIT binary patch literal 2669 zcmZ{ldq`7J9Drxqm8)f2kM!kX4=NLAsjD`96Ifm`3tp=@$H>mg_ zDodly9yg%wROd;p0mnDTViyp~t51l(_JR8K0Ht@N6oiQsI-n;rc|$(S634PwW#%toy-SA?hc51yP#V4od5 zznHmqOWYn~eY}`t$D^5qM z=-RpBO0Ybhy=5Kp@S0M_A=K@287%MW7-4{N!Sq^J(L8X?LVtm#Z7{B0)%^?DT&_*7 zciVVN8D5M0EO8mY#&Jd%plmSR(*AG*uqikz(6kN4s|q?@6G_Sg&QTFe_^0n48=pBK z=8E1e95|ON?`x2VDa%9y8O?t{%&?;br<5`KxO(Jd5Hq5Ju(U9S)z+Q{u)b}tpEpXz zv~p<5i*gNEpX!}3K)GPL(ckq-ZJO>Y!;Pcr82TBJxbr2(6$Xkmb| z!Ss&!GsA(6L!@aNj4SQjBoH%28Qer{K+L|=2OGh;^M5GrOUIns7RP~@v3&%mlyRT8 z!L}L@Q`JvcS{TEJG`te5??isL+r#UVY0XA6GSYj%`jlIQ0m=o_!KL3y!8H{H0!`as zJYiOSDX>XzDQuREX|)1U8>zu6k^|y=KjoXqq+~ z{YGBv6S zz48^rWX2PmQpW6&iV4*q=JkIpEsSBV#8v^;ml(D4p=3;R8Y+?Ry#Qk7MhgR!3#J3g TG_%1q2Sl2-!T6<+4@>?6@@6is literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917688.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917688.0 new file mode 100644 index 0000000000000000000000000000000000000000..2290e7627cb216348993fc04b8eea62400f7354d GIT binary patch literal 2669 zcmZ{ldrVVT9Ds{V>AlFSluU=MAmR+N8krbImJ?86M572oG;XxCa%-5-hFcUDH()A3 zG)8njqmXLQEse$mqmzKO2=Z`6-13NLFtTzMS{-z_@<<15rI!BvPhmQMJCc6{NJs5}Xy*qVYxP z;UMLiIfQ*rSBv~U$$S8o=YG=cnDFqKQf5H*=SC}7-q0`|U|cY*NMEJ{*SzSWG;4$L z+e2x3V3Sjo*6OkGl``rSI{x*+6kyX-O$QhoOuxuYN(44*c2b(P!T4{=u*&Hqf zvN}$;0h@&(bbztJ^puTHqkv6xD5Y5&jPGpOUJhce^$)*BY(UKPy!-GYJJP4_kdC=w zS#BbTY3V08qm27FbyWc%=86HrvceeFEp)rU`hHz85MK z`j{#5B3Pe1o6@We#(Y=56gKN(FmeQ;Z#`}!N zY``Ws{DRlU*Jsvx6q&Cl!~>f_B^_XFFdbQvR|agf5tL?aFrMks-T^VoZy0~^?B)X) zvwm{pVsLJStENXHrYsB9b=oF_n6^=ZGs>6^SWvJO#N0DRSXLOr|0q>If%P2?EWaTc m)0&Mwf4+ABtnWk+9bjBAUG{ob2)L#(n9{5b#;Z$Wh5rHm!!AYu literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917689.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917689.0 new file mode 100644 index 0000000000000000000000000000000000000000..384fa70195a490955a6ffe09cfc231622eaa1404 GIT binary patch literal 2669 zcmZ{le@t6d6oAVh8Sn992v$HT3OFE$MaUF0!gU*~ATR=rkVvGZ9c5vGlD$`^a}&^6 z2D*`LMh!8_Zwwd^MIl0jR&lUVV6u_ORK^lr6J*Fn!Z6e6rM3B@(3KgLs-p&@X?K!Z$;X=vMzK=RSL%ckrC3!_p%acN zUlYnq20>}qqO_%DPI3#lK55q#46fr4usB=|%84&M8a!RTT~6Es={VT%*4M{%kl!UpmIJXcaC9(2LW zMK8}q5ZL4R$_8ip^eu-+fQK8C;@!wR`a)tj8Cdu=Fp2xAg$^(-nD#CTco#g=*h*>E z2IDV=ZN>nb6Hor@wAtpyW={tSNjZB2*oe_|fU&`}f9iAfz-Gfmvo;t{Y)q~QCQr4` zY=(35+b2dHjeL2Y_dKdAm&=y^t5N&hISsaEMcBq9Qtr_pt9?BTq-@?va8_7@o{mjU z2Pw;=3HyMq1}!PZQo-@kmq#pH9&S_0^e8PeWdIz{wn7IO7fe6(r&l+4=BZnhW^FKD zvk>YFY{twgtHZ`!$|pL}Y-eEsu<@>^1B?x(gVJhqflZT(W^FJYUopEsl%#y`ws$PI zd2!uo;}Hj9&fPuT2KN4D{QG-}m@*9txgQ@2Vz!F}XOuCU8)@nXF^`!D%L-%I{`rM8 zaK50?pMQ{yY0;wv^R8@gzD)}qU|cZ0&(;s54Yb;OEK#EqcH-U@5@{|z_?)g zjeNyU@XSMblxA%(ej~mA46v#0shW_CY3V>SeK)&+jjERpur?j&E2}&h*yQz5`T?5` zl$oE?4Pt5&q9z?Sp75J#S52*ey}rTEUzCWs^eWnJ`RExC^WI|wXOuC!x^|@*#1s{T zWrZ;u+SFPD&R6hy^d*O|yI;)OLiDx$+*xqG#^ZE=al!P}Gd{0?XR1$7nzg~WE%Ms| zV3Qd1jnl^6zuA^1^x3K5Bw!OGrvr=)rcXZNR|ssbxoFk~<3h%G0*Lwcbm^4iHeN8M z@^;QUVDI3q2Af1oSswB{AO0+exp;@*j521;t7;pFd3l|%tT2YxlTzxy`DT5przK;y jsL{}0C+~stW&6_s#s$-J8~HQfne#51wZVALP(#lDd7>s6 literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917690.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917690.0 new file mode 100644 index 0000000000000000000000000000000000000000..bd72070c85a935adaebc85a9b3b37e215360ebd1 GIT binary patch literal 2669 zcmZ{ldq|UO7{F&n^ZP6{%}UW(Sr$?gEjxC5nU$nOnOa5>n_I0l(uv=^RI8kr6Sy^gftGqza=V}GH zc92qS)Msi{uD_&D4u2R%_NFXdJf4fVD7iDZH*yT`&wC5n_1OlaDl^+?43t#)pdIah zIYJ2D+E5Y7Ex$(a(Y!D3rPCUQ*mJXk`$7qu5#c&-?M*2EB3p5d2*J|DuTm z_AvE);tGxzIX(P5+?Zs{MQKSE9~l|weC{8^MV+Jrj0>ihMCYi$GY5uf)&}F|BPT+D z&E?78TsFhp*eGl$?(V8TfX%L{bbztJwBpL^JHY1PG)l8J7$03~nLmy^_4Sj_GA>@) zQs-mZ!o9$z z!~u`mEz$!CvV> z2+k{dQYIJE0 zAZC74c&pRK6Mi#Im1+Nhy~oDI-4}`3wGNqg?>-4)&Iu(rqm0=S>84N+bAgPotT2X) zF6Q-t^F?lnc;FOv_ls#>i`oh*6ySU%COW{lV0u#HlU(r39os3*+F(5HlH(?@NeZyL zY~1~ut+Jw;o<1*NQxHf87#mFYwwE0NHpyct&DvnxV!he~VirB#_|SQq;V|Y1)BY)7 z@AY2GBaxVr6y#Yhiv=-LJ`$W!#%xga+71xY)JIrW7{g(2v?sv%UU=_(EE=;a6}2Vw fJ_YBC_N4=i3#LoU<|Tt?z8|7l8;tu#ZyWm)z``Zg literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917691.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917691.0 new file mode 100644 index 0000000000000000000000000000000000000000..2006fc955b0da743eee7daf5400e2a1bfda100d9 GIT binary patch literal 2669 zcmZ{le@K&Q9Kbh!_P*_Eu4EOutgNi_R`f?icn?Z-4N5PqxUR0vP2Xan8@%t#eqAXM z8Wu(1(XF~BO$U|L8RFe}sZ&qOOsKP{?1x2Rlp-^z6{S6H56}2M?cJa9`F{C4-w$&J zMes9v!f=K&s8%<=&v=kA;~R@;w%YWob_zDf@+CQXKGh-EB)e#_o5g&+R8&}K7I*85 zMXN&)^};x1+GZ^fOu@fYI4_-@N%m&1SvigS_^0C9_}+z6c?~~N6!Ppg$y8vMq)y&l?$|#BkK-rv6SfPs5ZkawYDgq(?xLr2xXebvYIrrzjQ*Pq!Bp4neTY1Rhg zby@u#z@|3&bI>Ltj7_>5rF=X40I;c_M+X=iOe@p73gLys57nTgE-Yrc_wa!ONq5aJ->GI>5MKTBt3a51!dPL}}IraD$_n3v6i(Ci zHfW<<2x9iWvAh6#ZxnuLm5He^qKM?PWgwX$`X$QJ+8$7eJmC~#Y#(yrVe!x= zU=yRJ1B?x(uc$uxfX%}=O0zZ?uk~t=f|z$Bru;{2K+Kn&4MkvY{#{*%d`y3q?GA{E z9uk~U#(g4d)|7&nv3-PPg)yAFT3H9q*Pi)0;1Twl*^-O&|9_|f=PO-D2N)Mj|MYKK zGkE6iY)Z2>7}um$J_a^9=ZiXJW45@_x#NdLV3Tu^4zM9NW*2!uOlKltSz!#1 zN#$eUd~0{j>k0^m`^7BTjHVm^jt1wOypIkrE|_kbv9=04vo}PuHW=?-pjiQI=1!>( z+JyTz>-rtdZok+EY;^H-fU&{!j|nb4usIi^SsRS+_~cs(Vp@B)-3;6&3dZzUn>xW> zap-WjOiV>KsvoqS1~L6Uf-}mPwH;Yd31WIj2+Im%*ivJt2Io6DasMsZn64b8+^S0h d=X0s(0ONw`K6msn@XVx1lxA%(zQyjT`vRFoD*XTe literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917692.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917692.0 new file mode 100644 index 0000000000000000000000000000000000000000..0553e2f572f9ad55f1258cca880f4fdd9a138386 GIT binary patch literal 2669 zcmZ{leN4<@9KbJW`u(I*l9wyx6j{40&CJWAri2>_WeZ(*x$LaFv)`3tk!(V)*lgsr z7rNQJWVXymr$VLj(r7f9W;PbWxuur3r@P0~zK{FepY8Mg?(=*g5SLH2Rzc(QD;=WR6B}Gze;gO4uSd=IV4RVVB$_Xmd1zTr*55 zv$WY7rSmVU*CC2&WN)$}eiFC;nxt-c@9YV@A3s*mWE%8FWwya+43JcKqy7-!* zma=^g4_`MICK(IR@#^bKg2_Nvw_h~(yPOU%E|``r3hV^Wtf`cN%3#xUcT z(`J+lo7gJ!OKG$Ko5XNBz}R3qX5RFhz@~MCW^FKT&AS>ANS^xs@jx*5aZIz-(a4ME zdG{Tv>;Os6pS{R5O9r;y2x-47k}@_O4Q!3mf|N*w!`EHQYMEH_CMWp)LVL~ z5*%+zZ*Bdshs%@_H8KyrnE;L#^O+7XE|`87DLnz68Q(`~)&}DrO2YGjjZm1-;IMI( zGQ@(eMVJo*8`&W`z}R5=&d0*Nz$UDS(yR@}mzF-snoLrfzj@B$65GBzZQN#qn8m57 zR`0mcQ>x`EIk@XWAAO0zZ?pPJHg1lZWT)b~YW7FbZq7n>T`bb8YP#s<@8 zdz6X5M(IOo)&}DJ)^#bM(P zznL zhp?+(%z`vDro=oGobTWsI>5MKdYRB?1JBIJr!;GW@eLa-GGH^;|Ay1X)xX(_Qz$Z} zEEm`u37`Xv4W_HV-iiV?u@foH+F)Gjt5$=U3m<2-Ic_r=#x$HPc@FmWe=celi781& zo-6EqAZEvRf-}mPeYN|t7R0Qv6P6Xma6nAaVsO6FvH6cgV^*w1>#M7+;Cy;-I>5MK W`e(Pa59)*hr6TH@1 z8o*i365N~jl!^&~WAp~~G{rH1B zv2U&2!^_2qNy=Ds-Z(8*N(P#meImK|{d9nF!Ss$M>kRPB4_!2CgYmDwQxbuVY3Of< zO*bbtF(#DP@C*T)(;;+#vB7lCn5O5zW=klgSsRSE#4LO5PoDbj?oTN%VtUlU0gqe3-356MBw@(_Zv!YX8wnx}%4`JsgUPgjKs zW;MHmkwYR5;mc zw{ey-r~;+=mqi1cIUDH!V}t3U4%Go*lVzkdYlHDg6AOph&SumA3_aR~!4 z&E|2Bz}~9NU)M!qN>nH=Froy+T&5>Dqm0?{vfv~Tvr!-{D~#bmW2$F^^VupJt3+d3 z)JS>A76Q)q%0dSi7ffGhjH(0AEVELYwZZtGlY71a8;QHRS~O-}1qu#0`3BgydC~#K z2GjMAdu#lhgY#{UZ?$_k{bpKHkW|>`1ThELQ{TDI|^-1Q2IX`6KW2h7Ybs&9T{9} zw{eBv%z_P}AHd#g{-f(eV!oJ%!i-S`AZEp2f-}mPEy}!g1jO7LKv-57!wp#%%;0>s zr6KipVQ0UXc}ZyguG%DUzLJ%6fN{ZeUE;wB;F(ueQJS^ExTgJzE3g^md(L6w?B8tp z9(1_qQY5gM>qiF|8%)o3$u|NU|9+HaZ7{yRtj`S)GxtvVE&FY{!dslBlU8 ze@2~AG-@SfOeVvO#w7@-1y?o~6qV=%Q33{y#w99Ii>iqR$nTX`78=esa768e zo=5@@D17yiw^bB-J%$aCFwrHWLX-2pf(*1hk;n7@oT3Aa3#P?F?IN(|($kb?Z7{C1 zW~hM8ud}}SY<`lkNpT^aY*Zbv=~B@F#s<^PEvw+scP3JrwZVAV;=eYHBCB@a`mErK z-*;T{HX1AlLeOS?UMTmpuc#&|GZpL@F{9~nfRrg(ba04y4M>@hKyX%Ai!$yWdJ9rc zQ4#hBT`fA9ozVb}XYQ?b_j^dD(`t*m!J1uunzg~WeRApw zVBTJc=@$ysGGOEO)2t1~bsd#yV@S%qZ-Zm_)~1j; zavKn{e984X@NSo_*9%Jg{&8CLZ27|u5Houx!5L-D2BpgzLCm;agk^;>T%qdY!1?UW zkFEra={BIR(SMeL^R?Wj1B?r%XWE35U`^{CO0zZ?KU@5AEwE`EZ@3yTro9|3Tpt<- zY;Gy&0Aqvcwx1s*0~>WXrCA${uZZsSfSCVwCtM>oASQn(p%1(}r(1EoAC!*y>c20? zKuq&%f-}mv&%7I@_drbX4PjYf3~zbjnG4Q$XI+oiL-H@vtwSLNJ?-Fp+jMk*al!Ps z^Ho#9nsfD(W^FJYW0oBNHb<)pZUl_!b|LZcjxWHbvxW|^HZIiKY0d;TJ1$cC2OAeU zF)QgCh-t4#x#_hDiUl!WCRW`B?{1Gtst*wJX(k$OZi@#oE2k5jQO0cT^3`e(^N5nL ztT2Y%@_ZXO-(kn32CuNxe@y!tH1c_(1Dr3pgbpw+n2tRj{Sd6#;ip*}j7LS3sez4b z;yIs<)Yq)D6lrD#7Xh1wNpyg*!L)X?BL>)rlPS&GVBED|(+Xm`q8yFh+sI(df%lib z2k(yc>}d)RlT)K2!Oa_Cw1AUWnZ~4CB%#nS<+ULd&C4=$Bu{sb=k|Tv@BVC`?{}Z)``L9fMUKy(y)%z-v4Jy6 ze}p%L5A3Zmsr5!h)M}kEb?UfS1s|NMGn!LOniREZnZlg5YL(iwPO-+MPt}Uk7ZA<2xNT z?k;S?3ekbrF`>X_MF<^WY%rZ)@4gY(WK5(qYlCs^`o_0@FsiRBTt^^ z#jDi@si?7Q%dF`~)4|p!V`{Doqzp?yfcxIK8W^FKja%}lMU~_6;RE6Ef zRm#Cd=v{7WG_bk6pAIlKn7&f=v>e#<%A+)EgYj!k(^~vV%IvR_5uD$%c887IND%Xa zTt61*xUEg6W|~izj$y+#^b}HW+UW^|An)<^WBlV9fL)G)l7m7qE#8qyvl% zru)wk9|kthoHT2L@o5bQ{6Wm)zI|^I8xS*7J68qv<~GTygkx4@&vpkf13wa+QO12@ zlP?y4n36Wavcec{H|KSN^OeMZuzR@tW?EHf^xKWM!1)%a=m6t_>C}fIHtwuvvbM4zMpN6C+XXAaM$G;4$L>%YqifK9!;#9`y=->l^j zGF-kG1#DV}(gDT>(?goyTmm*Lhf$if!T9;J>N*hf(-T=?E$SKX*U_2tiCjAQ!Dk^RO literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490228.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490228.0 new file mode 100644 index 0000000000000000000000000000000000000000..b7a5b109ff6887f78bf4aebc6884732786123f7c GIT binary patch literal 2669 zcmZ{ldrVVT9Dv*5gL@TG&`K4swXP7viiqO#4C6pD0+~eT16H7LF^zpBw*`De5S2uP z7-e&)iHJ#=(L@JCM;wUl+~lRk88=}eiivL}qQ(spL=WxpG{0kS|7^bB$#;HVPMa+A zd`7C0PjEL?YnuOG)w60wfI-YLT6Bqpg2lcpHeJU@+XaiwDjKXgVxG=cU^eH7+jT{v z(JqKOVU)_sH=2Yj&o@bmhLf|%xocCt4&ii8+2zr5qo(pf`~*?RwOVXhCacXhO;$Sz zrHos0Je1(=t|MVwaXZ1M@PWMl7Qy0WmsieIhY>b={(TL%rGv0R{A8ZXGum>!@C2o6 zrkcR=oiG36YR;9qJtoV2m}DL&=^64d=nB5i!@VNTf{AqkrxxdO^y(ID)gndK8-$6#!Cd z77_L{T|L_IY<(kG-sz!c=ctFzlrjT)+#7!gEbrYg9bjBAts1_17+iDZ2&GvYjPJ`+ zyZ|;=eoS<^ZG5Fv)uH$eiDkehrJ4>fHkdAZQu7wrXpd5wwZVAT4~;SGF_oJHa*ky)!a{%(lO6IeA5A9 zHoYV`qm29f)jezmFY`n{VOe1emrjaF1?%hn{<+)3=gV|vqm4;T#bAAdS#*GL!F1<0 z8acS;g=|W*HW*)Veu^2`ocgVxO){p_fd(Qol7P+r7COM%I8f2Lm3M*7>dTb=%*KIi zAypSa%<2S9yW7SOzRZM-ZQp@&nRMDy%=C5U-vF2NaP%$n{Vt^hF`q6y0i zW4Qg&YA0A<#(r(5TiDk>=I+g?VD7V>V14lwbbxWebe!RAD!AqYFU{Iu{7&^ROM%Uh z=Ay^O*UxP2G4xgUrggyP>6dhXvBC7;BR^?@ji{wGYlHD!gX&-q^VPllF86N6z?dP2 z62rl{XGY4eOT?6|MU9cuoiL`~0T0e7W40u1YCed0Z46;qVGOrfl(WJ5G-1UzBxBa5 iBhwM(L9jlBnhr27m^NA;$AD|Lcxl!KX61Mn+J$PS++&-FC$uLp|B>{>Mf55;i32f z7T4jJ*%D`kYIIX04h;b*1yo*wNVW(nf^qQ`BkCrud$=9ivqOHz-u`*>{Z78~`*Lqn zC7$2w+SJ3`8r9m?SIMKvZ~N;7ozbLDDKwb!6P9gI^Rf8`lf^9P%{n1lZ7JBgRVVCF z7YW9EgP=A{QyDqNJVS=($-XtR$#8OR+9w}{aSNT2w&`=11oH~sS1@FmO_q#2v&9l3 zvCTm*Z%lqQpWr1*o=kB$r+2zN0;OI|G80hg!!N#9k%7_Mie+42BOPE|F#W0D$YF3za}%Xm z8;tKhB}mV}t4X8b>Fv`DQt#SsRR-ZK*%ZBUgPe z_=k$Khju&Mjr@3?mlx{tLL?LamhXydJOfT0TY9x!BxRBob#D$>1yaVw6Py*+BF*-d z>p;pd4PoEV)uNO8>V3fSHo97!(;i+^O7zIO<%KU;-sgYO0mcQ>?bo*lgKPTyO=;E! z;}f!mXTYZ6>y!?+jklB$$I#k)iqC+}qC<3mvBC8E21h%vnV6wj8;o}p$B(^5Qr0}5 z{VsQ6^f!-VK9M#_+5= zC9A;tz8o0o5{>E9qmK)sLcsbCT%!Yw3#J_ng)QKk;X{;WZ7^=@*Hr;ISXLOr%7+7aV12_|9=kofzM0NUv^RG!0<7;)CLLg0 zFr6ND{292WMn`GZ2ICid4^{%3mX?BU(U?vGW_>r3As)#n!W_K#Vy2^GXVDgo=eUrh%X7fi3Vuc!sr?5UwNYlHFkt)~&N znfuOpkBzs_tnFJ=on_}v%8JS!(GD~#cdFNZFG^?j!-8xW0Y k+kj-nRb60xO$+G&+7?M^)&}F(Zu*!12Zfs}a{vGU literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490230.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490230.0 new file mode 100644 index 0000000000000000000000000000000000000000..bb6dda668e210e447e58e22641a5034e992ba247 GIT binary patch literal 2669 zcmZ{lYfO_@7=R1j_&(&aDmoMjQfQcAofa_!HgZ7|;1u{Cf$x$3}; zF$q^bzxA5C(QKaQr|r}lLqzx96sXReT?$S;yQHI8Af-}`3hqa(0x9P%CpastMzVBQ z97s9xj(ty8jbhIEtq03%8n-(pJ-ntA=}fOF#FQ2hmKDbE_X`uE z!1|7L-uXo^rbCA=+*XBw_5IsL2N)MjXG|>k7F@HYo6@We#F+Rp7R^A0Wl>Xe>n)wUHVY+yKu}~ zVr~(Ld3=cAj56-y^1s;!VipY(mKDZu?+N`Au)f=0KXQ9`eVGmovOmr&1nWDdp#zKy zrnjc_7lLap(NdbV!T3CNZ!xeL{VA_iFs8$Xf}8TAfz9S?bbz(7p|Ul*{suNF&6Ix6 z#)h`+Xm|l)+LY2Zx6L&8GWp)m+rYV5v58KBnD;W!UjB*^#2k(zIHQc&(9ytZ5Oa>4 zu&gkK1F|MogY^|1mbJTuz5QeE`wBhi9yNpY?I@uGj0>h~qizL(Ypy&>Y1Rhg507-g zm-(0Uvd6~T&um>Ks{AI(32drk=m2AbY2R9-6WGXPlxA%(-q>9j0AfDv-|lknW;%@N zQ~zoPI9DM$(jgF2l!o#<$9zG|;u!>IlrcN+zb#imOzljV@c;k- literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490231.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490231.0 new file mode 100644 index 0000000000000000000000000000000000000000..df12fe86b0b0ee52987857203de9402284e06206 GIT binary patch literal 2669 zcmZ{mdrVVT9DvI}=)F1x)cHsO2@eTEHRa(W^C&nbgkSyp0Zw3t_IR4#vA2IV-|yr*zc2T; z$r9&h>SS^iSFmJr+v|jZgry&<1y#OIp0r1+D_p0@kn>T6TAe{JsP!sABRA~cxl<+V zlNSm3g<3(bouRUF^9!_D&Tl#uk86X-x#_8&Eaq$-k}EUkMlRs}`PqUtN3Szv73d9y zg_4@NXul$+P)6|U9TfpwcQ?WP_<6i^yH@97f4ZO79zfXam4hK%R}W$Rd0(E>7;K#e%S(Jq2N)Mj&--#j1FnfCDb3o< z;E}SAfz5BHk~$nV?ouvmK&syV>VQp2B^_XFFzxeigB95D-%*;i!T7z0iA76D%ChIu z2yS>_%xU8h31XgW9NPxYZ9O<~RV1cFiTqP6X&|P(gy4)aX7`Wox&&fI9U&|$jN#B2#yY#w>0? zXHs{J0vmY{9bjxQ9V;k`fK8{1W^FJYVQWeOF<;G}|0l5lF&l$^w19Kx4hLQnk9lc* z#bFTB`jp^|GVW7T7Zwg;9)3nxRv5#>39=ZlK8f;~!^7>9Y05_BwwpSzK5rErU|cX= zS@p{o;F=vSnzh0BabxQ=u<5_N+bkN>RF7VMh86&u@^(7F+SDWaSpOAZv&ls1_iXCX zou(6^Am+q|kn0W`Pxxf^mL`^gbBz&exgkoJQuPoaMmAE1V86HkkIg-2~qg11_4i!FWrVT?1m8{?6@j?8XbmJiGPR zH{h02Q^$HmVoK7Hv^*&k#5~~nwG(HQG5aDuu?xh^^CB!OjA8HiT=<8hD&U|+G-gc( js{1Q%1gvj-8699;FrDF7>jAD=9ZYG~2IIMX>#P3<$z?N7 literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436569.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436569.0 new file mode 100644 index 0000000000000000000000000000000000000000..0fa703ac99784215a51c17c7daf786829811120a GIT binary patch literal 2669 zcmZ{mdrVVT9DrLe3-_X;b-pS^h=2{zl8|T=^#Bo&h?zk&8sZAQTC|GA+!lzuWQiI9 zk%EzEM8OQ)QkS7|tTr-U(~1@G86ye41HKW|;3ML8XirY_JNEX^=KGy|=lA8_Hc96G ze5+16&P`FSxbQKqH%>W05OoHlCLvdEG)G6R(eU%kdZQ^@6tZ>VW{oLl#}1viOOq!W z%z9CyAEVMU4O#kh_cu-X-yY5;=cXjb&)}LIvVX?TjrfsQ@cyEHQ?}8Ro|SDfO_kM8 zM0{O^Um(G|Iu5J2Gu;HA%m?s(Tl7W`JNU-#qatiZT85m@RN9Mv%$2<11Fz- zJQzY?dBN)uuDJOZm&YWz7n6)=6cxNRCWs95_9>#cX*N2*xL~@ib@V*AX7~k4vo;uC zRW|D`u&G-z>bCLmViVhdo+(yZfz2^Q2N)YnTWz6fz~*oarCA${_oq4?)5%qz^?VBA zs)8=Jx*Cn=dEPfymo-)P=)c`*0p(TTRMn#HHi?w6T682MWDuluE+sfCtVMgqYpo!q zV;Nz;)77GArNai6w_M)RKIY*yrA$Co;W58~<@t=G1B?r%Ka>=G1lMfy(5wx{`|W*d zU{h0@(BZQ2mNK{zJ@^oE3)s9nK?fKcOz#;g>Hs#U>L|_HU_ANHRmC$&%JN}9HJ9?_ zz1v0}0b*KT#_t2?-r4)PQzE8Jiaywp~6MUm?e#9S7M?W z*aXg{1B?x(bNbHw4Q!kqnzg~WrP^{H#LSyD+dXD~#d4QeSDn`hqiGx;(r-ne7>X$`=_Tu) z9t77+Fi@Jc!MH=bT?A}~J992e#%ymuCu1L+1UAzy(E-+`0lk$M_ye035B;4@19IAQ zxgh5Cq&ZzK8(;Wjp4i@e3Y_~ca?urun2$Ch#o`;MLCk;!1ZR{n%g-F>2QeovBrGe8 zVd3_+IIzAgmT;#_*xN5=NgC>mh=~O2YpkXNj0>h!T|>{oHS-Qpnzg}r_#cZ$fX(W8 z&2AfSui5&es6eHj0BoW)bbztJ^zP&^0odI4(5wx{_iKb&5cAeR=2h2j#=)31%dejX z=ZZg+bxXvQr69|OZRbGDN}k}1GGr9WX=VGRGO7`O@6w=t~vnq6+>-P4J=WXBql;894d!HY33eNR$98{d*^`SGb ze^0b0ju@aZsdWZfQnuD$Nf@t`3DFjG?-H|jb`&uu6Q7tIm?(FMDWJi zBf^|`8XaIxKo0Y5NAHmjf@y*pDDW|HCV?ul-NU5Ama8_7_uJQvaK+3o&g#Axfg;t#B zfX0HP_9T9YZvBC7* z?6?qMlQV|WtPRGwvwk5UX0O1(w}}miIqtEs4eTBDL3&3#W?9BY9f-+&A~>Ur`^c^& zoC7gy|FNtvh8yJi6X1L!(>t6V9>1A2si;;_b`zZM;wn18xL~?{MQb&9<^~<5SsRQ$ z2@a?RHl1}@4Wcn?Y$#z=RROT!Z_@$R#)j4>R_p>cC2smZ8yi~uJ~S4@R4T$6oi<+Z zn;CsgodNa=vGI3BV!m3Ainezz05R=x1ZR{nJFzf78pPZ&fv~JFhDY0sdEk5*`y-m1 z!k&IHbC;sJD0vJxU->~gz_?&K=xf*~@J#C=O0zZ?FPYoY1#IR-SGsIG{hKX5hV)IF z_W_&JGCIK6VEXbN(*|I(EQZpo4aQ&YyO0ZFw!Tfj=e$jC7&F$g2JX%8yR%s&CYOx9 z6jnS2F&#bxXOuC!O(m-aF}L{=mKDaZRWSR2^X&-VbYC=Pu@be|+s=dY?T(}aj0>h8 VEb0>u&Sx1*Y1Rhgz3cBc{sqSRBe?(o literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436571.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436571.0 new file mode 100644 index 0000000000000000000000000000000000000000..e98b3c4e23c223bebe0baa98957001f7821f0397 GIT binary patch literal 2669 zcmZ{lYed9DJDuukw!+u+?L;JPA6?<~-GQvmr+`tHePn zO>f8)(_DYy_Rg#iAbZ!WTt0=jHE?$a_l8ao{De_vaZ|R*l9rimu}tL3#~`7s*n2v` z8}FAWc!Qnble-+yF~{W7fd@QcU6ODrrxGBYlHFBgcqg2=G>A$ zE}LNasAr2g`+lvk`E|`ux7;*_b^ZRg0vo;t%_NB@JZ1$GL-*?)0 zN;%^^dKs>+05<#1(E-K=)5$6Ea$r+jL21?oeB(sI4sgCHuU4Jcs72uhzZkn~hc;Lr6nJJ<2BldW zjE_Iie-YUDKG^n1GG?t6nHLtU1vabd=>ThEMZHBMZUP(s21*aHv7+{m#wQ?VTjH!n zr;QwbGq>a>+Q8ngVUbM|G21sHo4m^gV%9|voKeQ?yz4DjLCjTAgk^;>{Jip{44iMv z(cosMu%}wx&_8zG#tX)bP(~xLH)!NRyF^TG z4KmI#E&(y)eF)AdW46P)WG;x=|DR=rF&vhk6bH_iHM`)6WX$qpR61!?GdN$uTspwG WU^?X_uLaM{2&FV@gYn+a_PBpQ?ICgi literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436572.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436572.0 new file mode 100644 index 0000000000000000000000000000000000000000..bf9f983bfb900ee5758084c720c3e89c5be8037e GIT binary patch literal 2669 zcmZ{mS4>k;7=T-F-HQW?1Vx1^h*-5Cj-b{fxWE+^qG*Gq6be>L(U!V!fQccBkwB~j zXAR{Ic`#Cq3y>iy&bUycxIhgUH)tYwXipCLkG*|tzW?Mq|Cf8)7?JbSRTy)EEAorJ z)DhJbbbfPfcB4-KId|of`NO!Y4WcVO=gNli65hk8P0lcw67?A-laI(c82R;Ujvqd;q#+=zRGmGGWCm!dNz_rm3;?8T!AEY52C z#ci+1ad>!&U6`n6qQc1iNBzk_(+f#B_t8!V7#B>petWnJTr=bnrCA${-<5s14{SEh z`s=iDb72!{LuX%Y3j{W~3Oc~pV7gi=ng?tyAWE|~7(X_sF3y)+we?P?KPL^ie$~-v zAkXveTaxrXqUP>BYGqdfICVlqW2HdKNEPau7Eu9GT4xcQ6;`3}gzk|brCCAP|8!Nz zCeAGd%UdnJT-D>@GNnj^7JadogXLM>=>X$`>F7>{8@T40K9pu{Fy0g|wE&yX#}`yP zY+R+3mZI-D>vjU0*C*%zV}ognvhR6dv-cFGSsRR7-E!N8lazU12ZV6SH{YE$Vi|~8 z_t5hQI5#c#_cei-A{A0cO&SJb&OJnMMj5l^)7QQOF}LOumKDZu(1YR`V0{6PpVkP* ztkR&Y5BFoh`pz}c0mcQ>3&k%Ff@|8IQJS^E_|sJ_1A$F@kfv5JW_BskUG!1_o2YSg zfU&`Jy3{fS*gWc`SsRQ$$|`9AF*l9yt|K-eX1rvz4xH=T=3g%ybDwO(We`*Sh2V@b z?xWjVcof7;`AS$;7{l$2lgEJdMW%joc({BrtJLVGt)m{SFL6B`U|cYLI{W!fa7}L= zrCA${FAge712%p&=Ieqnt8D1t(*AZ}(^g9dSQ{I9qWA3vHs(4?|7T-EKTf}|1u@^o z1U5Kq+~JccsoB>G&ixr0aYGYb2@H-n0eC(%L-$7^QUDB zu)eT@;F}I%SHGCqN+c_(31=GrcJ09Sge|u@x2IHOKlRJRTqKOqw8&|Jc zYcXo^j*A60m2x`3*kJm=j#Zn0&6-e3vo;vdtU37y#7u5Yz2(?V9~g6aelomeSib{} z0x?A^QSg~EJ&2jd6P!`TtXVN82*k|yAS^44;Sxy;{KK(s?C#rwF|D!4S{tYW>#H9} Z2N)Mj=d9>o2G(~tgwm`H#>>M{`#&EJB|HEC literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436573.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436573.0 new file mode 100644 index 0000000000000000000000000000000000000000..d63a6b93f9cb9c9db8c5a2675fc2a6564c196a58 GIT binary patch literal 2669 zcmZ{mZA?>F7=ViqnHCv1s$>L{FDWH686)`4+8sk8xs6*Dt0MS%Ew8ulZ753K#RHqhbUMJ;U^-MkE&!XOh|;VL#wQ(W(F`Bhv}a3g@BkN7~gO zW%GOX1Km7Sb~{)OmUqDKihat%XG)m{sUGf62g`fyPX`zmOlO?2M1X7V44^b?gYmSt z$7chZCnvL8JvP2l#-2vCJ5pMJ&4YS6z}R5gwR>ACu&HXGG;4$LuTvx4ACr{T{{<#+ zwS%Kx8$TtCnc=<$&h0-q*(MQFmWKio<3teiY$d@NWy~th{ksXoEUzLgD~#cgUiDtE zz8QV}?UFI=8g#)O@dB*R)lUZ)7fkmaQci+vUUpKNwZXV0$_Zn-;x*SLV_Ht5>p}hD zz-B~22N)YnpO~9<0oeTdDWzE(jL+KIJpp1S&YN?C*npTW$7fn_u3{vnLpr9_wd!{e z6a7zcMj7|{<>|#MAm+L;!m`2`P9FQ}If%KY=$XgE=aXqyqjQzD31EHkIy%6(V0z2v zJyvi{?i)(8HW)Y8KNto!%i7C2C1cucX#OIZ4cPd1&;i!QhVGA57=g_{)AR>6Hso5D z+6-d0#{ai-}7#B=OAN{EcT=Ux_lxA%(KJowbW423lXJAjQvpaYBzroZU1zXCQJRFr0IFg~Ht4S|?BLq&gjb`t<&R&=be zfpeG7ta3=il&wbsFOGtkVL=3ElrbB8?}r)?^YMF@6~^%PoGk-jecFY4Zc4^%%tdm2 fF9+6_oInQ{7fkcxE8rgvE|Jo#4aWQ8o1E_essk&L literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436574.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436574.0 new file mode 100644 index 0000000000000000000000000000000000000000..235fd35744707fc3a49c44a197d3f8f9b69229c8 GIT binary patch literal 2669 zcmZ{lZA?>F7=SMmv)-#%%KU^u#S&0Liejb+as)*|-7@JvhXSo!Oa~Uy%f}ds%#`>U zKL~6>6p%rdCFl$rnPNde!4Ie!6BULDtYLu+K_?1s%ch6+VOropc?z$QF~4lp*D9;y;sflZ%_W^FKj#!|j%5xMHfy*B}T zU%;(4N29rdAk50wXD#4{-X8S2I`JboH9fNTx=hO06jU3u`ZY*7yq@5!a0=o|ZBIbT zmobEWPd5cA)w?ZVdBq;q_8AYiDLDhu6&~ln@{W1Z0mcQ>--*AR1lKH|O=;E!+JKIPJlF(m?q8q-j18t6KcA-oHh1eN&Dvmmc-kl}At{glrC82? zYM*r4c!YwOnU~Mzfpae%{<~8qCYOTH;>utUQ+1Nyj520zqxla(%=M=T%L-$7{MFS% zV153N`)i-w4@6qYtyAEM@|Cl9TAmysWQLw&-ayr1cVER&g>kr_Xx(Z6OHW*J{U49za zYzu34+PM47)>fj+r^cTHn}=FDz}R4V!$49Yu*nIhG;4$Lz(301XV!BxqsOruPZ)Fm zu!8MuV7YFT%3I7`6@+aA19Y!NoS&n6)}I hwliTTSf6qQ9bjBAU2`F046N^pi)L*wzQsG7dk5zfE8zeD literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329253.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329253.0 new file mode 100644 index 0000000000000000000000000000000000000000..38c9fbdfc75c204295e5fa659494f16288b13ea5 GIT binary patch literal 2669 zcmZ{ldrVVT9DpxA2KTCfU_nHSh*1<<15x83o)oj9iD3~nQzj|2T1*vMx#cltq8z&JqXZDjgzNne7VM3_DuQiz0_~83h z8i5zG8*~*@SvOwxE_Wj@eK`hfN{a}cMVI!!8L8iDb3no z-0|=8(ZD8V)(5Z6Fh4dbD_XyA?@eG+qND?i4W^U3qP_q&ve}eoZ7`nOAAB~FT=m(l zemU<9cG^9SMhk*4yjYtj`B z{{b?JHKjA7&X?h+8Q`I}ykhu@d!$U(c+PvOf9$)p2}3#MPZ zX|Dy>wD@S&2IIdsMsxxjYh#g9GN!|dgbSPHAZAq)9bj#&=zPTWL%=5OETuoQv7*e@ zC<};bn;+NWu^A3uro#EP2Ao?NJHJ&TW>*$!nsPS<#57MMIHQc&G+THyh^d-RSXLOr z>MK(pfb~^WPHXcB`}@bN_!hMd^n3}{cY6yRU|cZ$Ygfd0aLt-3O0zZ?PkMPf0oc5m z*yOeG_cLqx8KviM&;pzLayr1+V7g$}(s{rpJBHG%4aPg`T6;mvIi0!fp52UqG5btY z)`N3Z{o6YvVsaTsF=tg7i0S@Fa7G!kdkQ+zKuqf(VOe1ef5>u`g7pPPmfe($X<3Gf jS9Fg6>)SYi4lpj5zMFDpHMr)sXiBp-7(em-!grql);cW3 literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329254.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329254.0 new file mode 100644 index 0000000000000000000000000000000000000000..649785bd9719b30b0f80d0967291192d8ffdea46 GIT binary patch literal 2669 zcmZ{lYfO_@7=T-t==Y%((1IFTB*=AIW;2Y{cq|42!C=Jw7$O#0C<~*M*&@VL1vJJ~ zq8Js8CMs%GVrB-rU=&*c#Ue6eOk|SifGt7A%Pxv78q`C3Jk5LT*PrBh-#q7iQkp69 ze%>F*JkBkVWVE|d2U6$F(3!NQMn!s+-dMRzxlX|^tkfIL6((JU)>N!8Z!Ry_nzksa zO{JB3lR`g96_k`3^ab9(s8HK6jelap_z<2eE;SeV;E)TP zBbvZs%kwu}__;cdM~K*uN#QbdG%8CaCj$cy!<1Y~GaX=DFg-eEe*~V{>7!X2jN2^Y z6~N~1q6x1}fFB#R4Xuo@Gy)svS9E}}!SvYLFI$1loyC-9Z7?1oyEhm?p8EKXTh6VQ zI31ow(|Mi`tkN2!qM;8pmX;?fuvMAZcTFIrT7#0GE_ek}?)aMEtgr?R*95sh%3Udh z{Y+Pba^)d1aJ=;Y+dC#b{H7G?&~U&%C&BTu#^?a!g6U{$K0MyTH`m|V*!W91|1A1*VB}|DbEJU|FgBQer*A$4jxOI%Y1RhgZk@GvE=gJc-?RkIdGD3i zMw|pUA znnMQ|8%)oV^k)K_dLPZ&V0<|6cpZrOSMaQEVgq8P%ASeA-cv8+H-%%KnR*pvo@1>a9=dA`RSLtx0=6fM=@qP@1*D`0``>3W3e*$lts+ z{{GE29YxvYWm;f!Lrw=68%&#B{c*r1JBre*4aQT_Vq!qd!Vp4j2i~6FFyjiLnVg* literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329255.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329255.0 new file mode 100644 index 0000000000000000000000000000000000000000..1e90c7323d97d2905d92e2daa767dcf855ac2503 GIT binary patch literal 2669 zcmZ{lYfO_@7=VlQ!uO%BtHYv|x=LmY)2{~1(3+iKqERG^fGe3;=>-eaK%2A(!&_x8 zP6X6=AvjSHii>1fGOr1EDQr{9SQE*N#i*c((zs><%M?t!9NOa{@3CKhlIMN%ocBrT z8cF!`=J1-6oOH(OwwEhvGps-9D;YYAB*SGJOuxk|A>z+u-L{~XZp7f;|Xij^d)dZ?^!u7oQ)HG!kH2VZc?vlSr^SyTiQqZSQpdN`~i1HJd3#C~b zjOVvn3xUmN3txwACPc7Fb)!m)_X@DtC(r@L2GixboQ=RHQA26g2IJoeKW$Txrw;f= zR9u$q*7Z=MsXWh1OHDocx1Z z56;(f)Bl@jOph6TnWmP3^98+hfN{Zex$EUq@Jv@1rCA${_hq+i2R3c7=HEqQ?rui* z<-y6ord~k@7#mF2X$z}?4T_^QYlCs+5uFjlR7{)k2eAP$qmGyZV6V1MbyGa%qe;t3 zLCoM&f-}mvPycysHHfJS5SA6j@bZB&9XQ|B>}Mg5h~G?)5iRQ|=m6)7-#`Z#7fe4- zYBGXn2FGdE2IH>t3AMnc>B81qqA@*g)be9}JFuy_L_q2fX74D-0j0>h!RafqTXEs$*nzh0Bv5q(muz5b` zQrITazuAW4Xs5}$1lWwK=m2Ab>CV&ZoWN$!TuQSx81H)U8{ZI|QO0b{=xrB>S^t)>tT2Z2v!(xl^Z91(`b#usLl!!9 gzd{YpS2>#wFfN!@xT|d7nV%~u&DvmmFi&6i4)yUURNDu0p;(guC4$x-orj+#EiTpJmW&)9Z}6g?gh= zEHceTCEdA_aDsQW*GRa=R)T-b`}4j#G&(oiRN=^w5H@e|qiC+Ojj(}y0MF%XjoaMt zkJ%SZ^9bx!HvWp6+py2+5#Z&)ByR;8`}(iv5oBQSVW5nwKTii37fe4qWif+muD(EN z)&}D?S@2C@vtW_kW#jF^CdrJ}j8@bFo6n?lfU&`JQgwAIu<3QvtPRE;k2VbkldF#X z`8tBj^1t z#5o|PLQ2>VbaRm9NP`bp-qPo-wrLNKDMf135vKniEKl{44lpj5t|_|9gKLhuY1Rhg zXL^pu1DmE3DeX=hPbs4;sO#=gIk0K0rUQ%(rlT$;Z3H&)rzp+ZU_9CSwl0*Uta|1f z%f;M#;j;0H12GROS3U*j%1hq-CJ11B?x(Z*-jz1DlzllxA%(uI;!N0AkMe3%W^cK+J&#g&mx0ACBl0j%l6p zt_LyO9uu5V#(fT2?jHj&YsU!73S+oRsT>3A8_N98>EZEZ+Varlpmm49`mSZs0mcQ> zKW(`%0j^o9q%>=T@%+dyy@Ac;=3Tc0W7^DU;ok4VfK6}<9bj$DC~mUzDzIsC(;wKF zk=Ppa1jI~FjP7#U_`sKGugRPR&K;h&yjvjVNH&_#MK1#}cg7H$QO4}h;dAh1#>Wzt z6~=ILTKF@tzN~! zJ4?X1YhEAi6No8NpyYqUz5y}I?F46(G5g*1FMa?ov%*1GRv5!~R2vGw`nGi%<^84Ln&(0(&Dvmm#?!)8?*VA}DZc;! literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329257.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329257.0 new file mode 100644 index 0000000000000000000000000000000000000000..87b82da0e1dd9429629ba9cba5011401d74a9ff2 GIT binary patch literal 2669 zcmZ{ldrVVT9Ds{Y?nQ+a=>%=jK+u?4M~%aw9$q6u`D39j7}OPd;qr*k8(N<60YtvibG;x81DJB-f#>RAvM>27T_ISwe*xNsw?|1T@-# zmb@a-q7x^n^z6KRG2QdckmNFil5!`5-2HaKD)|7O%gVE6PQgDL zQ1cQ3`|KDT;WoBZxjh1WyqIJ}BP|y-rXd48y-Gc&KTii37ffe6UfclJjJrT-)&}F@ zjp+w~javK0V>8W*O{@()Pq6dA#&-oBU~DiwFdn@R*u<}-G;4$Lb$hcyz2N)Mjm)bWU2iHs*p)_lQ@h{3W zYG6}wIMLy@@s@INGkV(7e+AeasiOmo4W`Xe%1U69aFo)l4aT>3NkEvmC@6Ui{HQaBfQJ_zyBM6-HF;S9cb~JhhYHj520lw}w@Kn3r}DmKDZulyLJo zSl?(z_m8qM?E)&>OflXVGa7Q*~Wiyi2 z4$cBLPZrPt#s<^5qRhEq3uP@V{fg&G?--Gq7O``*h3#L;i zUbcX1_D#{O4aOT6{dXAHe0R0jDI3#nLxm?bfxxEq8XaJ5Y)EB{nGbAC*D3v;jSZCx zf7gPT18c(WxoxJym+2bc`8hcEt0imh%f#&4h7wCuW8h_uMi87)#%%YXe?5q4TS{0~ z7{k{aLvz9UYN{eS-NN4fF)O#AsFC_%u)eRW=>X$`=}oWpSAlDW@1-+n3aY)E`pd%6NF`jF???IolRhUwW{3@Wn(rN j&@VPA4XiIGm<}*5nAY#<+zqZ75khI!2IF-f_n!VA$Ui8{ literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329258.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329258.0 new file mode 100644 index 0000000000000000000000000000000000000000..f850c2ed3f5b4b91a9a612283ad7e9912f239eb6 GIT binary patch literal 2669 zcmZ{lZA?>F7=R0PgL^@Q0)ik#g6j-1WhtQ(loO0YF&afsNhc|7rI0GL$!&4Nk0FaA zA$}1RGhMoYRMD8J@q;XZX>qJp1joiQA#s8IFc>u>h>CL*JhUgLc@Mq)dGov{&v~ER z+kA=dH+>@C&NW1?>zG_Uv^rtFLC~Ab%Dg>Bvn^XysN|R1jApAvFj(|LnbKNaRizhd zlwSxYn^902XQ|S1Q>C%g_e9lww;-9EyJf>?@mznqMv`yF4Nrcs<52tdMI|(b}BYCdOWZmh9 zpC7Y^q!2i0--|a~>V-P5M`Ta{6J0hszOwZF46(U1=A;j_4VMIr!G;N zwZZt-wjMRG$y@I6*#rl$$#tNw8_q%-K}iP~8%$@nZp#5SIq8&UZ7`mZkeCxkt~&bg zt(+T)xOdyzD4gec=^lM$tYl=SR=#Fo6F9X>*?UJMWv&*TJ~dbmQhu08a8_80WFgWT zkaDS-uyb^^sQSsZM__rMzU*+$dIU@H3 zE8YxjzC4=O<+TZva>;pgd#^MO*jznM2N)YnFR9rd43@sTkk;tHbn4K!X4nPRm!+Wtj0>h) zt~XVHYyR)2SsRQiPybK^Y_8m>z9$;f=|Hdk2;C2Ce!oQrSQ`g&W^L>SHkNiu&#`f! zcU{qEK+LkNRJYeg3g67Nhhshl=W0?`+!u*CQiA$iPB)18&qoAjlrj5BO&a{n{-hu* zD~w^`-}O^qeLLzDJzn8J|Ck3qM-ExUeXzcfLv(;~!Suw6jAP)M_HQW7+F;zZ-mnVT zNRw{)Yyy2|o6n%g`JPT-qe-R%j18u1cco7Oo0oo?wZXWeUk78J9V~y~-OW50^Kxc? z1~_-Y+lF3|n365%q@oCcn9Do_XOuA;uF|A~H*?-JVOe1ecL^=!V0{l4e$^)$v$+s` h|F|v%tS>5o4lpj5mR!-bfNP%f)2t1~RWobH-UIV}BV7Oh literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800064.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800064.0 new file mode 100644 index 0000000000000000000000000000000000000000..2465a20bb26056efa0858701da3f2159073d90c8 GIT binary patch literal 2669 zcmZ{lYfy|)7=SlLeP5xLCQF-37ZtNiq7jZ7G{v~Z$fenBs}IYzJNAo&MlNAcVnm@P zW1Ask%ouV>aL&vj+@p!kRXuw{2lH~?-6W(K4WdSG6h&`Y=>Vi@;Ocz|UU#cN z$<;;@+>0N`yR8xoy{xR+_n?xnN#ox5bE~EhR?d6!T&hk?>4lH^gg1;Nu+!!b9o%F8 zoi-0oCkG};k*MWD=L$bE(ELgs#%&6s1B?r%Te}X2foFP+rZj7V@!6ff?f@I^Ot;;p zj{}=1GZGhk-2!a>l+yvm2GhH9it>TYj8^3=E1pMKn@y_OrcMjkxR zJ8#hHy=86PTer37;=tCDQ`VajDWlY=(LW>ur0hIRa8_82-c$zM11YbcA?$y;YE@Ay( zqm(1c(C6rf&w$N2jt($3n4X|KnFDMV@RVk4FkW_Xh^vC6-1o&Tm=or7*lnCbLClz4 zzVpCd?Z-d2C1T3dXiV!GBZ%4fjo^$jX62cgF(77IJ7HO24EIIh72tgBQ(xYZj9HCi zzHk-6`6{N<0mcQ>Q<{Wf;F;-BlxA%(zOMBB4q&r%U+P`SnAv5>syzP_*sRT^1B?x( zo315v0ULQ9rCA${FKHMcf|z2qcMY)tF(0IuRdV@AYgnL*4=2?S@9ai8DL z@1BF0YR!p4@XS$GO0zZ? z7sj721U7?$Hr7hUtTv<7Z7Kf1#xP^6(-M#mV&;|y*4b>F z;WslxbL9ZoTXZPWA`!DK5oIKntOhab3kl9BWA^&fU-LlB8OI6B3S+ptuto{aclL8| zy-nEBFJ|^~BtM$l3eH#2P6rqlOfN38z?hcrlxA%(?pfD07}(6uyKJ{{^l!Gb2!E}w)%>s@dbdb`l4aWaI(1(DSzA?J{w%fSCnD;x16=3g=)cp+-F=Y$U z)Db2h5HoQN!5L-Dj`1qL0AfyBOITJI!;AA=ioyBZw`Vs>#w?9T34Z-&fb)6oq63Tz XrXS^m27+gL?4~qpgYjie9xMI<49+J{ literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800065.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800065.0 new file mode 100644 index 0000000000000000000000000000000000000000..2ca3d7f3418b8d1c0b17efe05f62ec9994530329 GIT binary patch literal 2669 zcmZ{lZA_C_6oA{6DDOK&WGli%N^PhNuqG@6VR&hiiFFMSSR&3D3YBV_)Yi2fIF`*0 z-DGs2kb&8vkqOQ6KorrZsB4jO;DYy&BD3O_Thzyf^82K9YadW;|%M*!5q~9Y#9K@s~6U8d_j>nOKN54g;a^p%mz_?&~?+-2)Tlgn8XPam+}^qD5j3U9fNJgiMsU2(t#p8K!SvF;mM_3F+m2G2 zwZVA9%qRzJ{#>K&^4kPU8Sg>K>ik7u)5OsM#s<@&!ynuNHZ44*SsRRN{uvmFAt_yd zhbD4ay4iq@I0?iY(jLwSdryu3+bs}NRE)$aXV!z53ljuqlrd}QIgk!w)=UwW6~^!# zL;4kPzUx_!ZwtokG@z(!^KsyOm$K;q&^N5E~FPdniK=_C^%QdxT@Y zv9Tlp#9S;SIHQdF>>Rm}0b;s}2+Im%`27zzw1e|a_xb!DLBE-uB`D%G!anQAYkSsRQ`y*F9}Y~~Y=^$N!9bR%te?HOQmDv=JbHg2TbBbfmXJK}^|M z!m`2`&VB2;8=SA@uf%(P;b6a*&iyFATc!r*8=s~Fj0>jSt=&oBnM*U2W^FL8nLDos zHn&_i12)0_%{G6AhF&Rm0-NMII>6Xqy7xJ86|m`Dp;;S@pLi0r1;iBRnC|;;69Qw_ z#1@W%y-{VKJP?Q}DnRK)_pX4L(I$d3%9!2e8k2*Vv;VWKFouse+QY&5997PK!I;g3 kX#e+De*x#4tEK~t3#JvjpZdTvFMLdC)&}E}fg6th02fdti2wiq literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800066.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800066.0 new file mode 100644 index 0000000000000000000000000000000000000000..b47b770ce24d89d515e84a6c1832dfc516da7fdf GIT binary patch literal 2669 zcmZ{ldr(YK9Dp|^?v2oBJSHJU9<7;Jqs1Yq$z)JQ&0w~+x^c7Za<{_d(KJG36(eB= zVe=R>=pT<+B~%KfF^o5PXS@b&)~lR$&zzm#v3LKx^Zm|z=l9LM8?A8sI(J1M;64}5 zwfqQw7(TRbvZNLD%E)X%pQ#RwQ}Pos1-;QAB^$I-s?wO1k)f5=Dc4A1rXVSWZYnWN z)Cq}>CvBt02Ms6Z#>UJU%(YE+uj)Q`+(6!k_mYGZgWi~^GZ>A&igIsMQh9ZkAHf^0 zln&$4!w5cr@5lFEF6fGp7$Qjv^u3eIt0c?||Sz_?)Ab49{HaLw#hlxA%({ziNq2y8BTMqjbp zxJo&q0v$}B4O2#NbbztJwDGd39@so`(yR@}b8Im)hLDs+Uwa30M`Ai0Hg4lUOp6p( z3C`{L<@Z&Ymx0E%l0XAtxskO2(b1RU=YAyjbStdHb*kHP8 zUdv}-6S9NStPRFP_aCbQF{kGF))5;JGc5H$7C6_saAduF%rgN$JweQ~aRg_Sai30) z3x7e(n+b$vg)zLdTx$jEYi;~!_i*`U)+8a7@FE|qZ$lFuU|cZmf7=!guDPI@(yR@} zue*CS0~=e=+Uv40Ys@Ix+XQ1;g6RNjV@6xs@2P-|FqYE)*_hE=am7>+GyhajgWbj* zzL^DlYydbnVyC)MCT8n06#DqvEfDilDZv?K%uas4e>I3%w}-H-Fou^_Eo9Ds`$!^|Q@6hQ@qsHmV-5j;?yU~GbtxLb{^YfOR4)Kq(Dr{b-6VM0aG zh!L`|O{@w^%w`vlEa51gC}5VjN4iyelOqqeVLh5ndjGY zB(;>=R=BYGeTp+>Z;1pJWk?qRFf=Wtp=T;s%2u%e+~!(X6uw zW`)p4WoUJJA;a^Jg{%8@G&xt5wjh*y8}D1wcW%ruK8XLpEUYpa#SFbk6oX~905nl^ z>t-0iZ#as=xs2HaAHvIdztw`VpFKL>P!>+u%qfq?aG{BW4dMfNE?XzA>W8hWrfE?G z_SxFe&9#o%>-Grr@nVvhh{l(ht0TyO^L|hq7aL6n7#B=GkIw7>*SyeAvo;t%Q&?07 zY!nr}9-9GPY?AG$bdmOVV1v%m0mcT?Vp`~1U~{FPW^FJY`K6;Kgk1G`%ZCW=H)~_9 zyOBT7^S(J*`e51P-kn`rIvc>LjuS0)5-F3_XwQ?jPavhDoZzgm8o3_S&IKt0DhT_X zt{Qb+OHT*O`&nOG-{;{qrA&hoW(fzt^4^%}0ONw`aM8L0T(f5#rCA${YaQi(1Dihv zr8?X;-cm-MMB4+LNxbS1H|n7#orK`%HIjrw>6m#FfN!bYi^AK*IbrDY1RhgLtII65L0W-z9Jbj_av(BvKImyV*wpt zY%qP>FjNU_rW~X+YlHDaZTSumbHJ|PtHcJxd^+k>JUG{&in=BpGxidi0b=&16P!`T zeHMh2$Ag&sa>BC07!FMs8wS?*tg*xG;q}R^&qO`{4qFY@*V;@67#BY`>$n zf%P@MnQ+rB?Cp!0y8<1l`1As-uiHfj7#B>R8ZL0)nz!Fknzg}rVb!C2U{hng=&|wk znza?9u;BIxU~{*C4lp*D4wD&!37X#+9sYY5ILW43OpDi6d|t|crhjA8Bkk5yoOf9}q0mW*jzg0h#(jbMG7 b^XLHMg6VZ}GmF9c^7d1jwZZuHLHeq%2+b_s literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800068.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800068.0 new file mode 100644 index 0000000000000000000000000000000000000000..560abc9ad08b0afde6f2f89f99d488427107182b GIT binary patch literal 2669 zcmZ{lYfO_@7=X)7`94r9$h>g#G8wB-GniW}dH@lFAx2^12$?{k-$-3)w-#B^fsNt? za55V-Xwry4HCi=JOk5PPWw3FXY|~6-a{*_9i(&J!;fN@BXirY}9{crY^Smd|d7qp% zP2&2Do6{^@k$LTv|C0NY=S?fnE7e+gYN?V>AhYa%I=a?y>(@==g-;VFW%!nue)xce$d z>35p2uj%HZi=9K2V0posHv6Q9$CQ!+bb|BW3znxVrUQ%%rc-X_T?W@o(@~nW!FcyxD!I4xF3w_gK3~Oi3=ZG7O%teE@hLvZd>v!Yuh=HN%Dv3=8J5OcJi;EXb6vrfueK}_or z!m`2`{w8w$d9c2+C(%7lVNd^TG;|(exN2eqm0?eVc{T%8M}qBtT2YN9Ywdm`qForu8YQO k$V3;6@ArZAt=dBe7#B?UmfVJKA$BjNSsRRNj~`Ec1x?T=xBvhE literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800069.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800069.0 new file mode 100644 index 0000000000000000000000000000000000000000..cf6681b7a70a3c459de3532629ad9d09d68eaf2e GIT binary patch literal 2669 zcmZ{mdq`7J9Dt`*dT;NMy|cH7)q0A{?BHOUS(J;4u$Y@|xQAMy!v_o}=c?u?2XYUhg06O+8`+PS^4$gfajYBXCcp_0O0=(}~Qawx&8 zuJ0SheVRe=zI<;!WToEZV>_~2^M(;tGx6nUt|OMPQoaw*=?s?TK6uWAy6Gbc9I&ym znX4O}>+$Fl;KxK0gZNoxd&9_p>xFa*Cxp`h#s$-P@tFzWn(aQCwZXVU?`{D$ql!Dc zHi3R@;_N7Fp0Ovek)Ndlj18tO0hzhLrouZh!Oua@BXWtzq1Q{A!1%Q4gNy zgR-^8P|2%~EvI?QT5xLK$=YioDdW^AX_f6VNSRqga8_82IQQ?LAmyZD!v3eLM(yM3 z`-0`kjSgq0hu@TvH1r|R+6D(@?j4 zhpvG2Ma0no#s$;K+y)+8^S+N}Z7|-nDKHb*thDKFipI=2g{;e>zXF@XJLmvogXt8m zX%VoQT0m*m2IGpO$1*_7$2p<5hz*G8vLkAcaOczLF;WdkVG;4$LaY@d*2S zW9FowvdHfT!1}s2(*ec>)6#Bk_{Si)h0?4I##{4MW?if3}LzNBq*fN{a}l@Edq RTr=59Y1RhgH774u{smP{EzK7-J)hz1prFWo_+t%(Ac{Ce|!5GVe`U zn^`<2h{sqJWAo^O2`0pZka;X@Ua@4Hwa1a)(c3?7zTe4reqZiwq{RDc-xXQRMb|}M z{xPFrhCEO!XbdLR?Dcw6MtE4PiXWY!H(AVr)~pe9D$6>fQ6prjG6h41UQp>fsN_^b znm*b4#9DFW-9U0~Ow_C(Zs;2Ss*ZC*`|wgeK+va{O_t;|v&ABl*m|JPDr5Tqg4bRv zQgF@b1n-Qf2Bw!O$w0$%X&5&rfetV(n0_R+rGje)CsLZV!T9d*(cBPQ4pW-7!MJR(Z7{GYm^kN} z$HrI6U8M!*hHU8sTPfy>p%;1vrB+Y$$2`! z*kF2Kp;M+w%K*!;oc;q%ROsnL$NlbXQ#;#%kcuah41LF}3a? zwH}*J@Xb`$I~Rg;_nw_{TO?*<0*bgA9R^|!uOc|3jM+)*)syzY&=0})&}F3bn@3AX7O#qUC(Yh! abbxWe^z}UtjNqDY?38A0Fy5AkO8)?sOem)S literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244601.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244601.0 new file mode 100644 index 0000000000000000000000000000000000000000..0d2688ffbc10cdd011b3fea7488f3ca512edc69b GIT binary patch literal 2669 zcmZ{leN4<@9KbKd{C-|~QLAoTVOAIU;qB_5-#s) z_@y7A&qJl75_It@jeLHZph*o0j+FBOse(qY)g@@-bxCslhP7+sb(`erI#sHmlM4e> zT(U|n#99AI%gxR6BIiad4)fsVsE5=Ioa^VxyYM4)LZViqk5g;)dN)bwXe5i8su)Y~ zhO5SLT#cUK&b%W(e3hUXWcOtRyctJWW#H?HoGO*DF1!=ZC8_j@gK*VBG;}Ck}iC}3mPN@>;x}7<=LKxGOkZ%- zVq+_%cR4z0bguw5b7s&1#s<^1{$~~d8~>SG~uItnX)X^L5df zrUcX(cKIe)pM44)U|cZ$(R0jVa7}wPrCA${Pp;381vZAWi8n-JW|gA~^O^0yru`fp zU~DigTN0WEY;w<2nzh0B%6vmFh-r3lze#LB%w}0Z4>)&Ny7w*dm}BGG_JNq1%>-wZ zai3!8qh=7(V+&zfVGOtbzP}EvFX!t=i-*l8)1*ZG$`8R{eSzI{fN{a}tjvX(~frKC%K}BV9@dSer7$UEfg&Y@Q9$|JjtGs7>2;f|!jF zJ`ENdd-!B_M|bQ3=W0*SY!r#v5{uRserg3Vi!KnHQO4{A*{@CzbNfZYvcedS`O?(} z*4Nx-Ecg94R}lfe3-o#_DMg6R~)!!O{P9j=sSZ7{wrtEv&$9IZNMwXyY@ zEiFVH)k^rqJaC*2FgBQ8zN@Vq*q{@XW^FKj^3T>F5K~B8eaEsJ2N<(xLY@ws``3_t zS0tt+0-1fJ(?QJl*#u{lG0Qy+ZUQkM=Ma_^#_*!9q4i*WqYh=>6OCCKi4OiL2>>yR cE9d~@g6SjQ&prUxlpdorYlHD=FC#qv0iic8RsaA1 literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244602.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244602.0 new file mode 100644 index 0000000000000000000000000000000000000000..794c94be8eadad705aafb5f636acd2e50a097b65 GIT binary patch literal 2669 zcmZ{lYfO_@7=T-E;rmpu6$3b~8Cd|O28`#Z2_@~7DcD= zlE@&7GPH!k!ZM~3LqrD_1*dYE8Sa89TP30>3sZhf;{`pm$3xy@zy2i8`{p_ClhS07 z`!iLQT*vLcoqYM(+M%^e7b*=3txmSCRIM{5#AeC(U{TKqN-XlLdf3Clq7#Ht7ulo^xi06Udqoms8xEMF~3M}G%gWY zz0s==j(G(V{6<$z5La13a0&0jdui0V88*q^4%Q?5u0mcQ>8pn5kfoE3cQJS^E zc+O67Hn15Ro^spF@?eu_Lk*_xQ^4l2lMXO8m^R-V$N@Iz?@^ky!T1A-_HRG()Q1Dl zgE`Iktxi{?1w7BsE>#pQ5jo%NowKsF0c`!~_x>vaDHG+WCX%CMnMM@NZ+&Lc{@9=3lz_?&q77%_CJTv~=uTj>E{@Wy4aP4f-BtyVlt-R= zMR1D)pSf+sQ6Of9*e(Hk3j3#chZ2?*#_)*L z-~i|Qx$y2a!I*X>s{b-A7M!n3O9vPiOfM{*Is~4XzKznX4aTF&`zC?S_6w?R!I+jt zbjUg?0X8{pbbztJwC2>99@vb`(5wx{Bj#4yLCpGQ-yUKEV*2}ByA1X|*%f+SIHsn8 z-v?r@v=E$8#(h?gm(+ro85M+Og)#hVu1E&Xmpd`;^6>c0v=^YrgM;1ReAcIQfN{Ze z@Ufi;JTq~U(yR@}KmTt*53p(awCoqbn06Zq>s{{)Y+_RB0Bd7Ib!YwN0h<+Rlzz{~ zhHA}+-+-8R`onLyY-Yo6X5_d2;b8Bk*7%zOF`aqn(Z=8~5Od@r!5L-DPAKnBfS66~ zgk^;>Z0Vf)2RPq=PsA;ku%};4OD_6tbK4j=--g9>fN{ZeMT2t^JX7LJY1Rhg&EuIi zU~|5?&28i9->mf{k{paL2R0x7NCy}jOlvKb6~N}v49(hLy!t?j1H@FQzUp<|W)6%w z>Ofb)Uh}>qeF8B>nW#6ZT?%4)A0;@WjM&c07COMVV0y>!CNp^EvY#l;+F*R}v%=Q5-ts8B literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244603.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244603.0 new file mode 100644 index 0000000000000000000000000000000000000000..a25f3e3ea2958e7da7cda8dc76d3fd0612a1587b GIT binary patch literal 2669 zcmZ{lYfO_@7=R0Grr#$+xs6LiXl4kKr8qL;#N>n~s3qP)O+?Aomdl6WN;_MSprSRR z7>R;bBc$pG6pdzaOlKqOfWoFC1IN^DP&8mbC-a7e2*yQrXirY_9{cf=JnzZ#zE9dJ zC7!=`$CRhJiocbPDaD{7`lDQ9PLW=|V!NO(NljMC`AYmN40XB*eI>6XqS}_$c0d2fAYlHE~J6&}VWYvFeO)up7 z+PmA`jRJX|57?fg3zv+{?D<`05x}ddfA)2Vq)bzz(WG;`KuWEP;HU(51%O|xrkd5`~Vzp_Zd3CxM2G2c%u`nDgBnxtPRHXkylfI zO@7L%PPdJ(lyS{ylchcg*aUq+2N)Yn*UY%q1Dgw8nzh0Bl4|R!D3Y@Jr8I%-oSgR9 z_$7jvwd)^9!Mj4#+b)rq5;eNJJ%Pgb=j0>huK5|&Vn)a_L&DvnRa#+6)*u1dk{VE#M+>8<~y^aMo z%YLE*j18tM7H3ofo23p)vo;vN7+PxqF@FsB_!_YRF?(fW_2Au@^0@2bF;{e^^@5m9 zW`Z-yxKF(O-?JcQ`(DDb!WcgMnH9#=J%8f%@cGGfYSGQFYGc6p9=@alj0>h0oxhv` z)+~EPY1Rhg*Z)g71!A^ll-&@G>9nD$F?$8DsaZn@SQ{I%^j94QHW_Ov{ho~t75sJZ zDu{XQ=Y(#zO#u94O08oq@NU_qWj94)j(mwqKX2O)VrE?-IHQc&te(2%Am)sNu&gkK zqg!uufb$K`OXzV6`})N+XQRDGqg%lF)&WC0AtFs{nNpg zE32w|MPf=aQOJ~e9K;+rLU2YIvl|{q90W1%*ASK!#_)m3fqHO0^Qk?3qA?p(sO5@k fCph1xMmoT_V7lqDax+-7qlwb24aOgv4ypeKF)B6L literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244604.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244604.0 new file mode 100644 index 0000000000000000000000000000000000000000..f432a9768b3b05e4b4e07605e3d2371a1d21b80b GIT binary patch literal 2669 zcmZ{lZA?>F7=UlZAKvRa5J5g>E1fYxrrm-g2-6d*7+KUM5aZ^gr5{{2zS?*%0!2}X z(GAh4D8{cr4K*2VZW$_y3!@Bm%nDmF=GI0g#F&L>3~q=TwR&hzj(Ly0{nsMBY_VUcsT)nWMT(rv$$?C931$udNS(!z0 z>MNz9a#7NYlT?0TQK^_8`X*`3K+haX7AF z&h;kJ9Nm{To8WytO&Z=%PVgB*q%iF>(f*2Ukt`YwVa@N}n#;dmN!Vy1O5h8MWZNrv z(S(1mj=-ud_a^ucHtYy`M5)4 zFs}PA^?P9B8w!MMri8Is=0>)=b)N&98^d&fvBC7md)|rzHoHbB&DvnRLSx+#N3Qzs zz+){x)ZO<}u+bZWAVgGHN@Ka<7u&lwyVrwLBmV5~R!F(bgvR_f4@lYVBRDH;La}-w z7o>Fh3HzF^3B|a_Ik3D#$Gbd}9$`~*R+O-UyAPK4MKc{>Trho6)usm5)U{BWwZXVL zvLh4N>`7bI6SN7JGU*gr>bvYzgID)$BIVP?fby`t{2k*#s$-X3~e*GrlW+?tPRGmY2#~w&6;zz%Zf2;Pob;T zn>GTQgB^5$vB7j;WP3b6q(S9p)ZM>e@K>8f(fZAVV4 zLCpDE3C<|vJ~bs`FNmqCAuKD5;lG#vvlFau_t@Q_N7y&hV@9@5njV4mEgYui} zd1q`4tS{jqrCA${R~>9x0&EIaSN@_H)8j_Yw{^b*o6)s&fVFWW@8Pt0z~;tBlzz>| zjRv`_91wGxGr2El69L~$-|fMT;M}OQ8NVvT9L__d!?#T!rmK_Sj521|ogZ%nF+aRS zSXLOr?JbQ4u)gM~T#@ouD_GyL6WjY0 lW4dyY=Dq62V11n~I>5MKI!|mjf@|(QNom#w<8tTWftRg`D_8&k literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244605.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244605.0 new file mode 100644 index 0000000000000000000000000000000000000000..dc544a626ce9d449e1404be2512a62ebb839a2c7 GIT binary patch literal 2669 zcmZ{lSxi$|7=SNTC)}$DR>5V2DM7-bRwV%u%n3C!5rG&5F^X7dOTi*-wPle}2S%fb z!#p5rf{I#Ai5Pd|LR1hDm94}j5@ead2f-P|2r8PW=%GEH=0EoKvHAX!@BClxy|J9* zH&PaRgztYCd*g%RsbYqQ+N9DM4Y^TaiE?31uGU~Sn$$*>DMN0~)9Y2HLV1Bn zm#a0&wWCy8rY=XD=J=Pfn+uclDAf7Wf9L$jz|(GN7_UmD1B?r%^;umqa81d2O0zZ? z_l)<90ydpJBMuvh3!4}#3g6P$1Z*mL=>TJc>4apDK4265jMA(P#v||W;Zw;~``bSH z@oBY+ugdY}dcPJ;dwksjPEB}XyCsq`Mv2<=8#s`1RTsfoVI{g!vGoB+ zIj@_rKjW z*8>}M_|n^U8&@f1m1x}2zIb5MvXBliHkfWn30Veg43U&(Z7{xjaB$OflCosbJ(%}O z{^+oA3k5M>bp5>poEtaeQ@uz`PKja`H2Z;=du0S?lrcO1^~Sj%W|==>Sz!zpKian( ztgk<_^NwiDIyKrK(9;aow>X;)FfN$BIaF=~*Bo}ztPRFHT5WN_Ch@YSK{RG@CGzc% zt_C)`t8{>|!E{YuX%?_~mo6Mz%}9xt9xM_rzlcaXFPB z=FIH`XOwZD6GwN=0WpLBAS^44;qoi*3c>m;uV30dTt1m~=_q&0{Q$7O=s`NbxL`V@ z?JxKOYI4%74aPN9nlHena#_KB(U^5sRItV^0@zf?(*f4TiWEa)YsUYU8YJxM$m<^J@4+JsaerH)> z3{QTWa1*R=hgWd3UD(wZvp59}WdE}Qtk3982N)Mj&t5tVVacP3nzbB64FMzYJ<)N74lp*D4pElFn7htWnzh0BhkB3P3(~(@f>n)^v3nG*aHd7t?_duGdNFfie8nh)ff`fwFZM1Q{j$2mN!X! z34Xu1#E<=?Ah?*D#<}iPY3yuqdxFM~u!>;IJhnqgSWnJ_V^h?IWIKH2d}mM~fkoLL zez36{4q81tL=H?8Vd#`1t=gXqJni!gWt%th1C$G<*DqSr4X)|Eg{Ns7jK6OU$N@I- zJ)<_8Ne*n*8jfM)s&4rMN zW~+^(lmV6K;-2AtdIGjc~DCL>2*XV>L|m;wF-r<5@(>5*ImF`ecRmKMgai&NJ=u)e-sPg(?H znv&3R)&9d^eN)o-0m=o_E1Y$3eWiAqw!yfYM^7KHc~!4$6^xl*iK1lf2Z2pd13y67 zVA^$2h5;=8;rj>bfOK!WUqR*5gQOw;SqccoclP>|E_S%pFz>xAf~pE;FL1% zGyR?SOb~PHA;Qwa81C1(=Y#cq|Mkx5;qYae6lkctHxR6E88i5QfO5gK?3m6STvI0E zY1#(kHCA@A+h`bZDT|)3dX(zn*lpL!N!PIhMiaiV%9#J zd*5o~1YhQ`cyIumThOxNfk4dXJ5Y-{_c4eW*G_Os8MC8Dzib0Bea(cWg)uzT8kr5& z*B7{;-74(pA2UB5{n=i!1+339pC6!HFui%(vq5mpoFJa2Z7^Pc{PQ_rQ+BJ)X5;8* zw&FC>HiiENHf|<lUeD+*Az_?(#OBJykJX1H1(yR@}Uqv@1 z0h^@z-`zHUc(F;fBWqPk39#AvkPa|5nAS`7c3>0QMQPRs<4(UD2}<(Ro_l@a{KRu@ zS6z*U2!i0dUT+HFoIke}X*Qh%Tg5xKugRoL)uOEEq`U{DYX$`Y5%N8r@=EXcxcuJL7qjklDMb;wT{mIrLA6X*bAgXvHIC+q+=wTYBwZ7|MvTL*=bl;ynv9C{M0gZnGo1i6hfVHushF3}>uxa(szuDMPS5aD3%{8sx^`59y+UhJr%X)ea^z?nSr20V+eUCk8MBcGGNyu{!xFddaKvlKjY%2-OXHWdnVRzW@YrAff;O literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426000.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426000.0 new file mode 100644 index 0000000000000000000000000000000000000000..5b4799c1283f517d623bc1d32b0deb67de194665 GIT binary patch literal 2669 zcmZ{leN2pT9KfHH)$>s5R^FvzQtY%e=EjduQLAKmnd9oYx;W~xyVATBOGcPtrT3yj zGnv=9l8#7R8k2c{*P6FkYZF3#-TiuQ-(UCKpU*zu-#)+ZXU}s>81wJnktL<MXT{#IiOsk4Z*63mqlXW^bross=GGtyJPw^PFMsN=6RT>NH+ZBCr0%2okyqUymc*06JXO4|m>*6eMle&APCxLCYfBnHm zhwe6cINMk;iIt&+A0B7BlYwV1CH|~3To|BSFfEtML%}tBEHrI{@qnpAZGp|AhkwjA zL#)^=s6vxFRer#x_K`3^*8ru18(WQyA}JRrkfJOl1f(4Okl?hi0_ip-*n^Z$2iQTn3UoNM zT?>|%kymf%_pq9hQKCQ19`C{Om_lKIa>2CSnm!&}v%*5tHW+XGz3v;ZDfJ7wVY0E7 z(xVz(49cklHnsl30A+*efMsFbz$SR6K+`rDAJ^cRHHM@-(B&|dU7Ff$wz2UAF+02p zV!*lc#{Rk~5|dG&AXgtbh*{`Ca7r1oXJjvWK+Hf-!qUPR_Ia{$C|KX64NsdyV;YpG zMV1C*+HDdBC>Ko6(pT>Q*W7ENX&a0iOFc&c8~dyLEzy{}s*y{?$1D)jsZJQ6Y%uK@ z`TQKP$+gh54aO5D8vQ`bn`b4ri4BNZlz;jWIJYvxy;(e_LtbqZh}pY`;FL1%qlo9T zLCp3n!qUPR);|co0@l~u`^n^C_02TIqImChabSJ6%;$jt$_3M}l6G$f*YvOvXxawj zx62Iqz-G>h)H|Xv4OQsNu@d-^EnX!I&^A@5I4mFx*i2h3(1UEMP;X?-eh_osbFUVY zjV*jLT@!sA!MSYH?7Jc{->yX^b2nT8G3OWwPAOwHKR2QS#FVuXmKMhFUge2nvE8kw!!$1b8Q-6Q{HgNY-8;+TTzH2 z%iq5SHr@tdfU?2#>w=E+z~;Jzrfo2O#6JBXi1{up!D!lz9gO+=L9rg3`}f%XR*{%Y z6gu9m9tmPbloFg$#_SpC(N7>|=KxC!V|ZIlXE|73&*h!>MPpVhM=t3(^TGOfy)Zzz XV0v-H);4g>Lv;d8+hF{ALB`4dE95Kt literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426001.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426001.0 new file mode 100644 index 0000000000000000000000000000000000000000..be113a4bb322db9c2d07178e6e0437f78492e059 GIT binary patch literal 2669 zcmZ{ldu)qq7{EId+4l|G=^@N0y0~mP+OF;ruTD1RFk);L=3CpXvqg2ZyK!>pBFN&L z*v7_Wn}*g&T+VSjW|mT8wHXUS7z+zwSqyO}uJ77+Z}YtR_0Q$^Jo&xPFJ0p$-p{wP z_-gLy@j3M$V%lPc4bYo)*(OC?zQL3`L!GGLf6FzPEIDRN{xVAs>1?EslGvQR#-s6M}EBlQoikF zf6^6D&E)9~;COq>>s>t_ep5>HsM8tH298%(K?fKYOjmqONCwZW+(T*B2IJwHJ5FFz zt%|$gvGJF3lmnfND>(pc{#Mfg#s<>^gaxO8P4q-cvo;vFMYb1*k(AqC1&!z8^4@rD z0-`|7m0ADp2YYQ{pBhABN(6Mp*nAMgRE;D!ql{TJFR2#9)Q%!7D~w_7i(9+F`C78> zUKEY#(xWF;w-gse`WX*5Yx4V z;EXcvGizLUG>F-~m9VTZh8+>hwu1AOe(dsi`2A+Ow5TZd^%HQu^e=RPal!P$SwaeU z=KtR)&DvnRaNu(Xuu&)FT^5b$vZMHRiwf98Eu#ahjUDYOTH6Y2+&=mz8#`JQBvXQz z!|%(RJT`&wn;HLNZUxvosbOlfNX(8@R95Qf1~EgM3C<{EHqhD>24cEP3&lozuxL|rv{^z^knMq?Q&Dvo6k^S-|U{m8f<+btm zZ`M|cq-PgsfX%UUbbztJbV${NIlxABp3{f$F%xSD%L-$7W@h0QaK6;je_j)fX-h;)pQeU` c^F`Ou0mcQ>hjzT{2cFsJqgfk_m&nn~AC>DPo&W#< literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426002.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426002.0 new file mode 100644 index 0000000000000000000000000000000000000000..29a3fdac1b00dde52310aa6a95a0fdd65cea0a7a GIT binary patch literal 2669 zcmZ{lTTD|)7=RCgF6Vdw>$NVnictiitWYr;GEq@NAh_aPR|_qiDni?&^rV}S0tG%J(~>?6_bhM) z&LptUF4srikhsU~5$NN^ByBOuFIm>9AOl_h%c6KLnGP^6n2u~L*#xdxI!v=R7!ThP z83b%1+6O!~BfQuwsX|?2W$%H_gbq5u*kD@y)X@!W%7$sy2IJPz{cTgoRbRICD){x6 zn{K)rjS&RFH&2^AiR&J;W`)S_fm8eMwEiQJa)}20>dfy0DbL*}I4i6{jem=sAf>K@ zus`T(kZt~?E{sji?jYlHE!191jm zbM%)LPPdJ>lrwDTk9T?-u*r&|1B?x(e{QlB1Di3?lxA%(-W9m9RZdctyz`&ShvmNa z*!YBjn2wyd-Qe70`R4|Sn4AV#<#nS#%pxVh8D-4I9h~e3VlJCOSXLOr+Pz89V0{m^ zKWvnY>CmCw_NxhCeOt5X0ONw``D@Pq1g`00pfqcP@vZmUOMp#Jt@dBZn1wbJqqi%8 zjjE0gFgBQ0tKZmx%}hI`SsRSkc1}DDVir`({v$RZX8qmzGvM4jtD;Fd=EMB~5g?|z zh~SJe?$hz%_9+mv+}q>!@cL#t(vWLPUnyAMvwk|jxL|tuaq%O# z=HUTKvo;u)r{(tno7wAfZ%M{f}7nozVOXV$QsoE&W&wY*dh_LdkcE;GUf}2xv!bvj51~~o&B>E#Qdv;u&gkK z+f#};u)edwb8fqZz5QbrZbmUd)fHfU0mcT?*8Qy)fXy~1rCA${&r_y62QeoU=v&>p@q;mc+w^52 zIQMdS@g0enTnY+}wX6j(4^$ literal 0 HcmV?d00001 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426003.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426003.0 new file mode 100644 index 0000000000000000000000000000000000000000..48117b3e5384ccfac341bfab392ccf722a1e82ab GIT binary patch literal 2669 zcmZ{leN4<@9Ke6)Fuz}3a$ZAkm&GiH7@P4ZT}GyMHfu=N9lz?RyZK!&GpsUfd5IZ` zWY>1FEVET!Qt47eiG<0^y!?ULnh>W}kypxv9LCfz>3|S zPJUlhj?2Sa;lU(fDUz%g!u`m=i`S}9UKdRV7#B=GQTxY%Ynoyx&DvnR>3gvm*sN&l zciRl}U=wCXisYoXz^1jG4lp*D_Bno(2R4=`lxA%(-rA(BnnJGn>0yr_UtfOrmaEZd zK@gM~hLlO%yMZiiexnYYTK=G=P9|koJUSQ^a0{etXd^f)9FLx#6YU^n+hfB1ryGw# zBLBPs%TpFqI|e;GrsRz1)K^nKSf0L+4lpj5{`oQYHMnN-IZCrO7?1s3VFWh$^CRnB zHl9)jl%V6^5=($h?gBc%*kF2q3y20bGZ#{twZV8)NFoDdbis~F$cu_ zpf1pZbC>z_-j<2U#iQ;ZFZeFs_!68^#_W}6CsIMo3_rrM!Wic3G<{%wkCHnYWMevv zXhKw8D_Ea%I~`zLFx|l|$_Cfmw}aBG4aOD1Mt@*4qRP-H8#B8EHO9PD0-N2{bbztJ zbg{Gw{z@d*P@1*Dc<@%c?uX2E?5Fra%R58JOjFS3c(L-nc68GIje2&M4zP z+P5L11Cu&gkKyPeZ)V10*rKD#_TzD!2~(iLb`V11>%bbxWe^kP$57`W#4K1#DT z7%yI4mkMmcHl#Pn#&p=x^u_D;02^H_9bj$j=;{xj>%gXCi2l#Uj%IWQ4}h4LUa9Z7 zY?Sb2=7fI90q1%(EV(Ze^IaTLe2=&ZV%nPt&M0H{jHR>&#EfYnEGvxRWi$Pkfc4d= zgPUE#p8hejH=#TGyE4K0w83tPRG;dIgHWrl_{kZR6=@)^-Lp z&UFR?o5VUgz}R5=*yB97bcKV`tPRF<4)ZHPOlM|di)%N-Va&qPfDPc5XHVxokcr9Z zP;9NUuLCjL|FNtvh8KEwsKENJUfbI$8`HKPl~q*S0PE|lq63Tz XrY~OZ9}TWqbd%Do4aT#6J9hpBMR_bj literal 0 HcmV?d00001 diff --git a/tv-detection/maskrcnn_resnet50_fpn.isc b/tv-detection/maskrcnn_resnet50_fpn.isc index 0b65440d..c44bd096 100644 --- a/tv-detection/maskrcnn_resnet50_fpn.isc +++ b/tv-detection/maskrcnn_resnet50_fpn.isc @@ -3,4 +3,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_maskrcnn_resnet50_fpn" -command="train_cycling.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file +command="train_cycling.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/tv-detection/maskrcnn_resnet50_fpn_small.isc b/tv-detection/maskrcnn_resnet50_fpn_small.isc index 1e3448d1..505b6dd9 100644 --- a/tv-detection/maskrcnn_resnet50_fpn_small.isc +++ b/tv-detection/maskrcnn_resnet50_fpn_small.isc @@ -3,4 +3,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_maskrcnn_resnet50_fpn" -command="train_cycling_small.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file +command="train_cycling_small.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/tv-detection/retinanet_resnet50_fpn.isc b/tv-detection/retinanet_resnet50_fpn.isc index 61b4667a..c2e03e44 100644 --- a/tv-detection/retinanet_resnet50_fpn.isc +++ b/tv-detection/retinanet_resnet50_fpn.isc @@ -3,5 +3,5 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_retinanet_resnet50_fpn" -command="train_cycling.py --dataset coco --model retinanet_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --lr 0.01 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" +command="train_cycling.py --dataset coco --model retinanet_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --lr 0.01 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --prev-resume ~/output_retinanet_resnet50_fpn/exp_1639/checkpoint.isc" diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index c99cdb6e..a00c22f7 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -239,8 +239,10 @@ def main(args, timer): Path(args.resume).parent.mkdir(parents=True, exist_ok=True) checkpoint = None if args.resume and os.path.isfile(args.resume): # If we're resuming... + print("RESUMING FROM CURRENT JOB") checkpoint = torch.load(args.resume, map_location="cpu") elif args.prev_resume and os.path.isfile(args.prev_resume): + print(f"RESUMING FROM PREVIOUS JOB {args.prev_resume}") checkpoint = torch.load(args.prev_resume, map_location="cpu") if checkpoint is not None: diff --git a/tv-segmentation/deeplabv3_mobilenet_v3_large.isc b/tv-segmentation/deeplabv3_mobilenet_v3_large.isc index 2895d54c..3a76f734 100644 --- a/tv-segmentation/deeplabv3_mobilenet_v3_large.isc +++ b/tv-segmentation/deeplabv3_mobilenet_v3_large.isc @@ -4,4 +4,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_deeplabv3_mobilenet_v3_large" -command="train_cycling.py --dataset coco -b 2 --model deeplabv3_mobilenet_v3_large --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc --prev-resume ~/output_deeplabv3_mobilenet_v3_large/exp_1654/checkpoint.isc" +command="train_cycling.py --dataset coco -b 2 --model deeplabv3_mobilenet_v3_large --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --prev-resume ~/output_deeplabv3_mobilenet_v3_large/exp_1654/checkpoint.isc" diff --git a/tv-segmentation/fcn_resnet101.isc b/tv-segmentation/fcn_resnet101.isc index 20f2833b..bfdbb7c8 100644 --- a/tv-segmentation/fcn_resnet101.isc +++ b/tv-segmentation/fcn_resnet101.isc @@ -4,4 +4,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_fcn_resnet101" -command="train_cycling.py --lr 0.02 --dataset coco -b 4 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/workspace/datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file +command="train_cycling.py --lr 0.02 --dataset coco -b 4 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --prev-resume ~/output_fcn_resnet101/exp_1664/checkpoint.isc" \ No newline at end of file From a560f5950df5a68d73bd60c72ff02858453ba7b9 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Tue, 19 Sep 2023 14:31:55 +1000 Subject: [PATCH 29/44] Integrated tensorboard with maskrcnn and monai --- tv-detection/coco_eval.py | 3 + tv-detection/engine.py | 132 +++++++++++++++++++++------------- tv-detection/train_cycling.py | 23 +++--- tv-detection/utils.py | 30 ++++++++ 4 files changed, 126 insertions(+), 62 deletions(-) diff --git a/tv-detection/coco_eval.py b/tv-detection/coco_eval.py index 50b6246a..91b4269c 100644 --- a/tv-detection/coco_eval.py +++ b/tv-detection/coco_eval.py @@ -58,9 +58,12 @@ def accumulate(self): coco_eval.accumulate() def summarize(self): + results = [] for iou_type, coco_eval in self.coco_eval.items(): print(f"IoU metric: {iou_type}") coco_eval.summarize() + results += list(coco_eval.stats) + return results def prepare(self, predictions, iou_type): if iou_type == "bbox": diff --git a/tv-detection/engine.py b/tv-detection/engine.py index d4d9508c..87f8ecc1 100644 --- a/tv-detection/engine.py +++ b/tv-detection/engine.py @@ -7,18 +7,21 @@ import utils from coco_eval import CocoEvaluator from coco_utils import get_coco_api_from_dataset -from cycling_utils import InterruptableDistributedSampler, atomic_torch_save +from cycling_utils import atomic_torch_save + +from torch.utils.tensorboard import SummaryWriter +tb_path = "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/output_maskrcnn_resnet50_fpn/tb" def train_one_epoch( model, optimizer, data_loader_train, train_sampler, test_sampler, lr_scheduler, warmup_lr_scheduler, args, device, coco_evaluator, - epoch, scaler=None, timer=None + epoch, scaler=None, timer=None, train_metrics=None, ): model.train() - metric_logger = utils.MetricLogger(delimiter=" ") - metric_logger.add_meter("lr", utils.SmoothedValue(window_size=1, fmt="{value}")) - header = f"Epoch: [{epoch}]" + # metric_logger = utils.MetricLogger(delimiter=" ") + # metric_logger.add_meter("lr", utils.SmoothedValue(window_size=1, fmt="{value}")) + # header = f"Epoch: [{epoch}]" timer.report('training preliminaries') @@ -26,17 +29,32 @@ def train_one_epoch( # train_step = train_sampler.progress // args.batch_size print(f'\nTraining / resuming epoch {epoch} from training step {train_sampler.progress}\n') - for images, targets in metric_logger.log_every(data_loader_train, train_sampler.progress, args.print_freq, header): ## EDITED THIS - ARGS.BATCH_SIZE == DATALOADER.BATCH_SIZE? GROUPEDBATCHSAMPLER AT PLAY + for images, targets in data_loader_train: + + # for images, targets in metric_logger.log_every(data_loader_train, train_sampler.progress, args.print_freq, header): ## EDITED THIS - ARGS.BATCH_SIZE == DATALOADER.BATCH_SIZE? GROUPEDBATCHSAMPLER AT PLAY images = list(image.to(device) for image in images) targets = [{k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in t.items()} for t in targets] timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: moving batch data to device') + optimizer.zero_grad() + with torch.cuda.amp.autocast(enabled=scaler is not None): loss_dict = model(images, targets) losses = sum(loss for loss in loss_dict.values()) timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: forward pass') + if scaler is not None: + scaler.scale(losses).backward() + scaler.step(optimizer) + scaler.update() + else: + losses.backward() + optimizer.step() + warmup_lr_scheduler.step() + + timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: backward pass') + # reduce losses over all GPUs for logging purposes loss_dict_reduced = utils.reduce_dict(loss_dict) losses_reduced = sum(loss for loss in loss_dict_reduced.values()) @@ -48,26 +66,38 @@ def train_one_epoch( print(loss_dict_reduced) sys.exit(1) - optimizer.zero_grad() - if scaler is not None: - scaler.scale(losses).backward() - scaler.step(optimizer) - scaler.update() - else: - losses.backward() - optimizer.step() - timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: backward pass') + train_metrics.update({"images_seen": len(images) ,"loss": loss_value}) + train_metrics.update({k:v.item() for k,v in loss_dict_reduced.items()}) + train_metrics.reduce_and_reset_local() - ## Always update warmup_lr_scheduler - once progressed past epoch 0, this will make no difference. - warmup_lr_scheduler.step() - metric_logger.update(loss=losses_reduced, **loss_dict_reduced) - metric_logger.update(lr=optimizer.param_groups[0]["lr"]) - timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: updating metric logger') + report_metrics = ["loss", "loss_box_reg", "loss_classifier", "loss_mask", "loss_objectness", "loss_rpn_box_reg"] + norm = train_metrics.agg[train_metrics.map["images_seen"]] + vals = [train_metrics.agg[train_metrics.map[k]]/norm for k in report_metrics] + rpt = ", ".join([f"{k}: {v:,.3f}" for k,v in zip(report_metrics, vals)]) + print(f"EPOCH: [{epoch}], BATCH: [{train_sampler.progress}/{len(train_sampler)}], "+rpt) + + # metric_logger.update(loss=losses_reduced, **loss_dict_reduced) + # metric_logger.update(lr=optimizer.param_groups[0]["lr"]) + # timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: updating metric logger') + + # print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") + # train_sampler.advance() # counted in batches, no args to pass + # train_step = train_sampler.progress // data_loader_train.batch_size + + print(f"Saving checkpoint at epoch {epoch} train batch {train_sampler.progress}") + train_sampler.advance() + + if train_sampler.progress == len(train_sampler): + train_metrics.end_epoch() - # train_step = train_sampler.progress - train_sampler.advance() # counted in batches, no args to pass if utils.is_main_process() and train_sampler.progress % 1 == 0: # Checkpointing every batch - print(f"Saving checkpoint at epoch {epoch} train batch {train_sampler.progress}") + + writer = SummaryWriter(log_dir=tb_path) + for metric,val in zip(report_metrics, vals): + writer.add_scalar("Train/"+metric, val, train_sampler.progress + epoch * len(train_sampler)) + writer.flush() + writer.close() + checkpoint = { "args": args, "epoch": epoch, @@ -77,21 +107,18 @@ def train_one_epoch( "warmup_lr_scheduler": warmup_lr_scheduler.state_dict(), "train_sampler": train_sampler.state_dict(), "test_sampler": test_sampler.state_dict(), - # Evaluator state variables "img_ids": coco_evaluator.img_ids, # catalogue of images seen already "eval_imgs": coco_evaluator.eval_imgs, # image evaluations - - # "coco_gt": coco_evaluator.coco_gt, # passed in at init - # "iou_types": coco_evaluator.iou_types, # passed in at init - # "coco_eval": coco_evaluator.coco_eval, # generated at init + "train_metrics": train_metrics, } if args.amp: checkpoint["scaler"] = scaler.state_dict() timer = atomic_torch_save(checkpoint, args.resume, timer) lr_scheduler.step() # OUTER LR_SCHEDULER STEP EACH EPOCH - return metric_logger, timer + # return metric_logger, timer + return model, timer, train_metrics def _get_iou_types(model): @@ -110,7 +137,7 @@ def _get_iou_types(model): def evaluate( model, data_loader_test, epoch, test_sampler, args, coco_evaluator, optimizer, lr_scheduler, warmup_lr_scheduler, train_sampler, - device, scaler=None, timer=None + device, scaler=None, timer=None, train_metrics=None, ): timer.report('starting evaluation routine') @@ -120,41 +147,44 @@ def evaluate( torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() - metric_logger = utils.MetricLogger(delimiter=" ") - header = "Test:" + # metric_logger = utils.MetricLogger(delimiter=" ") + # header = "Test:" timer.report(f'evaluation preliminaries') test_step = test_sampler.progress // data_loader_test.batch_size + total_steps = len(test_sampler) // data_loader_test.batch_size print(f'\nEvaluating / resuming epoch {epoch} from eval step {test_step}\n') - timer.report('launch evaluation routine') + # timer.report('launch evaluation routine') - for images, targets in metric_logger.log_every(data_loader_test, test_sampler.progress, args.print_freq, header): + for images, targets in data_loader_test: + # for images, targets in metric_logger.log_every(data_loader_test, test_sampler.progress, args.print_freq, header): images = list(img.to(device) for img in images) timer.report(f'Epoch {epoch} batch: {test_step} moving to device') if torch.cuda.is_available(): torch.cuda.synchronize() - model_time = time.time() + # model_time = time.time() outputs = model(images) timer.report(f'Epoch {epoch} batch: {test_step} forward through model') outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] - model_time = time.time() - model_time + # model_time = time.time() - model_time timer.report(f'Epoch {epoch} batch: {test_step} outputs back to cpu') res = {target["image_id"]: output for target, output in zip(targets, outputs)} # res = {img_id: {'boxes': T, 'labels': T, 'scores': T, 'masks': T}, ...} - evaluator_time = time.time() + # evaluator_time = time.time() coco_evaluator.update(res) - evaluator_time = time.time() - evaluator_time - metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) + # evaluator_time = time.time() - evaluator_time + # metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) timer.report(f'Epoch {epoch} batch: {test_step} update evaluator') test_sampler.advance(len(images)) test_step = test_sampler.progress // data_loader_test.batch_size + if utils.is_main_process() and test_step % 1 == 0: # Checkpointing every batch print(f"Saving checkpoint at epoch {epoch} eval batch {test_step}") checkpoint = { @@ -166,34 +196,38 @@ def evaluate( "warmup_lr_scheduler": warmup_lr_scheduler.state_dict(), "train_sampler": train_sampler.state_dict(), "test_sampler": test_sampler.state_dict(), - # Evaluator state variables "img_ids": coco_evaluator.img_ids, # catalogue of images seen already "eval_imgs": coco_evaluator.eval_imgs, # image evaluations - - # "coco_gt": coco_evaluator.coco_gt, # passed in at init - # "iou_types": coco_evaluator.iou_types, # passed in at init - # "coco_eval": coco_evaluator.coco_eval, # generated at init - + "train_metrics": train_metrics, } if args.amp: checkpoint["scaler"] = scaler.state_dict() timer = atomic_torch_save(checkpoint, args.resume, timer) # gather the stats from all processes - metric_logger.synchronize_between_processes() - print("Averaged stats:", metric_logger) + # metric_logger.synchronize_between_processes() + # print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images coco_evaluator.accumulate() - coco_evaluator.summarize() + results = coco_evaluator.summarize() + + if utils.is_main_process(): + writer = SummaryWriter(log_dir=tb_path) + for i,val in enumerate(results): + writer.add_scalar(f"Eval/F{i}", val, test_step + epoch * total_steps) + writer.flush() + writer.close() + torch.set_num_threads(n_threads) + # Reset the coco evaluator at the end of the epoch coco = get_coco_api_from_dataset(data_loader_test.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) timer.report(f'evaluator accumulation, summarization, and reset') - return coco_evaluator, timer + return coco_evaluator, timer, train_metrics diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index a00c22f7..74ed8ee0 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -22,10 +22,7 @@ timer = Timer() timer.report('importing Timer') -import datetime import os -import time -# import warnings from pathlib import Path import presets @@ -38,11 +35,11 @@ import torchvision.models.detection import torchvision.models.detection.mask_rcnn from engine import evaluate, train_one_epoch -from group_by_aspect_ratio import create_aspect_ratio_groups, GroupedBatchSampler +from group_by_aspect_ratio import create_aspect_ratio_groups from torchvision.transforms import InterpolationMode from transforms import SimpleCopyPaste -from cycling_utils import InterruptableDistributedSampler, InterruptableDistributedGroupedBatchSampler, atomic_torch_save +from cycling_utils import InterruptableDistributedSampler, InterruptableDistributedGroupedBatchSampler timer.report('importing everything else') @@ -235,6 +232,8 @@ def main(args, timer): timer.report('init coco evaluator') + train_metrics = utils.MetricsTracker(["images_seen", "loss", "loss_box_reg", "loss_classifier", "loss_mask", "loss_objectness", "loss_rpn_box_reg"]) + # RETRIEVE CHECKPOINT Path(args.resume).parent.mkdir(parents=True, exist_ok=True) checkpoint = None @@ -245,22 +244,20 @@ def main(args, timer): print(f"RESUMING FROM PREVIOUS JOB {args.prev_resume}") checkpoint = torch.load(args.prev_resume, map_location="cpu") if checkpoint is not None: - - model_without_ddp.load_state_dict(checkpoint["model"]) args.start_epoch = checkpoint["epoch"] - + model_without_ddp.load_state_dict(checkpoint["model"]) optimizer.load_state_dict(checkpoint["optimizer"]) lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) warmup_lr_scheduler.load_state_dict(checkpoint["warmup_lr_scheduler"]) train_sampler.load_state_dict(checkpoint["train_sampler"]) + test_sampler.load_state_dict(checkpoint["test_sampler"]) if args.amp: scaler.load_state_dict(checkpoint["scaler"]) - - test_sampler.load_state_dict(checkpoint["test_sampler"]) - # Evaluator state variables coco_evaluator.img_ids = checkpoint["img_ids"] coco_evaluator.eval_imgs = checkpoint["eval_imgs"] + train_metrics = checkpoint["train_metrics"] + train_metrics.to(device) timer.report('retrieving checkpoint') @@ -279,12 +276,12 @@ def main(args, timer): with train_sampler.in_epoch(epoch): timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch - metric_logger, timer = train_one_epoch(model, optimizer, data_loader_train, train_sampler, test_sampler, lr_scheduler, warmup_lr_scheduler, args, device, coco_evaluator, epoch, scaler, timer) + model, timer, train_metrics = train_one_epoch(model, optimizer, data_loader_train, train_sampler, test_sampler, lr_scheduler, warmup_lr_scheduler, args, device, coco_evaluator, epoch, scaler, timer, train_metrics) # NEST THE TEST SAMPLER IN TRAIN SAMPLER CONTEXT TO AVOID EPOCH RESTART? with test_sampler.in_epoch(epoch): timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch - coco_evaluator, timer = evaluate(model, data_loader_test, epoch, test_sampler, args, coco_evaluator, optimizer, lr_scheduler, warmup_lr_scheduler, train_sampler, device, scaler, timer) + coco_evaluator, timer, train_metrics = evaluate(model, data_loader_test, epoch, test_sampler, args, coco_evaluator, optimizer, lr_scheduler, warmup_lr_scheduler, train_sampler, device, scaler, timer, train_metrics) def get_args_parser(add_help=True): import argparse diff --git a/tv-detection/utils.py b/tv-detection/utils.py index 37386c8a..4d09685d 100644 --- a/tv-detection/utils.py +++ b/tv-detection/utils.py @@ -7,6 +7,36 @@ import torch import torch.distributed as dist +class MetricsTracker: + def __init__(self, metric_names): + self.metric_names = metric_names + self.map = {n:i for i,n in enumerate(metric_names)} + self.local = torch.zeros(len(metric_names), dtype=torch.float16, requires_grad=False, device='cuda') + self.agg = torch.zeros(len(metric_names), dtype=torch.float16, requires_grad=False, device='cuda') + self.epoch_reports = [] + + def update(self, metrics: dict): + for n,v in metrics.items(): + self.local[self.map[n]] += v + + def reduce_and_reset_local(self): + # Reduce over all nodes, add that to local store, and reset local + dist.all_reduce(self.local, op=dist.ReduceOp.SUM) + self.agg += self.local + self.local = torch.zeros(len(self.local), dtype=torch.float16, requires_grad=False, device='cuda') + + def end_epoch(self): + self.epoch_reports.append(self.agg) + self.local = torch.zeros(len(self.local), dtype=torch.float16, requires_grad=False, device='cuda') + self.agg = torch.zeros(len(self.local), dtype=torch.float16, requires_grad=False, device='cuda') + + def to(self, device): + self.local = self.local.to(device) + self.agg = self.agg.to(device) + + # def report(self): + # return ", ".join([f"{k}: {v:,.3f}" for k,v in zip(self.metric_names, self.agg)]) + class SmoothedValue: """Track a series of values and provide access to smoothed values over a From 6859b184f4061ce838ea2614be0bba21cb3b0712 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Tue, 19 Sep 2023 16:24:39 +1000 Subject: [PATCH 30/44] backup --- cycling_utils/cycling_utils/__init__.py | 2 +- cycling_utils/cycling_utils/sampler.py | 2 +- cycling_utils/cycling_utils/saving.py | 41 ++- hello_world/hello_world.py | 4 +- monai/brats_mri_2d_0/brats_mri_2d_gen_0.isc | 6 - ...nts.out.tfevents.1695014380.sc10.1361389.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc10.1361390.0 | Bin 415 -> 0 bytes ...nts.out.tfevents.1695014380.sc10.1361391.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc10.1361392.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc10.1361393.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc10.1361394.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc13.1453448.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc13.1453449.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc13.1453450.0 | Bin 415 -> 0 bytes ...nts.out.tfevents.1695014380.sc13.1453451.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc13.1453452.0 | Bin 415 -> 0 bytes ...nts.out.tfevents.1695014380.sc13.1453453.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc14.1487328.0 | Bin 508 -> 0 bytes ...nts.out.tfevents.1695014380.sc14.1487329.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc14.1487330.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc14.1487331.0 | Bin 508 -> 0 bytes ...nts.out.tfevents.1695014380.sc14.1487332.0 | Bin 508 -> 0 bytes ...nts.out.tfevents.1695014380.sc14.1487333.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc15.1433694.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc15.1433695.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc15.1433696.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc15.1433697.0 | Bin 415 -> 0 bytes ...nts.out.tfevents.1695014380.sc15.1433698.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc15.1433699.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc16.1326408.0 | Bin 415 -> 0 bytes ...nts.out.tfevents.1695014380.sc16.1326409.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc16.1326410.0 | Bin 508 -> 0 bytes ...nts.out.tfevents.1695014380.sc16.1326411.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc16.1326412.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc16.1326413.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc18.1798304.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc18.1798305.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc18.1798306.0 | Bin 415 -> 0 bytes ...nts.out.tfevents.1695014380.sc18.1798307.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc18.1798308.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc18.1798309.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc19.1242796.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc19.1242797.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc19.1242798.0 | Bin 508 -> 0 bytes ...nts.out.tfevents.1695014380.sc19.1242799.0 | Bin 415 -> 0 bytes ...nts.out.tfevents.1695014380.sc19.1242800.0 | Bin 508 -> 0 bytes ...nts.out.tfevents.1695014380.sc19.1242801.0 | Bin 508 -> 0 bytes ...nts.out.tfevents.1695014380.sc20.1423153.0 | Bin 508 -> 0 bytes ...nts.out.tfevents.1695014380.sc20.1423154.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc20.1423155.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc20.1423156.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc20.1423157.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014380.sc20.1423158.0 | Bin 461 -> 0 bytes ...ents.out.tfevents.1695014380.sc9.1402602.0 | Bin 461 -> 0 bytes ...ents.out.tfevents.1695014380.sc9.1402603.0 | Bin 461 -> 0 bytes ...ents.out.tfevents.1695014380.sc9.1402604.0 | Bin 415 -> 0 bytes ...ents.out.tfevents.1695014380.sc9.1402605.0 | Bin 461 -> 0 bytes ...ents.out.tfevents.1695014380.sc9.1402606.0 | Bin 461 -> 0 bytes ...ents.out.tfevents.1695014380.sc9.1402607.0 | Bin 461 -> 0 bytes ...nts.out.tfevents.1695014637.sc10.1364265.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc10.1364266.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc10.1364267.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc10.1364268.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc10.1364269.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc10.1364270.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc11.1659942.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc11.1659943.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc11.1659944.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc11.1659945.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc11.1659946.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc11.1659947.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc12.2917688.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc12.2917689.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc12.2917690.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc12.2917691.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc12.2917692.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc12.2917693.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc14.1490226.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc14.1490227.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc14.1490228.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc14.1490229.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc14.1490230.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc14.1490231.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc15.1436569.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc15.1436570.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc15.1436571.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc15.1436572.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc15.1436573.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc15.1436574.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc16.1329253.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc16.1329254.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc16.1329255.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc16.1329256.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc16.1329257.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc16.1329258.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc18.1800064.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc18.1800065.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc18.1800066.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc18.1800067.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc18.1800068.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc18.1800069.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc19.1244600.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc19.1244601.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc19.1244602.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc19.1244603.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc19.1244604.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc19.1244605.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc20.1425998.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc20.1425999.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc20.1426000.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc20.1426001.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc20.1426002.0 | Bin 2669 -> 0 bytes ...nts.out.tfevents.1695014637.sc20.1426003.0 | Bin 2669 -> 0 bytes .../brats_mri_2d_diff.isc | 2 +- monai/brats_mri_2d_V1/brats_mri_2d_gen.isc | 6 + .../loops_0.py => brats_mri_2d_V1/loops.py} | 111 +++--- .../prep.py | 0 .../train_cycling_diff.py | 176 +++------ .../train_cycling_gen.py} | 85 +---- .../utils.py | 0 .../brats_mri_2d_diff.isc | 0 .../brats_mri_2d_gen.isc | 0 .../loops.py | 0 .../{brats_mri_2d => brats_mri_2d_v0}/prep.py | 0 .../train_cycling_diff.py | 0 .../train_cycling_gen.py | 0 .../utils.py | 0 .../backup => monai.bundle}/LICENSE | 0 .../backup => monai.bundle}/brats_mri_2d.isc | 0 .../configs/inference.json | 0 .../configs/inference_autoencoder.json | 0 .../configs/logging.conf | 0 .../configs/metadata.json | 0 .../configs/multi_gpu_train_autoencoder.json | 0 .../configs/multi_gpu_train_diffusion.json | 0 .../configs/train_autoencoder.json | 0 .../configs/train_diffusion.json | 0 .../backup => monai.bundle}/docs/README.md | 0 .../docs/data_license.txt | 0 .../scripts/__init__.py | 0 .../scripts/ldm_sampler.py | 0 .../scripts/ldm_trainer.py | 0 .../backup => monai.bundle}/scripts/losses.py | 0 .../backup => monai.bundle}/scripts/utils.py | 0 tv-detection/engine.py | 20 +- tv-detection/maskrcnn_resnet50_fpn.isc | 2 +- tv-detection/maskrcnn_resnet50_fpn_small.isc | 6 - tv-detection/retinanet_resnet50_fpn.isc | 2 +- tv-detection/train_cycling.py | 13 +- tv-detection/train_cycling_small.py | 335 ------------------ tv-detection/utils.py | 33 +- .../deeplabv3_mobilenet_v3_large.isc | 2 +- tv-segmentation/fcn_resnet101.isc | 2 +- tv-segmentation/train_cycling.py | 274 ++++++++------ tv-segmentation/utils.py | 1 - 155 files changed, 358 insertions(+), 767 deletions(-) delete mode 100644 monai/brats_mri_2d_0/brats_mri_2d_gen_0.isc delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361389.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361390.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361391.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361392.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361393.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361394.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc13.1453448.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc13.1453449.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc13.1453450.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc13.1453451.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc13.1453452.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc13.1453453.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487328.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487329.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487330.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487331.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487332.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487333.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433694.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433695.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433696.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433697.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433698.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433699.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326408.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326409.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326410.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326411.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326412.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326413.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798304.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798305.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798306.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798307.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798308.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798309.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242796.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242797.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242798.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242799.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242800.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242801.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc20.1423153.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc20.1423154.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc20.1423155.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc20.1423156.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc20.1423157.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc20.1423158.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc9.1402602.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc9.1402603.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc9.1402604.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc9.1402605.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc9.1402606.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc9.1402607.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc10.1364265.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc10.1364266.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc10.1364267.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc10.1364268.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc10.1364269.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc10.1364270.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659942.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659943.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659944.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659945.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659946.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659947.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917688.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917689.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917690.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917691.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917692.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917693.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490226.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490227.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490228.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490229.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490230.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490231.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436569.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436570.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436571.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436572.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436573.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436574.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329253.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329254.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329255.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329256.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329257.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329258.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800064.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800065.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800066.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800067.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800068.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800069.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244600.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244601.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244602.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244603.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244604.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244605.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1425998.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1425999.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426000.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426001.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426002.0 delete mode 100644 monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426003.0 rename monai/{brats_mri_2d_0 => brats_mri_2d_V1}/brats_mri_2d_diff.isc (74%) create mode 100644 monai/brats_mri_2d_V1/brats_mri_2d_gen.isc rename monai/{brats_mri_2d_0/loops_0.py => brats_mri_2d_V1/loops.py} (80%) rename monai/{brats_mri_2d_0 => brats_mri_2d_V1}/prep.py (100%) rename monai/{brats_mri_2d_0 => brats_mri_2d_V1}/train_cycling_diff.py (50%) rename monai/{brats_mri_2d_0/train_cycling_gen_0.py => brats_mri_2d_V1/train_cycling_gen.py} (73%) rename monai/{brats_mri_2d => brats_mri_2d_V1}/utils.py (100%) rename monai/{brats_mri_2d => brats_mri_2d_v0}/brats_mri_2d_diff.isc (100%) rename monai/{brats_mri_2d => brats_mri_2d_v0}/brats_mri_2d_gen.isc (100%) rename monai/{brats_mri_2d => brats_mri_2d_v0}/loops.py (100%) rename monai/{brats_mri_2d => brats_mri_2d_v0}/prep.py (100%) rename monai/{brats_mri_2d => brats_mri_2d_v0}/train_cycling_diff.py (100%) rename monai/{brats_mri_2d => brats_mri_2d_v0}/train_cycling_gen.py (100%) rename monai/{brats_mri_2d_0 => brats_mri_2d_v0}/utils.py (100%) rename monai/{brats_mri_2d/backup => monai.bundle}/LICENSE (100%) rename monai/{brats_mri_2d/backup => monai.bundle}/brats_mri_2d.isc (100%) rename monai/{brats_mri_2d/backup => monai.bundle}/configs/inference.json (100%) rename monai/{brats_mri_2d/backup => monai.bundle}/configs/inference_autoencoder.json (100%) rename monai/{brats_mri_2d/backup => monai.bundle}/configs/logging.conf (100%) rename monai/{brats_mri_2d/backup => monai.bundle}/configs/metadata.json (100%) rename monai/{brats_mri_2d/backup => monai.bundle}/configs/multi_gpu_train_autoencoder.json (100%) rename monai/{brats_mri_2d/backup => monai.bundle}/configs/multi_gpu_train_diffusion.json (100%) rename monai/{brats_mri_2d/backup => monai.bundle}/configs/train_autoencoder.json (100%) rename monai/{brats_mri_2d/backup => monai.bundle}/configs/train_diffusion.json (100%) rename monai/{brats_mri_2d/backup => monai.bundle}/docs/README.md (100%) rename monai/{brats_mri_2d/backup => monai.bundle}/docs/data_license.txt (100%) rename monai/{brats_mri_2d/backup => monai.bundle}/scripts/__init__.py (100%) rename monai/{brats_mri_2d/backup => monai.bundle}/scripts/ldm_sampler.py (100%) rename monai/{brats_mri_2d/backup => monai.bundle}/scripts/ldm_trainer.py (100%) rename monai/{brats_mri_2d/backup => monai.bundle}/scripts/losses.py (100%) rename monai/{brats_mri_2d/backup => monai.bundle}/scripts/utils.py (100%) delete mode 100644 tv-detection/maskrcnn_resnet50_fpn_small.isc delete mode 100644 tv-detection/train_cycling_small.py diff --git a/cycling_utils/cycling_utils/__init__.py b/cycling_utils/cycling_utils/__init__.py index 3a5de648..22b215e8 100644 --- a/cycling_utils/cycling_utils/__init__.py +++ b/cycling_utils/cycling_utils/__init__.py @@ -1,5 +1,5 @@ from .timer import Timer -from .saving import atomic_torch_save +from .saving import atomic_torch_save, MetricsTracker from .sampler import InterruptableDistributedSampler, InterruptableDistributedGroupedBatchSampler __all__ = ["InterruptableDistributedSampler", "InterruptableDistributedGroupedBatchSampler", "atomic_torch_save", "Timer"] \ No newline at end of file diff --git a/cycling_utils/cycling_utils/sampler.py b/cycling_utils/cycling_utils/sampler.py index 6c7325a2..02cfcbf1 100644 --- a/cycling_utils/cycling_utils/sampler.py +++ b/cycling_utils/cycling_utils/sampler.py @@ -251,7 +251,7 @@ def load_state_dict(self, state_dict): def advance(self): """ - Record that n samples have been consumed. + Record that one batch has been consumed. """ self.progress += 1 if self.progress > self.num_batches: diff --git a/cycling_utils/cycling_utils/saving.py b/cycling_utils/cycling_utils/saving.py index 0331b71d..40db3ff5 100644 --- a/cycling_utils/cycling_utils/saving.py +++ b/cycling_utils/cycling_utils/saving.py @@ -1,6 +1,7 @@ from pathlib import Path import os import torch +import torch.distributed as dist def atomic_torch_save(obj, f: str | Path, timer=None, **kwargs): f = str(f) @@ -15,7 +16,39 @@ def atomic_torch_save(obj, f: str | Path, timer=None, **kwargs): else: return +class MetricsTracker: + def __init__(self, metric_names): + self.metric_names = metric_names + self.map = {n:i for i,n in enumerate(metric_names)} + self.local = torch.zeros(len(metric_names), dtype=torch.float16, requires_grad=False, device='cuda') + self.agg = torch.zeros(len(metric_names), dtype=torch.float16, requires_grad=False, device='cuda') + self.epoch_reports = [] + + def update(self, metrics: dict): + for n,v in metrics.items(): + self.local[self.map[n]] += v + + def reduce(self): + # Reduce local over all nodes, add that to local store + dist.all_reduce(self.local, op=dist.ReduceOp.SUM) + self.agg += self.local + + def reset_local(self): + self.local = torch.zeros(len(self.local), dtype=torch.float16, requires_grad=False, device='cuda') + + def end_epoch(self): + self.epoch_reports.append(self.agg) + self.local = torch.zeros(len(self.local), dtype=torch.float16, requires_grad=False, device='cuda') + self.agg = torch.zeros(len(self.local), dtype=torch.float16, requires_grad=False, device='cuda') + + def to(self, device): + self.local = self.local.to(device) + self.agg = self.agg.to(device) + + # ## ENABLING ACTIVE PROGRESS TRACKING +# # If ths is a desireable proto-solution, how should this be integrated with the cluster_server repo, +# # and how then imported for use in training scripts? Authentication required? # from sqlalchemy.orm import Session, sessionmaker # from sqlmodel import SQLModel, create_engine @@ -63,11 +96,11 @@ def atomic_torch_save(obj, f: str | Path, timer=None, **kwargs): # class AtomicTorchSave: # def __init__(self): -# self.progress = 0 +# self.progress = 0 # Stored as internal state, posted to database upon save # self.experiment_id = os.environ["STRONG_EXPERIMENT_ID"] -# def commit_progress(self): -# db = SessionLocal() +# def commit_progress(self, authID): +# db = SessionLocal(authID) # db_experiment = db.query(Experiment).filter(Experiment.id == self.experiment_id).first() # assert db_experiment is not None # db_experiment.progress = self.progress @@ -88,7 +121,7 @@ def atomic_torch_save(obj, f: str | Path, timer=None, **kwargs): # try: # self.commit_progress() # except: -# print("Progress commit failed.") +# print("Progress commit impossible without experiment_id.") # else: # print("Experiment id not set.") # timer.report(f'committing progress to database') diff --git a/hello_world/hello_world.py b/hello_world/hello_world.py index 0b0eab52..f351d4a9 100644 --- a/hello_world/hello_world.py +++ b/hello_world/hello_world.py @@ -1,4 +1,6 @@ import socket +import os print("Hello world!") -print(socket.gethostname()) \ No newline at end of file +print(socket.gethostname()) +print(os.environ) \ No newline at end of file diff --git a/monai/brats_mri_2d_0/brats_mri_2d_gen_0.isc b/monai/brats_mri_2d_0/brats_mri_2d_gen_0.isc deleted file mode 100644 index 6829fad6..00000000 --- a/monai/brats_mri_2d_0/brats_mri_2d_gen_0.isc +++ /dev/null @@ -1,6 +0,0 @@ -experiment_name="brats_mri_2d_gen" -gpu_type="24GB VRAM GPU" -nnodes = 9 -venv_path = "~/.venv/bin/activate" -output_path = "~/output_brats_mri_2d_gen" -command="train_cycling_gen_0.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361389.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361389.0 deleted file mode 100644 index c66dc3c9667008bf810abbbbf31a309c30b79af2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 461 zcmeZZfPjCKJmzv?s(HtiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@~75bMjmQzptp+zC)fY2H6U}fM7V^wxQkMg z^Ye=1bMlLeMHv?G+0{*|`X>Wbe&XO%S9_rW!wxdo{DXCY@1iZ@RqRHM5FsC;44{SXw_2wyt_aSam|*ElS(d*O0Q3&l0d QZ@@L6xdy0OvWH0p0E3T<;{X5v diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361390.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361390.0 deleted file mode 100644 index cb8d4131a461d18fa23c586f8ebd65de58cbd918..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 415 zcmeZZfPjCKJmzvn*z;C@cf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZSrn9VT^?#~ptp+zr{CYLH6U}fM7V^wxQkMg z^Ye=1bMlLeMHv?G*{u*-`c4L_{KUbjvYc-iw?UK(a`AINnpwAMEt11-V0j5sOO_tn7@Yn@x~|YPfkv z$cQs!FI)qfTY##yFD;Hi@uucJxN3CQ0F^u1xxPnnjl{Jh5Z5qaagDNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZiL!AFmWP@f=M-d>8W`Lwe1HPf@Ps<6ZG#Z zbMAww72*=$;!MdbPDUvHy!@SsB2+PB^_y>;A9s4yg4`j%h{YudR(3U6Pg&)l8gAYZ zGU7Df2iJh+7NF|fpyWOjZ~jE6Mt2QR`PuV_T2WjhcI^nnHB4AsgKu?XY diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361393.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361393.0 deleted file mode 100644 index 67675700629e71807b09e52c3871f3885678799d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 461 zcmeZZfPjCKJmzv**z;C@cf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z5k7ymN*-!%ptp+zXZYW(H6U}fM7V^wxQkMg z^Ye=1bMlLeMHv?G*;QS8wnYZ2{KUbjvYcUz+aSsXx%jy_(o^#gYW3P!+yYe18xs2*#hdF-z*VEW2B>`6M%&dWu2H;p1mYSdEUs}_Vi)Im>?ewA Rl5W5?pt%O9+Kfx$CIFVkjO73T diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361394.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc10.1361394.0 deleted file mode 100644 index 7f0ad2699af738ec7093d63ab389a55e2a19be3a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 461 zcmeZZfPjCKJmzxd+w)d`cf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zk(RJ6lZToc=~7t2iIagUKXGuXEayk&Z4l*xT>M-d>8W`Lwa;%)+$IZEo1lMR znR7Y|RILz~02gOUW^pn?vB@>&97U*N#_BiUIPG?M)q>n1z=*{q308JXI2~`xK{edG zBV@$6&^GLpm=ljLAYvk*8r8oSob|YCz^{iEs&VaTlc~ z=jRp2=j0a`i!vNnpw#gF;ag4`j%h{YudR(6FGdn)9h8gAYZ zGU8-C3)g_=7NF|QnTrKbyjgkNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z(LPi9R32(>ptp+zXEn!;8j!hKB3wdT+(oI$ z`FX|hIr+uKq6`c8>?R2XXURa7pEx*GmNSYUs$7tZpNk_sH4mXybV=tCS*Y3s{rk$C z3Ib5ILRb$q2=asryY8p^6!+-+beYJ?c{na)$sT7MCPg+07Jx6(NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z5f|=zC=WF^(A!0Vb0gP|8j!hKB3wdT+(oI$ z`FX|hIr+uKq6`c8>;fKa5SD=|KXGuXEN8nQRJkA*KNm-OY92zZ>XYsmS*Y3s{rk$C zfkIHVLRb$q2>m3BM;PLKQPszxl@La@405oK+kz{lH45vrK6`pq{^uA@G+Aa@8bVsS}=l^wrpgS;G6!_7NF zMx1Al!!@9}1*rPF{+qif-mE?WSB>r(pmJyD+kGgmne^}GJoJw3fYCz^{iEs&VaTlc~ z=jRp2=j0a`i!v BfU*Dp diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc13.1453453.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc13.1453453.0 deleted file mode 100644 index 690d7f733f166bd3cfd929fb69da8549d839fabb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 461 zcmeZZfPjCKJmzw)@Zhih?s(HtiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!k_W@fIQUPKyMcb&Q9(fH6U}fM7V^wxQkMg z^Ye=1bMlLeMHv?G*)`=GyUIY7pEx*Gma|zHs$7tZpNk_sH4mY7^Mc4yS*Y3s{rk$C zMj}wPLRb$q2va1Cf~0jl<2p|}IZn>nZ9s?l8oR9^HeY7UBPavmOoxP}RfYaEu?&9)NmL2(V| Q6SxL6*8o+|k9I2u0Ko~24FCWD diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487328.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487328.0 deleted file mode 100644 index 2f9a2aa7aec0632f4f5fde7165f2f6c9c7d01652..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 508 zcmeZZfPjCKJmzxFbmp!8?s(HtiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!hOTUNgir$ptp+zryaw#8j!hKB3wdT+(oI$ z`FX|hIr+uKq6`c8>=rkru9bl*KXGuXET;r3RJkA*KNm-OY92zZB_q##S*Y3s{rk$C zcVKFTxCFR3Q!ar5sel%{xLy zoV)hHHK4f#sG4u%Cm$4VChv!vptp+zrxoM28j!hKB3wdT+(oI$ z`FX|hIr+uKq6`c8?CK7rn9D$wpEx*GmeZ9Ts$7tZpNk_sH4mZI&wJ-%S*Y3s{rk$C z9vo1$LRb$q2={uCt~qLKQPszxl@bX^&Se$Q=TVSX`1|Ww(9D8E-kLhMRYU zj5s$Pglj-^3s7}R=IT8t-fTDoSB>r(pmOG!x96j{hV#Y|h-;XzxW-|L-I1uBpHN&A ReG9Gu%{4&PPYO2Y0RZ48j;{a! diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487330.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487330.0 deleted file mode 100644 index c7dc4277a3d646ae82144a3f9d4feffa68907c52..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 461 zcmeZZfPjCKJmzw8I`dY4cf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIXbaBQyyw=ptp+z=QD;J$t-rLGBP>#Nv_!D?4|yKQ?ku4L9!y z8F3yu1lNG(7NF{DcD|M<-aK^_t{UAnK;_%kZQqFEnqSwCKwQIw#WfB~?9MftpF(ks f#Vxo7G}i!CyRXrEi{hGV2-WDW0V+S!%H{_E$#tG! diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487332.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc14.1487332.0 deleted file mode 100644 index 2156c28f55f02d45f03aecedfcd5800c40429a71..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 508 zcmeZZfPjCKJmzvTx$sthcf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZdH*wYzdY33KyMcb&PNQ}YCz^{iEs&VaTlc~ z=jRp2=j0a`i!v@4L9!y z8F5-4fNMZ=3s7~pipxNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z>9crIAP+S+(A!0VGl*eZ4ai(A5iTJv?xNJ> z{Ji4$oc!WqQHBM4cH7Tf|1JYne&XO%SNnpw&GvZJg4`j%h{YudR(AK#UJH?fYPfkv z$cR(>09*r_TR^I3CV8WH^9@2Zx@&;S6TVBcp}6MT^&=41Fkx|x!xB5+e#gfsuJOGE O*MR04py~;MH@g5kn2@gk diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433694.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433694.0 deleted file mode 100644 index 0a53c6468459c6702ae8888ea0ca4cc26c6ac636..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 461 zcmeZZfPjCKJmzxxd-7I)cf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIkzKat31@)KyMcb&MJ;=H6U}fM7V^wxQkMg z^Ye=1bMlLeMHv?G*-fc%nj`~Ne&XO%Sr(pz<$FpI4!{X6gMS5Z5qaagD`CeHQc-- zWW@RAI9vmoTY#$HzyGI;;?4ah;HuGG162Mj<#!s2YZlx;0&xu!7S}i|v0JHi;vkA^ Rcpkwupt%O9x^2ysV*owhlY{^O diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433696.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433696.0 deleted file mode 100644 index 1b277af681ed19677a1c1e3fb77158430f378474..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 461 zcmeZZfPjCKJmzxV_u#Gm?s(HtiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@}*TjR32(>ptp+z=Sq%kH6U}fM7V^wxQkMg z^Ye=1bMlLeMHv?G*-bY3o*@HOe&XO%S|Q00PL{9GLAsd)&s`SukivQV`N`uCMN zoA{t=g}4N`I8!oNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIi&X6Lmp~wptp+z=TDApH6U}fM7V^wxQkMg z^Ye=1bMlLeMHv?G*}ZskuS^E2{KUbjvYZF_pvncg__;XJQ}YmNALPAyD+^VdpnqSP zvxpz6R)|Z0i!&v&I2ob%^NZkkMW|xN>Nnpw1CDsrg4`j%h{YudR(5_q0#R~M4L9!y z8F2=kglj-^3sCjPei0uOZ}Ojlt44PXP`P=F!UPo8%)fsG;uPt8N9ef)IS4q2$$1pWKU zoV)p*bT97*g7_qn{!OG6m`S)=-sD_(& zgp4>PkHR&exdo`&IFE}B#hVKes?l8oRGu7A--zOxx%ZDiT*HLLH4aPcUX_%*KygjX QBe(`M*8o+&_z=zq0Q};Sb^rhX diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433699.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc15.1433699.0 deleted file mode 100644 index d5adfef7425e8808c77854238bf5957e577e06af..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 461 zcmeZZfPjCKJmzw)@Zhcf?s(HtiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!n1GhetD?5f!;0>oTi-HYCz^{iEs&VaTlc~ z=jRp2=j0a`i!vK%z`H6#5WjUYnL6r+~@pEycr{*ElYX2(fl7*^G(7&(D z*~br6E5s$h#hH>>oQzPMQX?j-2vy8j{pK5Iz!9%nkUInzvA86`%FfS6AW9CZ;pQD7 zBhJ|;;2O}}0#wa1FZDZ$H=R$yRinEGsGMnrs62{mX5BvmaSam|*ElS(J9TyHVHDRC RJcMgNa}7}S*4Ia;0Ra4yj)4FG diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326408.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326408.0 deleted file mode 100644 index a26724a61f88f0ccf27718ef513426950f921810..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 415 zcmeZZfPjCKJmzxBTJctYcf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZnRPAHMjmQzptp+z=ZarjYe42|iEs&VaTlc~ z=jRp2=j0a`i!vs<`$r8lY%KqDBk>vP>t>ypmLVkIp!#?S$**c#5GJNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zd6`qWSRQI_ptp+z=iwh)Ye42|iEs&VaTlc~ z=jRp2=j0a`i!vj(SRju6rqY4tKWR%)Y|G*3v!15BNmq=SlQYAE?pr9)o}BU zkP&C@Hn;{fw*XZy)bdP1@n-W5xN3CQfDG?@RD$A~ITw#WT*HLLH4aPc#LPAZqqyeO QWw-`3*8o+oVxQ^;0EtYHVE_OC diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326410.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326410.0 deleted file mode 100644 index 02efdbf3f8e3d5d4cebf1ef340262203f5448428..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 508 zcmeZZfPjCKJmzvnTk%$Zcf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zd2jnoMjmQzptp+z=aHXVYe42|iEs&VaTlc~ z=jRp2=j0a`i!v>oQzQXZ)50HMW|xN>Nnpwxwd-Mg4`j%h{YudR(2aqw8gAYZ zGU6272G@Y*7NF{eHs&f6Z`SO9t44PXP`TShZ&?)Athjgt;uNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zi84Q%BM&t<(A!0V)AHxm8j!hKB3wdT+(oI$ z`FX|hIr+uKq6`c8?2d#qo|J(qKXGuXENAy$sB%FrelCvm)I5aR2bmFevQV`N`uCMN zXZ?e!72*=$;!MdbPDUvH;hMig5vrK6`pq{^t*u_QAa@8bVsS}=m7U%1(iL)04L9!y z8FBh-higD{3sAMll9+QS-n@uVjqVzt@}Gf9MJTSBbMXkoHB4AsNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z`S(1*Tpns}ptp+z=gA*iYe42|iEs&VaTlc~ z=jRp2=j0a`i!v(3F8HKXGuXEa%(5Q00PL{9GLAsd)&sAG;_1l!dBI(7&(D zdGQ}qtq_+07iUUlaWX=2;KBe_MW|xN>NnpwZMJ&Vg4`j%h{YudR(1h>E|cV-8gAYZ zGUB|s9j*b*EkMAg*D;;u?n~c4fWax=>tG QbOo*f%{4&PZ%W;(086=&JOBUy diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326413.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc16.1326413.0 deleted file mode 100644 index 78fec76651295e8ba46367dac24ab10fb2dd3a07..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 461 zcmeZZfPjCKJmzv{S@BkXcf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIn;OjhCI~VKyMcb&UZhy)_}~_65$f!;x0-} z&d)23&&e+?7G+q#XBT&7r?CuF`H6#5WjRm&fhrf|;^*Q>Pt8N9wc6oPDGODbpnqSP zv+6HYtq_+07iUUlaWX>j!^c0CD?$}BR=@ei>9EzS7UT{AMl3E#u(H$Mm*6A^)o}BU zkP+vi?QjifZUL&k;uX-3;?1xfaMkFp0V>z4t)7MAng_L+hdk8WKyMcbPNw!9H6U}fM7V^wxQkMg z^Ye=1bMlLeMHv?G+0}7&GRQ!cpEx*GmUDS8RJkA*KNm-OY92!Eq&M>p%R<#A=-*f7 zyxj*?E5s$h#hH>>oQzOh9sKQ~B2+PB^_y>;SDbunLGBP>#Nv_!D?4ir;TdvJ4L9!y z8F6~J!8M?{1*qEUyHGcZH#ywls?l8oRDM_Jsw9eQuE!mRxP}RfYaEu?eXe}555+Y{ Q6W|)qTmw`cUpjj#02dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWS_>zBl1vl1HD}&I2qe^)PT&@65$f!;x0-} z&d)23&&e+?7G+q#XO~p|?}H3f`H6#5WjPObLzN41@pEycr{*ElmZ!H_%0ks9=-*f7 z?Cyc872*=$;!MdbPDUt>l&_qq2vy8j{pK6zWM|)6kUInzvA86`%FaRdR;?UV!_7NF zMx41Ga1Cf~0jdtZAs&R{O)gKkYIN5Cm75uspt|Ns+;NC&n6S9UVTs+0+f}S6uHjFF OYd~`iP<7X&TfYHJxQ#>r diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798306.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798306.0 deleted file mode 100644 index 883221b8b4a42c17bfb527964fe92878b89555f4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 415 zcmeZZfPjCKJmzwyAKWt4#`KXGuXEN5;vRJkA*KNm-OY92zZ;jIrpWua;l^zSQk z*7iWv3ULW=ai(MzCnFRGpDT;pQD7 zBTj2~xCS)009BW@{GWp2O<51PYIN5Cm48^@c?ZQcKJmvPu3^IB8iyrz%O2@>0RZp- Bf8YQB diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798307.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798307.0 deleted file mode 100644 index fd153772ccc935436ea59f98e1ec039947c00393..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 461 zcmeZZfPjCKJmzvvKf+)A-SMWQ6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v>oQzO>>7`1KB2+PB^_y>;ZqB~7Aa@8bVsS}=m7Ury`Hyl?4L9!y z8FB7(higD{3sAMltwT{L-pu!at44PXP`UUlp$HV${Ej;gaSam|*ElS(`**oH3&k}F QiEs^Qt^ulE@qM2v0B@v_CjbBd diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798308.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798308.0 deleted file mode 100644 index fb9ae9465c75aa87b445cec61917c9d4ca9f2530..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 461 zcmeZZfPjCKJmzxB9^$Y5?s(HtiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-qAto-A`dk;(A!0V^K;vd8j!hKB3wdT+(oI$ z`FX|hIr+uKq6`c8?27)cl#_ufKXGuXEGK6lRJkA*KNm-OY92yuan|M8vQV`N`uCMN z&%o3QaS3p7reqc;BNT7xn-ir7Rm@oZ<{RgFC*N9-I|LZ9xFo^KuHnbWW;v*an|Fka zIA^%SHK4f#sCt*gnL8-nwDf?hMt2QRdG#M(KNQ#8i8~H)4HFjEI4rRX*vEee#Wg?U P;Tq6f15~Z>e(78Q5rdD7 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798309.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc18.1798309.0 deleted file mode 100644 index a775d97eee2e7d78477260508fb91124646e9866..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 461 zcmeZZfPjCKJmzvvKgeJG-SMWQ6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vS;1iPt8N9RZjcwCJR-YpnqSP zQ?3uHR)|Z0i!&v&I2ob1$5lvJ5vrK6`pq}aDktAskUInzvA86`$}a7;y0{!v!_7NF zMx2eVa1Cf~0jl2E5>k%h&6#d+)#$DPDwpn^zXioL7V*a+u3^IB8iyrz8jfj_D6ZL? P2-kq-8ldW>p}X_}H3W;> diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242796.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242796.0 deleted file mode 100644 index 8e24d704a6519265930ca150b3791bf91577a5fa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 461 zcmeZZfPjCKJmzxFFyXKM?s(HtiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^5CmcfjrdQKyMcb&JSO<*MQ8`65$f!;x0-} z&d)23&&e+?7G+q#XD3waS1$uqe&XO%Sx&ECQ00PL{9GLAsd)&slXQK5$U@a7=-*f7 zRR0ZCE5s$h#hH>>oQzQHFsnOR5vrK6`pq}aRU3S2LGBP>#Nv_!E4%(F-)_o5HQc-- zWW@PuD_jGbTY#!RXxH?hc=OOUxN3CQ0F|@t`1J|JHKu2dL0rRx#WfB~?2NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z>C5ChE)O*~(A!0VQ}pZh8j!hKB3wdT+(oI$ z`FX|hIr+uKq6`c8>|9D7-;seTKXGuXEa!sXQ00PL{9GLAsd)&sGtNZy%0ks9=-*f7 zobv~&R)|Z0i!&v&I2ob%^q-D(icrOj)o;FW&feft3v!15BNmq=SlOND4p=J()o}BU zkP&Cs7Ptm9w*XaZr0lwY;!UTmaMkFp0V@CaP9YY>HQ8s5L0rRx#WfB~>|)mD{X}t1 R#5uSIG}i!Cvuv?f0RXG9ksts7 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242798.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc19.1242798.0 deleted file mode 100644 index 344552e22fe27d895d1e4229a77ef7ee0b189308..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 508 zcmeZZfPjCKJmzv9H|DSY?s(HtiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(xJHEl|0nkKyMcbPV2ARYe42|iEs&VaTlc~ z=jRp2=j0a`i!v>oQzPs@nZ8#MW|xN>Nnpww{7sL1-V0j5sOO_tnA`74l2q)HQc-- zWW@Pk3tR)5TY##$x@NCO@#em*aMkFp0V+>YKV*jDn%pzTAg*D;;u?n~c8o_9%}`uZ gbq=lp%{4&Pr=%;tqqxT6JX|%pYkNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z@s+vqMjmQzptp+z=Z7!bYe42|iEs&VaTlc~ z=jRp2=j0a`i!v>oQzN`)3!-W5vrK6`pq}aOB;M@LGBP>#Nv_!E4wE7oyl@g4L9!y z8FBh-g=;`_3sAMqV(w}bZ^~?gt44PXP&wz(U#%#vNjh^3;u>oQzO>P$$|$5vrK6`pq}a#T$HTLGBP>#Nv_!D?9af7E*Fh4L9!y z8F3!o0@r}%7NBb5k~tqxycxR{t{UAnK;RNj!55;uNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZxmkLkSsrR`ptp+zr|{S9H6U}fM7V^wxQkMg z^Ye=1bMlLeMHv?G*%`<7-;jYSKXGuXEa${uQ00PL{9GLAsd)&s(ZUh8Wua;l^zSQk zCjEx072*=$;!MdbPDUt}Tz7q+B2+PB^_y>;vp4wEg4`j%h{YudR(9vP1J=qxHQc-- zWW*V?1+D?jEkM<(w!f=Uys5est{UAnK;^&egv3x>Q+VbW#5GJb$q2<078dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq$PXlKY6IRf!;0>oYgC~)_}~_65$f!;x0-} z&d)23&&e+?7G+q#XZQL>(+L@<@)HNA%5vteg(?^1;^*Q>Pt8N9UAOY7vn*6?g8qGF zPN#KHwL)A1T%0MH#mNZ8k&c|x6`_h5tKWR%luY)j1-V0j5sOO_tn3V*N1T^~YPfkv z$cU3O4Xy#rEkM(X3UfuMV#ew>-#FEiy=p=35MadOk_0O|mGyDva!?I7?+6)j zu1ke$KywRFb*`zNG>SJ<(%`DmT?16^@2dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vr0Gvyygbz0KyMcb&WIITYe42|iEs&VaTlc~ z=jRp2=j0a`i!v#XC|EDKeepnqSP zbMIQHS|Kh0F3yz9;$(zk9@R%WicrOj)o;FWN+f&Lg4`j%h{YudR(5X+kLk-nHQc-- zWW-sN3fF+<7NBbJb$QJw-n2-At44PXPdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWC6SJVtJ^!f!;0>oTpZ7tpS;bi_ZUL&^V|RNqiZ_>}!&RfZ2B`eV;^?&~uBop-0&xu!7S}i|u`8ZaS&HJC RT@7#zXs!XO4qzw`0|0o$kpTbz diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc20.1423158.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc20.1423158.0 deleted file mode 100644 index 641f99beff96f88c01e5e2d2d4dc1070b753fd98..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 461 zcmeZZfPjCKJmzw)dC61#-SMWQ6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vB-m``DS4>5f!;0>oGvT2)_}~_65$f!;x0-} z&d)23&&e+?7G+q#XUE@tYk~|^`H6#5WjSruLX``0@pEycr{*ElK9BO$kcFyE(7&(D zDYOo%R)|Z0i!&v&I2obXFqTzF5vrK6`pq}awq&nbkUInzvA86`%I-|_BO^JehMRYU zj5wLn;Tq7~0#t1`^|1_!H)kPKqq_#EeACxI(ewMf!;0>oL-{aYCz^{iEs&VaTlc~ z=jRp2=j0a`i!v>oQzPMAE9(d5vrK6`pq}a+zVc{Aa@8bVsS}=m0gF1gSi}3!_7NF zMw}*>;Tq7~0#v<%xAh!~H<_-$RinEGs9g4H5HpHvj=nhpaSam|*ElS(%hc#Nfa03G QcW@18t^ul!2@7Nb0JlMlo&W#< diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc9.1402603.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014380.sc9.1402603.0 deleted file mode 100644 index 7a6dc43e22f684012af988061996116f5a075ed3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 461 zcmeZZfPjCKJmzx7hwxT^cf9E+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZnR+aXLmp~wptp+zC%@>n8j!hKB3wdT+(oI$ z`FX|hIr+uKq6`c8>^Afk1j;~_pEx*GmQza-s$7tZpNk_sH4mY7tLIz?S*Y3s{rk$C z7h!6JxCFR3Q!$lb9RiG4T#{gAH&0OGryNwn%{xLy zock`rHK4f#sCo{o;|~;X=3IfRMt2QR`IM~>?xMKn^_wFQ*Dzsmjl&W<{u$@BQC!pa Q9;xfi@@LGBP>#Nv_!E4vO22Xi^7hMRYU zj5zJCz%`(`1*qCF*<26Bn}S#2s?l8oRK9Wj=QtGC9C&jC;uoZm&a)qu>^65$f!;x0-} z&d)23&&e+?7G+q#XP2qet|bFie&XO%SLRb$q2<9Cko#yLKQPszxl>_;euB!$Q=TVSX`1|Whe09dzBni!_7NF zMw}ln!!@9}1*m#?d$t{lH`}hjRinEGsJvmp)p8Wqe1CHU;uNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIb7BrD-Sg{(A!0V(?x1q4ai(A5iTJv?xNJ> z{Ji4$oc!WqQHBM4c1Nea@0WopKXGuXET_6WRJkA*KNm-OY92!EC&k@MWT9#k^zSQk zDk?(N3ULW=ai(MzCnFT^nz%4U5vrK6`pq}a@C#nGAa@8bVsS}=m0kAbTwgh;hMRYU zj5rG~!!@9}1*m$HX8dMUZ(f0`Mt2QRx#Q(CAt>oQzO>|I4pNnpwqb_*Wg4`j%h{YudR(7Vde)7pdHQc-- zWW-r^5v~EvEkM<~`4f6jy!r7mTs69DfXZ`Ln|7nP=JlH+5Z5qaagDNS(3Th zyk$dcla~_CTWzc+u{7^m$V(~n_H_5`xqYAR*Pre4efIf&Kl@dT#QF30K+FkFCZARD zbK0|Ms{RHc#bnV%XBjOS5n=Ond`O1TVoete=_x|0&YEd9rwG})HG(O_DCmsclzzD> z&8T<&rfhoDXn*o<+?*&Mu7A1YdiT3p1uy5Nf^k{8#i~zBw_1ltiUy#nc$vg9EiN{&-m~71u*qXy25^_I6IRX-q+w@XR2A;i;hjEk6(*ec>)BAs|7z)cB;AJLG~MtMqWJ6 zduFAi4Ushe%e9}bt^=?7Xdm4aNg0`hW)+ML1u3N=1ZRblkoW4iV35)rM%W&@NeCtA z3&HUkzExCqd$>(0F(B{Exy9gkx7+Ce77^Pi4t^hWAA3DI;V0uaD zyU)Nzp`mo6o_ccuyt}hWeOElD4;R!3W4<9cqm273k}g~Y zV*0-&EGvv*W&8Sb;Cvh6TO1y4zs$;Hl$uw%4V>?L0v%vnFnuwB~sd?KY;8;l=n z>+}US;a4*4qA@FN$ZJ*3V_-A4j1I6iHl*2nL=9}r<&^GWV?(hCTl#{Srig%h4jWJS zWtxtaAn)s=UcFy4lpj54l0P!f;Br_G;4$LgRhUb0GlkoGN+BZKeI*osQUHD z8eo&{PX`zqOqWbiv;iBLhSIDJ#^-JCEr6J#8kRqF+@?2-8FscdAG|xcGw+c|Oi3Kt zg$yPTv+*~<8D-4IH+`^xn3;bF%L-$-MSUU=oKNGuu|_mz(R_3atzHby*P)~Xj0>g% Ub!R?-HS<)IW^FLuI>GerKjNbwga7~l diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc10.1364266.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc10.1364266.0 deleted file mode 100644 index e0dc66ce7acc4644551c27cf7061b4f03b596777..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{mX-pGA7(iQ5eqYn9~Ogjun^Tu`~}TGzQ_2k;8Mw_sReHi^k;W>NH#Sv=6=M@eW1!JBFi z_;FvZ5xgJo$@fY%n4GLh`_kq|SZ&C&;aqzaVHJE|o=Y)`OPuhwfbvZN1eRyM`^H&M zWZOOZ%3YXfBazTAX|F#SXl+x3ac@r10mcQ>%YSQi;F`s!Db3noyma2h%fMz}@E?aw zHy1WBMQBE-{2{Q}9YO~f8%%q>o~-~jBSI<7+F(3mZ*GSVxoXGV@BZ9<&&DcyBX^$X z-PY*RykyV+ZdsJ;I{=(|FtE8=B4vyQmDWcoLCR;r1ZRacC~Mi-lOW|L6=DC=)u8=i zd;(bB`_C0MT^=q|%Je9p^@18KZ_-yfz_?&qo^v=HT+`zlrCA${kJ(_E32Y*B<7(|T zu2K#yMz5-3jsu&ReRP1a!F1*XqZ_cPcham4#y9>csUApD=6vckhHHQL#bF~K3t~1D zTARSRI+D=$j7{m7_7q4V?|*~XfS6&UMKd^en$^ESI%fXn*lZBft)1YEGVXKq z%(*fU)B81HSz!$8!lMp@^_47mYxi*ZWY%br=VNsNSYN!F4lpj5?#wDo0oR5MK`eRUB9=N7@C#6{%j1SJgY6Uhc{mL9Ru3od2 zL&*Q`qC8;pPDuwC8%)Q0Z(Rs%s)tgVwZXVC|KLp!v!r$DZToJz!u^Xarl~Da)vwxnFfQ8kW}PRyl4%!rWYtqhMfU? zKu|aQgNYeSiqFI+51op|2b9M)A0bNAi~z&97$=S)YGgaK$J6|dz5Vm%`<;B}_vPLu z%bdUQ?~@O6hvZ3@UM}lh7U-ihW$O*<#6rQ4uZdZu=0D9B45HDbGiIAMsKtUUTe3~t z)Mk@DUofeK2`V#JpC@EGpA`RczBH7ao4Pz`+nXhw>OsbAk>qE|@O(I=LEL^Nx#V zZ7?3W)zScLCP%$>+IYILiM67D%S{}xITTF?7#mD04512O^Tm8hvo;t%YdL<=pIr6v zZ(|{x;M?BfX!HTk^InD7d496K|8{&8P^kx}{`pDwRf&|b8Aw-JKOLkTiXu2GoPpZB z#akd{g_^MM>1Lpi72yM5dBZO**(N;Prj+SWsxa^aSl;P>=>X$`Y5%ePE5S93Mk&qO zV7zQW<{eAJC}TEf(i}U8IlP;&tT2Yd zpCycd_3doGe_b-BO^5ufmXlz8dpqd>NEW&u>mp9H2pjP&b|34q)j?z zRY7k$h#B2aa7G#TS#jNS9K`H@$Fjm0R`Z1>u)cF?e>ptdzL~ZxH0|EH!(e^$wRC`S z!E|D!MGLO!wT{xP4aO}!lV1UwtcHSi$(S}PIybjH57?A7(E--Rih}Od76BV&Go|0N zv7*_vi+%?&b2Slmhm99}GneYu4}o*L!awhjh}oBp%ma-_Kuo!c;EXb6H<`=fXV&(P zWrZ;uqkM7uDSeUO0zZ?*9zL|Uw~Ok*}dcZx~_u`WJlqcN_fg<`7M>Mf{q(N&`H z6*u3;WoW~S8WyuS6(RV*2O>~4pjkA*Ld4hRmWAO~k)S)YXQ%lcd;4ee{Z78~`*PZR zF7SD~Kfi*X6qQ%+oAq#3LZn$VS#4_VI*Y9|d+H*!@KLG7=CF%qyGblkJIYqCHi=)U z*NfIti>S5?QpRFyiNzTBW+3PEU&-X$g}U5$p1Z(Z8a!7OBglo}qNUJoa~MnP4o58K z`Vck$&~kPh!EaqXl*kXeNbr$Dln}APVhgg}ljSXmge}PIOyev5XXQe)z!zB^g+bV* z^!BC_SoZZF|MDN4+~W6$mW41W$VM?s6UQczfrlOPsk~542N)Mj$KSQe!8JWWnzh0B zjIG)tU^9F2K)_~52%8)?>gZcD3E0g2m<}*Dm_GbVbq}yfokD5W2IIz>ivBoq)#rC! zC-GCFn=bnsjSvJOY@Mkjmg{==O>X;{4dB!`mFJ2?${Yg{(;I6*$__QbSz!ZmX+Dnu zDUW3m_B~w#s&ooB!ScHPu5TFh2$_;IBjdqozTrlnZzAOb?^I9vVSsRQi7T0zH8#LN{Lo%kb3e7vP zz7E*b#M1%B2Ggw>O?qIX8$)T<2IDJV^mstb@W@e(#0JFdOcvY0xx6>&rgTitr%_qp zWv+Nka7G#TsXJAh2Vy376P6Xmu-!545m?{GCBOSULcYv~0+jokZX#G;+A=!8xM2E> z&lCx+c_K)&HWCF-`yOyK5&eMG$=1K*@8D-4AvBiVtWRF9Xz>e&`p0xGLkYtlF97Sy+C~Q$7fg$F1sZV8wjj;gVB9g&J_Oi^iRS}0 zp?+pvN6~q1(|llaAejy@HkjTgi`)ThtSOXcZ7?qP3{`@dg%65b{ksW=F<-XroC(g= z`*wOHVsZ;n%6H-LW!`^Ha7G!kU*7wy9mGr@AS^44;mUqjJy>5v+~(VoFQ%%W*02zkdzhJ1?$I5L+5Oh)!OQtm&`79p$kD5 zQL;=jT3Lb--N_2@i{tE*6f(&Nd8emL)!sWW-l6r1g~jIbKjt58l+L)ZX*1kbHDSTa1YMsc7%gup)e zpS!vCGuvGrBYeD=XlA48(o(aW479xnh~aLWqXUc!riB9&V!<^vm6T>}Fh1>p;{>qr zkLq>X4Dezz*NP77^O}LpS`{5&Y%tw=wJ09g42-5UYlHD84bp8v$zU3R0d>5%xdb zR8(EF!3mZ(w##Pk_wbrhqD7VG_nrjHo77DQ7#B=S{lZ|%kYAK$Z7`m&rM4Z|Y&jTz z%Vpy&W$0?RC{onl*+~vnzZX2Hn z5YwTa*9Oil$nB{Ui781%@(O(xh^fvaIHQbNUQuKOF{?KdmKDbEQdLhBSl^sRXT4}l zyA}-+O3#7yIh*JJ0U~+HW+W4_o)Qf{D=V^zVK5MPf=4 zQLg6C5AZTK_Y#~@#%$5`6(t~MLmy#TVGPSPY1_g2f`hiUh{h~mj!MvmB(T2yl_sLY_yJOZ=V?e&%CZQuTs-}CT$pC5A(68q=xv50K0$u0bD zM`&HBr>n-K)*0ksTeSwWB6yXYpK8_`1fxk~RGT)+g`~tpwP~9?*`zaTO>%876_=pX zYvb&Hc^z|Lq9572a>=5x-0WMD`@MUEM)ER#s7V`dGzf8eqaciu%5` z;ltg#Pw?UVFn-7;t)ZW_c-&U{5LPw)(*$na1H#I9cb?m*6XN^f)P}6N{seZ~(bCSf zT-amtaCdTGqEetHX;iH*8K|q51#|WJbbxWe^klD2GkE6J%amqqFkYpQP69U4Y29`k zX9qS5t>|dNwn||0b~+tkY%uNmbEyj0+?+va)&}FzI}6r~Ay57CqSKeFaW8*hYb51) z-es#=KT6Wjvpe8r@ei;yJFx1ZNXmsuRQ2urVUTjLoZzgm5WTocM!8Zo#2czW}A|NCW4r+_7Rp9#&F2+fHH8t1?9D6qA^P} zXpre_J~*FmB^_W~Fdca(dJ=f%hkly1!Fc@a(kx&TFkBsf|WnKA6Jhvf3pZX8>kF5CSOJIi`9|kyS z%`U6QBnLYtDdFhs;;vcVWT36xHJD3ipaYBxrh8U)#er)&-=;KcgYn|T5FcRUDF0=% z8D+;N(v0?2zs>|U8>8p|V}t4RtGf$ncZ~D+GqmL z^P@K_vOGkV;oQ|>S|4y~o~)%tAZ26{QWQTHfs{o{2+j&8p~}*^_d!a}D8l}yn}lLw z>pQ^m4E&AS5f8g5MM{)CId~>mUdIGFz_?&~-dMM0aLr0FrCA${TlBKiz~*p;yv}N4 zFXi+K)HG(_S^J?gAU*IZCrO7$5K`8kkB_=6`XV&H48B+iV;HLCk=h zaXH}J)rW@a1!9VlP~yR(K_F&b0l^t%%sQLrTmUi877~^f#<0Q1^AcEJZpVuT!I-s5 zl)F8130R-Hn+`B8m_9w2kOr<9`G(T04aWUS4#=m2Ab z>6n4NkAThm`IKgDFg~~@dk%=%GsEpRu>mowo`hZj=WhM#eMdOv_h+0Jh#5aba7G#T zi3nBXgO~|F2+InOd`hmhEduM?rtY|BM&M4pN$#$`8K`)F>_-5 znyogY;gh-6AiFzO3VPfN{a}5u@cDxTd6((yR@}6P_>M2Vy=7zGAbn_cdEy zf=-m~UIT1$L+AiwgXuBl?JmH^GnCS-4aTeIj#>p`rhH7hZ{5up81vJix2M3lUE+f+ z0x?B#sMlk^KZx1mMsP+MvsEqe1t8{icfzv57;YaPdJ5Jjm+X8X7_&ScZMoDg0qavN aqyvl#rgKu&@UOwLFiNvF81M5M7XJYS{wDDN diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659943.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659943.0 deleted file mode 100644 index daf6b320f7158aedc904b61798e8e6dbaf2737ab..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{lYe-X39Kh#%uzRzmGkc+y7-7?9S!Nj?H52VYSyWKT=BC#&bfeulE6G;&GU+M3 zjLK}%i=YH6dTr~eX=R_%=&c}GeXyc3Dm%{8k^gD;ejLC5;dlN&?rw&x|M{~&;~2Lh zF8%uVl(v*ngAHbVzEP8A6^xdI*hLyX+9DW5li6U>n{zc{QDLFpyiQYW&bJ6=jnGHw z^70D=9r-2l{F>3^+=X*zjo^A)WHz{`V4b^=nER2Ra?K-&vdEGOQk1B?r%Z*I5|53V`+4y9QejKBTR-~cv%X8rEB zk$bR7wxezGw?<%NO{W8l4W@EWwI8PU!Uy@Qa+nWa8_80HhfA@fRuTug#Axfi<+uj zbzpgO`5VqY505Ej26TA&wFI!dN`E@QxM12fdCDVj&5*&AW^FKjq)a&$*woafx!g9M zQjV!b#lI}j=ErF|z}R5g_rd}Nu<<%eY1Rhg`;wD7Mv{~j-HJ%=Zr7K78!t78IX-KP zA2>I4ch7B!m@+Nml}kH7%z1kW&M0H{gl%glh&lEkVOe1eFR4=7!TL78YP};F(`i74 zjv5ZcT>Xv?FfN#0V#_@auBmRPG;4$LWTmbW*z`mj?n=hA)uML4u8Y7%9zzEh8%!@w z3*QB7b_~$04aQq0IZlF@2P1;+5gQQmoS2pl&eeSlyDuHHHAWT)VxH+CIHQdFc+Z&r z6~uh;ld!BXhU0CzJz#y-!Y;Rm$2ZfNjjT?~DX_jTMmoT_V7fPJ-EVNsIuoT?8;q-; zUorukoQFjZBx5@5XjfCB7T9Q;=>ThEM>D4AeSl5p0R5kh9WCftc>u%=$&6@l+jzq_ zvtespF*x^K{Io`im~XOBc5rtXh^b2?IHQc&(zPS@gP0=`VOe1eb3PT3V145cjeF=8 z_VmTHEk{Y4%A&yfo>$QU#s$+EE%hhCHP0TUG;4!#M_F?buz4I?-*4mTZ?^gf`dL|V z4cL^#(E-K=)1ths2H1?6N@>;x<5!~OnIPt#PkBx5-T1(mwtzSoGhDf|St6!vAsXga zz6`{CuOc|3jM>KSAOXZ|4I(TnjA655y$h_bJ*xDPWX$SC=t*E`J6K=hL^{B@V0wLN T#7nTg36m(z+F*S2m@?O2V(K4z diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659944.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659944.0 deleted file mode 100644 index 2e5fc78c65f4f2b5304433febd779d289cf5e3ee..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{leN4<@9Ke59Out`J>Jp)}2+`Ftvf`m=7xETDv%Bs%?Wl{p>uAl(jZ8EeBY9i1 zYW(G8Xy{T(lrYB9yhV9g6nV*c2~T&A=k|Tv@BVC`?{}Z)``L8~oc;5AU&1k-pDe%f zEw(w<-$SieX?3!AlSXHZj!cpXw!g>o{0>4UYSm}VnYFZu!5m=J*zMGd_ zHQPMABu-3HqfuGhi;w^^(EP+Zk{?t<2N)Mj=iZ+*4?J^JEu~o-j1O#3>;yJ$@jZ4M z7biB;OVEz;r)z=DQaK%9Y%o18e}6w<)9s*H8;om)J`VRIPknR88o<9AeWTLWsJ|cx zt|nEwFW2&COPyhV8Q8i!w(+V+%IOMpddc~|AZ6eTg0sR3)bGy$SCCSW6ZSt{1s3%*|yiODI@HTPT&#B3}iIHQc&&e3BdK}_=@!m`2`PORMN0?v2o*`pfKm{n?2 z5vMu=&X@C&4lpj5PHu|Y1)e#zmC~#Y#@9!;E(SJ3!ql~*F}IeY4#k6Qz$QDK4lp*D z{(3rXBe3y`pfqcP@#LQNY7ldKu+Md317gM`>N~;S@-G2(;xX?nZ4Lo3J-!p1QO13Q zXUcORX45~G6~?e>^9Ln3-qS>V4d|7oR3>a2N)Mjhnl+N;F%T&&DvnR z-SeOk*vxInx*-~~ssz1DG2I0=NsV-XwJAZ-1!tcFn^p(?pG^tM-4k&P#5^%S_@>Rq z6@D|X{FvYe_DZ8B*Nen#S%Lh8p0^<8rx=1W%9xeBm-v8~mMMf~g)uzMKW!*D-^jv{ z2Ai<6U(Bs3$XpTl37oI3hz>9=m_C@l;0}0Z?O{r@HW(Lt$HoAgyO9^{HqQRd78fD8 zC1?e(xjc~$FgBQW@1B?gY>bmA&Dvo6_w02=Af~co^)1_N`oNe+6WdH+Z{>hpjUq9* zg=l2G`6r0k;X`mn8MDjweOwG;K9mxc6~=HyVU`~_U)#9Nw?$(XC!r4!zv{sG)`Zgm Y#s$-!oz2(5Gqn+vW^FLeXQ#XT1*MZFTL1t6 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659945.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659945.0 deleted file mode 100644 index ef79dc1b783089ad781111791db0157fd5740b7b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{mSxi$w6hMpm!Fz}zYH*`qP*j9g2}C72S`i_l5yT~KrL7dKLW?b88Wkn2QE@>K zH;95siSXf%3PA;_btwXFi5tcRB?vM2p(27C4(&M29sBy(oI5#lPu^?eM9$Bj!nm`X zF(UT*cUg;UoVQA^)MzC!*=lW8RQL)BADX4s8gzPMPb zVtzYq2D!0WU_sf@#X-O(FqRH5HkkIR^REOpRu|3MU_83-Tecs$YR7|KF;_jse#6md zD9`hQvz6(iMXmjN>X$7)0#419HP;KITp&l!x3%Vglp_`soE4TM#r#!aAm#gi?0>p) zw6^3`KUm%tzScV6;Wnj6g}mk@hJocx^QHrg3#QwQjl01$?Jk)%U*Id_0Y1RhgT`MDPz-DZi>b79az2)eW{D}$JOq0+7 z#s<@hnTI97=8crntPRHJw*54Lm_~^re8Z8}QTrh2rg`Wf0T)mCb ztPRFX4>mLc8*SruyI@SK1tk?$1^}DPW;(#ySWr=HM;@^0anb+TSkOUpjU2=rlMvM4 zuo(=WOur@mb>LiX*6e!%FzRELnfN{a}%hx(PxaO>4O0zZ?4>fOX0ydArtDH9OUbAIq zkg~d~3fR1hpaYBzraL}Cjz`DF9`esJ#OVR_90 zF-3`}_WWb`_s8x_a7G!k%PLHTAf|03VOe1edoDuz!TRz-bM6bqEL(wom4-!v^>s_> Z0ONw`-o9NbaLuY2lxA%(UUDtr&|koPEs_8L diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659946.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659946.0 deleted file mode 100644 index 1660384d92ac387aa8bd00089a6d81a7db023590..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{ldq`7J9Dui8bZ?rEvNFmvwQST(rKfSq7nAIzrGL1ZZaQVCTW#ukP}*Ayi|mbs zCbI`o6s%IL+{(;MqbQ<>n$ZiatSB?1TF32lzdxn1$GLA| zvf7@Arx9aaReGgHD~-ulYjfv@CP?|IxoWLJr&sBe`gExwFDFN-Une!{HMwfNRNY6V zWN5P0DUK&Cmba%TkaOeXqQ`Jy4@B4c&XtVhJ^5jJb(&6VNXgb23|^v&;pjqFR_Zu{ zH(C#l=O#TSxCigX56x6-``IV!mV6jb*wo+`{@j~?tS9fzbLkpGT0i`y-jwb~VDW|z z-?@Og&2|rWu@jTjxu{xhYV;ukPg^}hxg-l6U|cZ$Ox`UA*9^2$nzg~W$Hs|uz-C0u zZ->nQCpHUAD9PVg1Z-x@=m2AbX|DUX6455jwV-qckhP}cYpHj>-I)2 zJkJlzS7v*OI&1}V0yVAR)cFz3bpk0DD3Jb_xe27~i6%HJtUz;Yi)0|BIfk(R=_*iQ z-I5Zpye$5zrO(4@N|6dB`95y}%RA&k2N)MjTR$7~!8I4UQku2F_zaJPNMN(ED#mKJ zahB4z3h6H2i3B$0Q*?l_!So-^lyYFR_%x+i8;mcnkeR(n%H3av25~bxzdCHh5)gC8 zrD8vL?w;No0x?AjRII&d12O0CCpe>w*_R*9=^$qO0m8Dv7`|T0Z3XKqXn%53Fs4O? z3f5;uf%PT7q63TzrqdFoqrf#!zNR#5gK@=~40m7?JWW+E7_+bnl@#qw1~zfi=>TJc z>6=RpJAuur5K6N)7_Y3~t^+ZR0V8e^8xV8b#HMg?uH>6fgK$jL>FNn$%6kdUDC0h3 zy4u=7%#fdiWrZ>Pt2}TYSYKXFr`^Npn`uc!zEStKf%Pd?(*ec>)1fJ!y}>mL)=-+Y z!T7u_4Yz=evMKMjU`&e%4H0{n0h_{RI>6fCPqHoRIk1Utq4a+?CiF%ervWinEc0)) z+YE$n=FDN?1HidQX3f4M5VKw*&oNtHi4Lx^9aidWB6h7X$`=}#*%%fL0O`)Sq&WnCF}?YkKSV|rQLLcqD@F1wlq zVv6EXebk8n5c8cI!5L-DzL?Ni2x6XhCoC(B;ftlqE`aqJr*3Kyj9HO@UY_gd0qe`0 aP6rqlOm|lYOTabdA(UoqFkadg9sU>fhbch- diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659947.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc11.1659947.0 deleted file mode 100644 index 93a91c305b23de1449f619fc7c25b28087e1a630..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{ldq`7J9Drxqm8)f2kM!kX4=NLAsjD`96Ifm`3tp=@$H>mg_ zDodly9yg%wROd;p0mnDTViyp~t51l(_JR8K0Ht@N6oiQsI-n;rc|$(S634PwW#%toy-SA?hc51yP#V4od5 zznHmqOWYn~eY}`t$D^5qM z=-RpBO0Ybhy=5Kp@S0M_A=K@287%MW7-4{N!Sq^J(L8X?LVtm#Z7{B0)%^?DT&_*7 zciVVN8D5M0EO8mY#&Jd%plmSR(*AG*uqikz(6kN4s|q?@6G_Sg&QTFe_^0n48=pBK z=8E1e95|ON?`x2VDa%9y8O?t{%&?;br<5`KxO(Jd5Hq5Ju(U9S)z+Q{u)b}tpEpXz zv~p<5i*gNEpX!}3K)GPL(ckq-ZJO>Y!;Pcr82TBJxbr2(6$Xkmb| z!Ss&!GsA(6L!@aNj4SQjBoH%28Qer{K+L|=2OGh;^M5GrOUIns7RP~@v3&%mlyRT8 z!L}L@Q`JvcS{TEJG`te5??isL+r#UVY0XA6GSYj%`jlIQ0m=o_!KL3y!8H{H0!`as zJYiOSDX>XzDQuREX|)1U8>zu6k^|y=KjoXqq+~ z{YGBv6S zz48^rWX2PmQpW6&iV4*q=JkIpEsSBV#8v^;ml(D4p=3;R8Y+?Ry#Qk7MhgR!3#J3g TG_%1q2Sl2-!T6<+4@>?6@@6is diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917688.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917688.0 deleted file mode 100644 index 2290e7627cb216348993fc04b8eea62400f7354d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{ldrVVT9Ds{V>AlFSluU=MAmR+N8krbImJ?86M572oG;XxCa%-5-hFcUDH()A3 zG)8njqmXLQEse$mqmzKO2=Z`6-13NLFtTzMS{-z_@<<15rI!BvPhmQMJCc6{NJs5}Xy*qVYxP z;UMLiIfQ*rSBv~U$$S8o=YG=cnDFqKQf5H*=SC}7-q0`|U|cY*NMEJ{*SzSWG;4$L z+e2x3V3Sjo*6OkGl``rSI{x*+6kyX-O$QhoOuxuYN(44*c2b(P!T4{=u*&Hqf zvN}$;0h@&(bbztJ^puTHqkv6xD5Y5&jPGpOUJhce^$)*BY(UKPy!-GYJJP4_kdC=w zS#BbTY3V08qm27FbyWc%=86HrvceeFEp)rU`hHz85MK z`j{#5B3Pe1o6@We#(Y=56gKN(FmeQ;Z#`}!N zY``Ws{DRlU*Jsvx6q&Cl!~>f_B^_XFFdbQvR|agf5tL?aFrMks-T^VoZy0~^?B)X) zvwm{pVsLJStENXHrYsB9b=oF_n6^=ZGs>6^SWvJO#N0DRSXLOr|0q>If%P2?EWaTc m)0&Mwf4+ABtnWk+9bjBAUG{ob2)L#(n9{5b#;Z$Wh5rHm!!AYu diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917689.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917689.0 deleted file mode 100644 index 384fa70195a490955a6ffe09cfc231622eaa1404..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{le@t6d6oAVh8Sn992v$HT3OFE$MaUF0!gU*~ATR=rkVvGZ9c5vGlD$`^a}&^6 z2D*`LMh!8_Zwwd^MIl0jR&lUVV6u_ORK^lr6J*Fn!Z6e6rM3B@(3KgLs-p&@X?K!Z$;X=vMzK=RSL%ckrC3!_p%acN zUlYnq20>}qqO_%DPI3#lK55q#46fr4usB=|%84&M8a!RTT~6Es={VT%*4M{%kl!UpmIJXcaC9(2LW zMK8}q5ZL4R$_8ip^eu-+fQK8C;@!wR`a)tj8Cdu=Fp2xAg$^(-nD#CTco#g=*h*>E z2IDV=ZN>nb6Hor@wAtpyW={tSNjZB2*oe_|fU&`}f9iAfz-Gfmvo;t{Y)q~QCQr4` zY=(35+b2dHjeL2Y_dKdAm&=y^t5N&hISsaEMcBq9Qtr_pt9?BTq-@?va8_7@o{mjU z2Pw;=3HyMq1}!PZQo-@kmq#pH9&S_0^e8PeWdIz{wn7IO7fe6(r&l+4=BZnhW^FKD zvk>YFY{twgtHZ`!$|pL}Y-eEsu<@>^1B?x(gVJhqflZT(W^FJYUopEsl%#y`ws$PI zd2!uo;}Hj9&fPuT2KN4D{QG-}m@*9txgQ@2Vz!F}XOuCU8)@nXF^`!D%L-%I{`rM8 zaK50?pMQ{yY0;wv^R8@gzD)}qU|cZ0&(;s54Yb;OEK#EqcH-U@5@{|z_?)g zjeNyU@XSMblxA%(ej~mA46v#0shW_CY3V>SeK)&+jjERpur?j&E2}&h*yQz5`T?5` zl$oE?4Pt5&q9z?Sp75J#S52*ey}rTEUzCWs^eWnJ`RExC^WI|wXOuC!x^|@*#1s{T zWrZ;u+SFPD&R6hy^d*O|yI;)OLiDx$+*xqG#^ZE=al!P}Gd{0?XR1$7nzg~WE%Ms| zV3Qd1jnl^6zuA^1^x3K5Bw!OGrvr=)rcXZNR|ssbxoFk~<3h%G0*Lwcbm^4iHeN8M z@^;QUVDI3q2Af1oSswB{AO0+exp;@*j521;t7;pFd3l|%tT2YxlTzxy`DT5przK;y jsL{}0C+~stW&6_s#s$-J8~HQfne#51wZVALP(#lDd7>s6 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917690.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917690.0 deleted file mode 100644 index bd72070c85a935adaebc85a9b3b37e215360ebd1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{ldq|UO7{F&n^ZP6{%}UW(Sr$?gEjxC5nU$nOnOa5>n_I0l(uv=^RI8kr6Sy^gftGqza=V}GH zc92qS)Msi{uD_&D4u2R%_NFXdJf4fVD7iDZH*yT`&wC5n_1OlaDl^+?43t#)pdIah zIYJ2D+E5Y7Ex$(a(Y!D3rPCUQ*mJXk`$7qu5#c&-?M*2EB3p5d2*J|DuTm z_AvE);tGxzIX(P5+?Zs{MQKSE9~l|weC{8^MV+Jrj0>ihMCYi$GY5uf)&}F|BPT+D z&E?78TsFhp*eGl$?(V8TfX%L{bbztJwBpL^JHY1PG)l8J7$03~nLmy^_4Sj_GA>@) zQs-mZ!o9$z z!~u`mEz$!CvV> z2+k{dQYIJE0 zAZC74c&pRK6Mi#Im1+Nhy~oDI-4}`3wGNqg?>-4)&Iu(rqm0=S>84N+bAgPotT2X) zF6Q-t^F?lnc;FOv_ls#>i`oh*6ySU%COW{lV0u#HlU(r39os3*+F(5HlH(?@NeZyL zY~1~ut+Jw;o<1*NQxHf87#mFYwwE0NHpyct&DvnxV!he~VirB#_|SQq;V|Y1)BY)7 z@AY2GBaxVr6y#Yhiv=-LJ`$W!#%xga+71xY)JIrW7{g(2v?sv%UU=_(EE=;a6}2Vw fJ_YBC_N4=i3#LoU<|Tt?z8|7l8;tu#ZyWm)z``Zg diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917691.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917691.0 deleted file mode 100644 index 2006fc955b0da743eee7daf5400e2a1bfda100d9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{le@K&Q9Kbh!_P*_Eu4EOutgNi_R`f?icn?Z-4N5PqxUR0vP2Xan8@%t#eqAXM z8Wu(1(XF~BO$U|L8RFe}sZ&qOOsKP{?1x2Rlp-^z6{S6H56}2M?cJa9`F{C4-w$&J zMes9v!f=K&s8%<=&v=kA;~R@;w%YWob_zDf@+CQXKGh-EB)e#_o5g&+R8&}K7I*85 zMXN&)^};x1+GZ^fOu@fYI4_-@N%m&1SvigS_^0C9_}+z6c?~~N6!Ppg$y8vMq)y&l?$|#BkK-rv6SfPs5ZkawYDgq(?xLr2xXebvYIrrzjQ*Pq!Bp4neTY1Rhg zby@u#z@|3&bI>Ltj7_>5rF=X40I;c_M+X=iOe@p73gLys57nTgE-Yrc_wa!ONq5aJ->GI>5MKTBt3a51!dPL}}IraD$_n3v6i(Ci zHfW<<2x9iWvAh6#ZxnuLm5He^qKM?PWgwX$`X$QJ+8$7eJmC~#Y#(yrVe!x= zU=yRJ1B?x(uc$uxfX%}=O0zZ?uk~t=f|z$Bru;{2K+Kn&4MkvY{#{*%d`y3q?GA{E z9uk~U#(g4d)|7&nv3-PPg)yAFT3H9q*Pi)0;1Twl*^-O&|9_|f=PO-D2N)Mj|MYKK zGkE6iY)Z2>7}um$J_a^9=ZiXJW45@_x#NdLV3Tu^4zM9NW*2!uOlKltSz!#1 zN#$eUd~0{j>k0^m`^7BTjHVm^jt1wOypIkrE|_kbv9=04vo}PuHW=?-pjiQI=1!>( z+JyTz>-rtdZok+EY;^H-fU&{!j|nb4usIi^SsRS+_~cs(Vp@B)-3;6&3dZzUn>xW> zap-WjOiV>KsvoqS1~L6Uf-}mPwH;Yd31WIj2+Im%*ivJt2Io6DasMsZn64b8+^S0h d=X0s(0ONw`K6msn@XVx1lxA%(zQyjT`vRFoD*XTe diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917692.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc12.2917692.0 deleted file mode 100644 index 0553e2f572f9ad55f1258cca880f4fdd9a138386..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{leN4<@9KbJW`u(I*l9wyx6j{40&CJWAri2>_WeZ(*x$LaFv)`3tk!(V)*lgsr z7rNQJWVXymr$VLj(r7f9W;PbWxuur3r@P0~zK{FepY8Mg?(=*g5SLH2Rzc(QD;=WR6B}Gze;gO4uSd=IV4RVVB$_Xmd1zTr*55 zv$WY7rSmVU*CC2&WN)$}eiFC;nxt-c@9YV@A3s*mWE%8FWwya+43JcKqy7-!* zma=^g4_`MICK(IR@#^bKg2_Nvw_h~(yPOU%E|``r3hV^Wtf`cN%3#xUcT z(`J+lo7gJ!OKG$Ko5XNBz}R3qX5RFhz@~MCW^FKT&AS>ANS^xs@jx*5aZIz-(a4ME zdG{Tv>;Os6pS{R5O9r;y2x-47k}@_O4Q!3mf|N*w!`EHQYMEH_CMWp)LVL~ z5*%+zZ*Bdshs%@_H8KyrnE;L#^O+7XE|`87DLnz68Q(`~)&}DrO2YGjjZm1-;IMI( zGQ@(eMVJo*8`&W`z}R5=&d0*Nz$UDS(yR@}mzF-snoLrfzj@B$65GBzZQN#qn8m57 zR`0mcQ>x`EIk@XWAAO0zZ?pPJHg1lZWT)b~YW7FbZq7n>T`bb8YP#s<@8 zdz6X5M(IOo)&}DJ)^#bM(P zznL zhp?+(%z`vDro=oGobTWsI>5MKdYRB?1JBIJr!;GW@eLa-GGH^;|Ay1X)xX(_Qz$Z} zEEm`u37`Xv4W_HV-iiV?u@foH+F)Gjt5$=U3m<2-Ic_r=#x$HPc@FmWe=celi781& zo-6EqAZEvRf-}mPeYN|t7R0Qv6P6Xma6nAaVsO6FvH6cgV^*w1>#M7+;Cy;-I>5MK W`e(Pa59)*hr6TH@1 z8o*i365N~jl!^&~WAp~~G{rH1B zv2U&2!^_2qNy=Ds-Z(8*N(P#meImK|{d9nF!Ss$M>kRPB4_!2CgYmDwQxbuVY3Of< zO*bbtF(#DP@C*T)(;;+#vB7lCn5O5zW=klgSsRSE#4LO5PoDbj?oTN%VtUlU0gqe3-356MBw@(_Zv!YX8wnx}%4`JsgUPgjKs zW;MHmkwYR5;mc zw{ey-r~;+=mqi1cIUDH!V}t3U4%Go*lVzkdYlHDg6AOph&SumA3_aR~!4 z&E|2Bz}~9NU)M!qN>nH=Froy+T&5>Dqm0?{vfv~Tvr!-{D~#bmW2$F^^VupJt3+d3 z)JS>A76Q)q%0dSi7ffGhjH(0AEVELYwZZtGlY71a8;QHRS~O-}1qu#0`3BgydC~#K z2GjMAdu#lhgY#{UZ?$_k{bpKHkW|>`1ThELQ{TDI|^-1Q2IX`6KW2h7Ybs&9T{9} zw{eBv%z_P}AHd#g{-f(eV!oJ%!i-S`AZEp2f-}mPEy}!g1jO7LKv-57!wp#%%;0>s zr6KipVQ0UXc}ZyguG%DUzLJ%6fN{ZeUE;wB;F(ueQJS^ExTgJzE3g^md(L6w?B8tp z9(1_qQY5gM>qiF|8%)o3$u|NU|9+HaZ7{yRtj`S)GxtvVE&FY{!dslBlU8 ze@2~AG-@SfOeVvO#w7@-1y?o~6qV=%Q33{y#w99Ii>iqR$nTX`78=esa768e zo=5@@D17yiw^bB-J%$aCFwrHWLX-2pf(*1hk;n7@oT3Aa3#P?F?IN(|($kb?Z7{C1 zW~hM8ud}}SY<`lkNpT^aY*Zbv=~B@F#s<^PEvw+scP3JrwZVAV;=eYHBCB@a`mErK z-*;T{HX1AlLeOS?UMTmpuc#&|GZpL@F{9~nfRrg(ba04y4M>@hKyX%Ai!$yWdJ9rc zQ4#hBT`fA9ozVb}XYQ?b_j^dD(`t*m!J1uunzg~WeRApw zVBTJc=@$ysGGOEO)2t1~bsd#yV@S%qZ-Zm_)~1j; zavKn{e984X@NSo_*9%Jg{&8CLZ27|u5Houx!5L-D2BpgzLCm;agk^;>T%qdY!1?UW zkFEra={BIR(SMeL^R?Wj1B?r%XWE35U`^{CO0zZ?KU@5AEwE`EZ@3yTro9|3Tpt<- zY;Gy&0Aqvcwx1s*0~>WXrCA${uZZsSfSCVwCtM>oASQn(p%1(}r(1EoAC!*y>c20? zKuq&%f-}mv&%7I@_drbX4PjYf3~zbjnG4Q$XI+oiL-H@vtwSLNJ?-Fp+jMk*al!Ps z^Ho#9nsfD(W^FJYW0oBNHb<)pZUl_!b|LZcjxWHbvxW|^HZIiKY0d;TJ1$cC2OAeU zF)QgCh-t4#x#_hDiUl!WCRW`B?{1Gtst*wJX(k$OZi@#oE2k5jQO0cT^3`e(^N5nL ztT2Y%@_ZXO-(kn32CuNxe@y!tH1c_(1Dr3pgbpw+n2tRj{Sd6#;ip*}j7LS3sez4b z;yIs<)Yq)D6lrD#7Xh1wNpyg*!L)X?BL>)rlPS&GVBED|(+Xm`q8yFh+sI(df%lib z2k(yc>}d)RlT)K2!Oa_Cw1AUWnZ~4CB%#nS<+ULd&C4=$Bu{sb=k|Tv@BVC`?{}Z)``L9fMUKy(y)%z-v4Jy6 ze}p%L5A3Zmsr5!h)M}kEb?UfS1s|NMGn!LOniREZnZlg5YL(iwPO-+MPt}Uk7ZA<2xNT z?k;S?3ekbrF`>X_MF<^WY%rZ)@4gY(WK5(qYlCs^`o_0@FsiRBTt^^ z#jDi@si?7Q%dF`~)4|p!V`{Doqzp?yfcxIK8W^FKja%}lMU~_6;RE6Ef zRm#Cd=v{7WG_bk6pAIlKn7&f=v>e#<%A+)EgYj!k(^~vV%IvR_5uD$%c887IND%Xa zTt61*xUEg6W|~izj$y+#^b}HW+UW^|An)<^WBlV9fL)G)l7m7qE#8qyvl% zru)wk9|kthoHT2L@o5bQ{6Wm)zI|^I8xS*7J68qv<~GTygkx4@&vpkf13wa+QO12@ zlP?y4n36Wavcec{H|KSN^OeMZuzR@tW?EHf^xKWM!1)%a=m6t_>C}fIHtwuvvbM4zMpN6C+XXAaM$G;4$L>%YqifK9!;#9`y=->l^j zGF-kG1#DV}(gDT>(?goyTmm*Lhf$if!T9;J>N*hf(-T=?E$SKX*U_2tiCjAQ!Dk^RO diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490228.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490228.0 deleted file mode 100644 index b7a5b109ff6887f78bf4aebc6884732786123f7c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{ldrVVT9Dv*5gL@TG&`K4swXP7viiqO#4C6pD0+~eT16H7LF^zpBw*`De5S2uP z7-e&)iHJ#=(L@JCM;wUl+~lRk88=}eiivL}qQ(spL=WxpG{0kS|7^bB$#;HVPMa+A zd`7C0PjEL?YnuOG)w60wfI-YLT6Bqpg2lcpHeJU@+XaiwDjKXgVxG=cU^eH7+jT{v z(JqKOVU)_sH=2Yj&o@bmhLf|%xocCt4&ii8+2zr5qo(pf`~*?RwOVXhCacXhO;$Sz zrHos0Je1(=t|MVwaXZ1M@PWMl7Qy0WmsieIhY>b={(TL%rGv0R{A8ZXGum>!@C2o6 zrkcR=oiG36YR;9qJtoV2m}DL&=^64d=nB5i!@VNTf{AqkrxxdO^y(ID)gndK8-$6#!Cd z77_L{T|L_IY<(kG-sz!c=ctFzlrjT)+#7!gEbrYg9bjBAts1_17+iDZ2&GvYjPJ`+ zyZ|;=eoS<^ZG5Fv)uH$eiDkehrJ4>fHkdAZQu7wrXpd5wwZVAT4~;SGF_oJHa*ky)!a{%(lO6IeA5A9 zHoYV`qm29f)jezmFY`n{VOe1emrjaF1?%hn{<+)3=gV|vqm4;T#bAAdS#*GL!F1<0 z8acS;g=|W*HW*)Veu^2`ocgVxO){p_fd(Qol7P+r7COM%I8f2Lm3M*7>dTb=%*KIi zAypSa%<2S9yW7SOzRZM-ZQp@&nRMDy%=C5U-vF2NaP%$n{Vt^hF`q6y0i zW4Qg&YA0A<#(r(5TiDk>=I+g?VD7V>V14lwbbxWebe!RAD!AqYFU{Iu{7&^ROM%Uh z=Ay^O*UxP2G4xgUrggyP>6dhXvBC7;BR^?@ji{wGYlHD!gX&-q^VPllF86N6z?dP2 z62rl{XGY4eOT?6|MU9cuoiL`~0T0e7W40u1YCed0Z46;qVGOrfl(WJ5G-1UzBxBa5 iBhwM(L9jlBnhr27m^NA;$AD|Lcxl!KX61Mn+J$PS++&-FC$uLp|B>{>Mf55;i32f z7T4jJ*%D`kYIIX04h;b*1yo*wNVW(nf^qQ`BkCrud$=9ivqOHz-u`*>{Z78~`*Lqn zC7$2w+SJ3`8r9m?SIMKvZ~N;7ozbLDDKwb!6P9gI^Rf8`lf^9P%{n1lZ7JBgRVVCF z7YW9EgP=A{QyDqNJVS=($-XtR$#8OR+9w}{aSNT2w&`=11oH~sS1@FmO_q#2v&9l3 zvCTm*Z%lqQpWr1*o=kB$r+2zN0;OI|G80hg!!N#9k%7_Mie+42BOPE|F#W0D$YF3za}%Xm z8;tKhB}mV}t4X8b>Fv`DQt#SsRR-ZK*%ZBUgPe z_=k$Khju&Mjr@3?mlx{tLL?LamhXydJOfT0TY9x!BxRBob#D$>1yaVw6Py*+BF*-d z>p;pd4PoEV)uNO8>V3fSHo97!(;i+^O7zIO<%KU;-sgYO0mcQ>?bo*lgKPTyO=;E! z;}f!mXTYZ6>y!?+jklB$$I#k)iqC+}qC<3mvBC8E21h%vnV6wj8;o}p$B(^5Qr0}5 z{VsQ6^f!-VK9M#_+5= zC9A;tz8o0o5{>E9qmK)sLcsbCT%!Yw3#J_ng)QKk;X{;WZ7^=@*Hr;ISXLOr%7+7aV12_|9=kofzM0NUv^RG!0<7;)CLLg0 zFr6ND{292WMn`GZ2ICid4^{%3mX?BU(U?vGW_>r3As)#n!W_K#Vy2^GXVDgo=eUrh%X7fi3Vuc!sr?5UwNYlHFkt)~&N znfuOpkBzs_tnFJ=on_}v%8JS!(GD~#cdFNZFG^?j!-8xW0Y k+kj-nRb60xO$+G&+7?M^)&}F(Zu*!12Zfs}a{vGU diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490230.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490230.0 deleted file mode 100644 index bb6dda668e210e447e58e22641a5034e992ba247..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{lYfO_@7=R1j_&(&aDmoMjQfQcAofa_!HgZ7|;1u{Cf$x$3}; zF$q^bzxA5C(QKaQr|r}lLqzx96sXReT?$S;yQHI8Af-}`3hqa(0x9P%CpastMzVBQ z97s9xj(ty8jbhIEtq03%8n-(pJ-ntA=}fOF#FQ2hmKDbE_X`uE z!1|7L-uXo^rbCA=+*XBw_5IsL2N)MjXG|>k7F@HYo6@We#F+Rp7R^A0Wl>Xe>n)wUHVY+yKu}~ zVr~(Ld3=cAj56-y^1s;!VipY(mKDZu?+N`Au)f=0KXQ9`eVGmovOmr&1nWDdp#zKy zrnjc_7lLap(NdbV!T3CNZ!xeL{VA_iFs8$Xf}8TAfz9S?bbz(7p|Ul*{suNF&6Ix6 z#)h`+Xm|l)+LY2Zx6L&8GWp)m+rYV5v58KBnD;W!UjB*^#2k(zIHQc&(9ytZ5Oa>4 zu&gkK1F|MogY^|1mbJTuz5QeE`wBhi9yNpY?I@uGj0>h~qizL(Ypy&>Y1Rhg507-g zm-(0Uvd6~T&um>Ks{AI(32drk=m2AbY2R9-6WGXPlxA%(-q>9j0AfDv-|lknW;%@N zQ~zoPI9DM$(jgF2l!o#<$9zG|;u!>IlrcN+zb#imOzljV@c;k- diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490231.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc14.1490231.0 deleted file mode 100644 index df12fe86b0b0ee52987857203de9402284e06206..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{mdrVVT9DvI}=)F1x)cHsO2@eTEHRa(W^C&nbgkSyp0Zw3t_IR4#vA2IV-|yr*zc2T; z$r9&h>SS^iSFmJr+v|jZgry&<1y#OIp0r1+D_p0@kn>T6TAe{JsP!sABRA~cxl<+V zlNSm3g<3(bouRUF^9!_D&Tl#uk86X-x#_8&Eaq$-k}EUkMlRs}`PqUtN3Szv73d9y zg_4@NXul$+P)6|U9TfpwcQ?WP_<6i^yH@97f4ZO79zfXam4hK%R}W$Rd0(E>7;K#e%S(Jq2N)Mj&--#j1FnfCDb3o< z;E}SAfz5BHk~$nV?ouvmK&syV>VQp2B^_XFFzxeigB95D-%*;i!T7z0iA76D%ChIu z2yS>_%xU8h31XgW9NPxYZ9O<~RV1cFiTqP6X&|P(gy4)aX7`Wox&&fI9U&|$jN#B2#yY#w>0? zXHs{J0vmY{9bjxQ9V;k`fK8{1W^FJYVQWeOF<;G}|0l5lF&l$^w19Kx4hLQnk9lc* z#bFTB`jp^|GVW7T7Zwg;9)3nxRv5#>39=ZlK8f;~!^7>9Y05_BwwpSzK5rErU|cX= zS@p{o;F=vSnzh0BabxQ=u<5_N+bkN>RF7VMh86&u@^(7F+SDWaSpOAZv&ls1_iXCX zou(6^Am+q|kn0W`Pxxf^mL`^gbBz&exgkoJQuPoaMmAE1V86HkkIg-2~qg11_4i!FWrVT?1m8{?6@j?8XbmJiGPR zH{h02Q^$HmVoK7Hv^*&k#5~~nwG(HQG5aDuu?xh^^CB!OjA8HiT=<8hD&U|+G-gc( js{1Q%1gvj-8699;FrDF7>jAD=9ZYG~2IIMX>#P3<$z?N7 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436569.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436569.0 deleted file mode 100644 index 0fa703ac99784215a51c17c7daf786829811120a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{mdrVVT9DrLe3-_X;b-pS^h=2{zl8|T=^#Bo&h?zk&8sZAQTC|GA+!lzuWQiI9 zk%EzEM8OQ)QkS7|tTr-U(~1@G86ye41HKW|;3ML8XirY_JNEX^=KGy|=lA8_Hc96G ze5+16&P`FSxbQKqH%>W05OoHlCLvdEG)G6R(eU%kdZQ^@6tZ>VW{oLl#}1viOOq!W z%z9CyAEVMU4O#kh_cu-X-yY5;=cXjb&)}LIvVX?TjrfsQ@cyEHQ?}8Ro|SDfO_kM8 zM0{O^Um(G|Iu5J2Gu;HA%m?s(Tl7W`JNU-#qatiZT85m@RN9Mv%$2<11Fz- zJQzY?dBN)uuDJOZm&YWz7n6)=6cxNRCWs95_9>#cX*N2*xL~@ib@V*AX7~k4vo;uC zRW|D`u&G-z>bCLmViVhdo+(yZfz2^Q2N)YnTWz6fz~*oarCA${_oq4?)5%qz^?VBA zs)8=Jx*Cn=dEPfymo-)P=)c`*0p(TTRMn#HHi?w6T682MWDuluE+sfCtVMgqYpo!q zV;Nz;)77GArNai6w_M)RKIY*yrA$Co;W58~<@t=G1B?r%Ka>=G1lMfy(5wx{`|W*d zU{h0@(BZQ2mNK{zJ@^oE3)s9nK?fKcOz#;g>Hs#U>L|_HU_ANHRmC$&%JN}9HJ9?_ zz1v0}0b*KT#_t2?-r4)PQzE8Jiaywp~6MUm?e#9S7M?W z*aXg{1B?x(bNbHw4Q!kqnzg~WrP^{H#LSyD+dXD~#d4QeSDn`hqiGx;(r-ne7>X$`=_Tu) z9t77+Fi@Jc!MH=bT?A}~J992e#%ymuCu1L+1UAzy(E-+`0lk$M_ye035B;4@19IAQ zxgh5Cq&ZzK8(;Wjp4i@e3Y_~ca?urun2$Ch#o`;MLCk;!1ZR{n%g-F>2QeovBrGe8 zVd3_+IIzAgmT;#_*xN5=NgC>mh=~O2YpkXNj0>h!T|>{oHS-Qpnzg}r_#cZ$fX(W8 z&2AfSui5&es6eHj0BoW)bbztJ^zP&^0odI4(5wx{_iKb&5cAeR=2h2j#=)31%dejX z=ZZg+bxXvQr69|OZRbGDN}k}1GGr9WX=VGRGO7`O@6w=t~vnq6+>-P4J=WXBql;894d!HY33eNR$98{d*^`SGb ze^0b0ju@aZsdWZfQnuD$Nf@t`3DFjG?-H|jb`&uu6Q7tIm?(FMDWJi zBf^|`8XaIxKo0Y5NAHmjf@y*pDDW|HCV?ul-NU5Ama8_7_uJQvaK+3o&g#Axfg;t#B zfX0HP_9T9YZvBC7* z?6?qMlQV|WtPRGwvwk5UX0O1(w}}miIqtEs4eTBDL3&3#W?9BY9f-+&A~>Ur`^c^& zoC7gy|FNtvh8yJi6X1L!(>t6V9>1A2si;;_b`zZM;wn18xL~?{MQb&9<^~<5SsRQ$ z2@a?RHl1}@4Wcn?Y$#z=RROT!Z_@$R#)j4>R_p>cC2smZ8yi~uJ~S4@R4T$6oi<+Z zn;CsgodNa=vGI3BV!m3Ainezz05R=x1ZR{nJFzf78pPZ&fv~JFhDY0sdEk5*`y-m1 z!k&IHbC;sJD0vJxU->~gz_?&K=xf*~@J#C=O0zZ?FPYoY1#IR-SGsIG{hKX5hV)IF z_W_&JGCIK6VEXbN(*|I(EQZpo4aQ&YyO0ZFw!Tfj=e$jC7&F$g2JX%8yR%s&CYOx9 z6jnS2F&#bxXOuC!O(m-aF}L{=mKDaZRWSR2^X&-VbYC=Pu@be|+s=dY?T(}aj0>h8 VEb0>u&Sx1*Y1Rhgz3cBc{sqSRBe?(o diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436571.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436571.0 deleted file mode 100644 index e98b3c4e23c223bebe0baa98957001f7821f0397..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{lYed9DJDuukw!+u+?L;JPA6?<~-GQvmr+`tHePn zO>f8)(_DYy_Rg#iAbZ!WTt0=jHE?$a_l8ao{De_vaZ|R*l9rimu}tL3#~`7s*n2v` z8}FAWc!Qnble-+yF~{W7fd@QcU6ODrrxGBYlHFBgcqg2=G>A$ zE}LNasAr2g`+lvk`E|`ux7;*_b^ZRg0vo;t%_NB@JZ1$GL-*?)0 zN;%^^dKs>+05<#1(E-K=)5$6Ea$r+jL21?oeB(sI4sgCHuU4Jcs72uhzZkn~hc;Lr6nJJ<2BldW zjE_Iie-YUDKG^n1GG?t6nHLtU1vabd=>ThEMZHBMZUP(s21*aHv7+{m#wQ?VTjH!n zr;QwbGq>a>+Q8ngVUbM|G21sHo4m^gV%9|voKeQ?yz4DjLCjTAgk^;>{Jip{44iMv z(cosMu%}wx&_8zG#tX)bP(~xLH)!NRyF^TG z4KmI#E&(y)eF)AdW46P)WG;x=|DR=rF&vhk6bH_iHM`)6WX$qpR61!?GdN$uTspwG WU^?X_uLaM{2&FV@gYn+a_PBpQ?ICgi diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436572.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436572.0 deleted file mode 100644 index bf9f983bfb900ee5758084c720c3e89c5be8037e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{mS4>k;7=T-F-HQW?1Vx1^h*-5Cj-b{fxWE+^qG*Gq6be>L(U!V!fQccBkwB~j zXAR{Ic`#Cq3y>iy&bUycxIhgUH)tYwXipCLkG*|tzW?Mq|Cf8)7?JbSRTy)EEAorJ z)DhJbbbfPfcB4-KId|of`NO!Y4WcVO=gNli65hk8P0lcw67?A-laI(c82R;Ujvqd;q#+=zRGmGGWCm!dNz_rm3;?8T!AEY52C z#ci+1ad>!&U6`n6qQc1iNBzk_(+f#B_t8!V7#B>petWnJTr=bnrCA${-<5s14{SEh z`s=iDb72!{LuX%Y3j{W~3Oc~pV7gi=ng?tyAWE|~7(X_sF3y)+we?P?KPL^ie$~-v zAkXveTaxrXqUP>BYGqdfICVlqW2HdKNEPau7Eu9GT4xcQ6;`3}gzk|brCCAP|8!Nz zCeAGd%UdnJT-D>@GNnj^7JadogXLM>=>X$`>F7>{8@T40K9pu{Fy0g|wE&yX#}`yP zY+R+3mZI-D>vjU0*C*%zV}ognvhR6dv-cFGSsRR7-E!N8lazU12ZV6SH{YE$Vi|~8 z_t5hQI5#c#_cei-A{A0cO&SJb&OJnMMj5l^)7QQOF}LOumKDZu(1YR`V0{6PpVkP* ztkR&Y5BFoh`pz}c0mcQ>3&k%Ff@|8IQJS^E_|sJ_1A$F@kfv5JW_BskUG!1_o2YSg zfU&`Jy3{fS*gWc`SsRQ$$|`9AF*l9yt|K-eX1rvz4xH=T=3g%ybDwO(We`*Sh2V@b z?xWjVcof7;`AS$;7{l$2lgEJdMW%joc({BrtJLVGt)m{SFL6B`U|cYLI{W!fa7}L= zrCA${FAge712%p&=Ieqnt8D1t(*AZ}(^g9dSQ{I9qWA3vHs(4?|7T-EKTf}|1u@^o z1U5Kq+~JccsoB>G&ixr0aYGYb2@H-n0eC(%L-$7^QUDB zu)eT@;F}I%SHGCqN+c_(31=GrcJ09Sge|u@x2IHOKlRJRTqKOqw8&|Jc zYcXo^j*A60m2x`3*kJm=j#Zn0&6-e3vo;vdtU37y#7u5Yz2(?V9~g6aelomeSib{} z0x?A^QSg~EJ&2jd6P!`TtXVN82*k|yAS^44;Sxy;{KK(s?C#rwF|D!4S{tYW>#H9} Z2N)Mj=d9>o2G(~tgwm`H#>>M{`#&EJB|HEC diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436573.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436573.0 deleted file mode 100644 index d63a6b93f9cb9c9db8c5a2675fc2a6564c196a58..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{mZA?>F7=ViqnHCv1s$>L{FDWH686)`4+8sk8xs6*Dt0MS%Ew8ulZ753K#RHqhbUMJ;U^-MkE&!XOh|;VL#wQ(W(F`Bhv}a3g@BkN7~gO zW%GOX1Km7Sb~{)OmUqDKihat%XG)m{sUGf62g`fyPX`zmOlO?2M1X7V44^b?gYmSt z$7chZCnvL8JvP2l#-2vCJ5pMJ&4YS6z}R5gwR>ACu&HXGG;4$LuTvx4ACr{T{{<#+ zwS%Kx8$TtCnc=<$&h0-q*(MQFmWKio<3teiY$d@NWy~th{ksXoEUzLgD~#cgUiDtE zz8QV}?UFI=8g#)O@dB*R)lUZ)7fkmaQci+vUUpKNwZXV0$_Zn-;x*SLV_Ht5>p}hD zz-B~22N)YnpO~9<0oeTdDWzE(jL+KIJpp1S&YN?C*npTW$7fn_u3{vnLpr9_wd!{e z6a7zcMj7|{<>|#MAm+L;!m`2`P9FQ}If%KY=$XgE=aXqyqjQzD31EHkIy%6(V0z2v zJyvi{?i)(8HW)Y8KNto!%i7C2C1cucX#OIZ4cPd1&;i!QhVGA57=g_{)AR>6Hso5D z+6-d0#{ai-}7#B=OAN{EcT=Ux_lxA%(KJowbW423lXJAjQvpaYBzroZU1zXCQJRFr0IFg~Ht4S|?BLq&gjb`t<&R&=be zfpeG7ta3=il&wbsFOGtkVL=3ElrbB8?}r)?^YMF@6~^%PoGk-jecFY4Zc4^%%tdm2 fF9+6_oInQ{7fkcxE8rgvE|Jo#4aWQ8o1E_essk&L diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436574.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc15.1436574.0 deleted file mode 100644 index 235fd35744707fc3a49c44a197d3f8f9b69229c8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{lZA?>F7=SMmv)-#%%KU^u#S&0Liejb+as)*|-7@JvhXSo!Oa~Uy%f}ds%#`>U zKL~6>6p%rdCFl$rnPNde!4Ie!6BULDtYLu+K_?1s%ch6+VOropc?z$QF~4lp*D9;y;sflZ%_W^FKj#!|j%5xMHfy*B}T zU%;(4N29rdAk50wXD#4{-X8S2I`JboH9fNTx=hO06jU3u`ZY*7yq@5!a0=o|ZBIbT zmobEWPd5cA)w?ZVdBq;q_8AYiDLDhu6&~ln@{W1Z0mcQ>--*AR1lKH|O=;E!+JKIPJlF(m?q8q-j18t6KcA-oHh1eN&Dvmmc-kl}At{glrC82? zYM*r4c!YwOnU~Mzfpae%{<~8qCYOTH;>utUQ+1Nyj520zqxla(%=M=T%L-$7{MFS% zV153N`)i-w4@6qYtyAEM@|Cl9TAmysWQLw&-ayr1cVER&g>kr_Xx(Z6OHW*J{U49za zYzu34+PM47)>fj+r^cTHn}=FDz}R4V!$49Yu*nIhG;4$Lz(301XV!BxqsOruPZ)Fm zu!8MuV7YFT%3I7`6@+aA19Y!NoS&n6)}I hwliTTSf6qQ9bjBAU2`F046N^pi)L*wzQsG7dk5zfE8zeD diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329253.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329253.0 deleted file mode 100644 index 38c9fbdfc75c204295e5fa659494f16288b13ea5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{ldrVVT9DpxA2KTCfU_nHSh*1<<15x83o)oj9iD3~nQzj|2T1*vMx#cltq8z&JqXZDjgzNne7VM3_DuQiz0_~83h z8i5zG8*~*@SvOwxE_Wj@eK`hfN{a}cMVI!!8L8iDb3no z-0|=8(ZD8V)(5Z6Fh4dbD_XyA?@eG+qND?i4W^U3qP_q&ve}eoZ7`nOAAB~FT=m(l zemU<9cG^9SMhk*4yjYtj`B z{{b?JHKjA7&X?h+8Q`I}ykhu@d!$U(c+PvOf9$)p2}3#MPZ zX|Dy>wD@S&2IIdsMsxxjYh#g9GN!|dgbSPHAZAq)9bj#&=zPTWL%=5OETuoQv7*e@ zC<};bn;+NWu^A3uro#EP2Ao?NJHJ&TW>*$!nsPS<#57MMIHQc&G+THyh^d-RSXLOr z>MK(pfb~^WPHXcB`}@bN_!hMd^n3}{cY6yRU|cZ$Ygfd0aLt-3O0zZ?PkMPf0oc5m z*yOeG_cLqx8KviM&;pzLayr1+V7g$}(s{rpJBHG%4aPg`T6;mvIi0!fp52UqG5btY z)`N3Z{o6YvVsaTsF=tg7i0S@Fa7G!kdkQ+zKuqf(VOe1ef5>u`g7pPPmfe($X<3Gf jS9Fg6>)SYi4lpj5zMFDpHMr)sXiBp-7(em-!grql);cW3 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329254.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329254.0 deleted file mode 100644 index 649785bd9719b30b0f80d0967291192d8ffdea46..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{lYfO_@7=T-t==Y%((1IFTB*=AIW;2Y{cq|42!C=Jw7$O#0C<~*M*&@VL1vJJ~ zq8Js8CMs%GVrB-rU=&*c#Ue6eOk|SifGt7A%Pxv78q`C3Jk5LT*PrBh-#q7iQkp69 ze%>F*JkBkVWVE|d2U6$F(3!NQMn!s+-dMRzxlX|^tkfIL6((JU)>N!8Z!Ry_nzksa zO{JB3lR`g96_k`3^ab9(s8HK6jelap_z<2eE;SeV;E)TP zBbvZs%kwu}__;cdM~K*uN#QbdG%8CaCj$cy!<1Y~GaX=DFg-eEe*~V{>7!X2jN2^Y z6~N~1q6x1}fFB#R4Xuo@Gy)svS9E}}!SvYLFI$1loyC-9Z7?1oyEhm?p8EKXTh6VQ zI31ow(|Mi`tkN2!qM;8pmX;?fuvMAZcTFIrT7#0GE_ek}?)aMEtgr?R*95sh%3Udh z{Y+Pba^)d1aJ=;Y+dC#b{H7G?&~U&%C&BTu#^?a!g6U{$K0MyTH`m|V*!W91|1A1*VB}|DbEJU|FgBQer*A$4jxOI%Y1RhgZk@GvE=gJc-?RkIdGD3i zMw|pUA znnMQ|8%)oV^k)K_dLPZ&V0<|6cpZrOSMaQEVgq8P%ASeA-cv8+H-%%KnR*pvo@1>a9=dA`RSLtx0=6fM=@qP@1*D`0``>3W3e*$lts+ z{{GE29YxvYWm;f!Lrw=68%&#B{c*r1JBre*4aQT_Vq!qd!Vp4j2i~6FFyjiLnVg* diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329255.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329255.0 deleted file mode 100644 index 1e90c7323d97d2905d92e2daa767dcf855ac2503..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{lYfO_@7=VlQ!uO%BtHYv|x=LmY)2{~1(3+iKqERG^fGe3;=>-eaK%2A(!&_x8 zP6X6=AvjSHii>1fGOr1EDQr{9SQE*N#i*c((zs><%M?t!9NOa{@3CKhlIMN%ocBrT z8cF!`=J1-6oOH(OwwEhvGps-9D;YYAB*SGJOuxk|A>z+u-L{~XZp7f;|Xij^d)dZ?^!u7oQ)HG!kH2VZc?vlSr^SyTiQqZSQpdN`~i1HJd3#C~b zjOVvn3xUmN3txwACPc7Fb)!m)_X@DtC(r@L2GixboQ=RHQA26g2IJoeKW$Txrw;f= zR9u$q*7Z=MsXWh1OHDocx1Z z56;(f)Bl@jOph6TnWmP3^98+hfN{Zex$EUq@Jv@1rCA${_hq+i2R3c7=HEqQ?rui* z<-y6ord~k@7#mF2X$z}?4T_^QYlCs+5uFjlR7{)k2eAP$qmGyZV6V1MbyGa%qe;t3 zLCoM&f-}mvPycysHHfJS5SA6j@bZB&9XQ|B>}Mg5h~G?)5iRQ|=m6)7-#`Z#7fe4- zYBGXn2FGdE2IH>t3AMnc>B81qqA@*g)be9}JFuy_L_q2fX74D-0j0>h!RafqTXEs$*nzh0Bv5q(muz5b` zQrITazuAW4Xs5}$1lWwK=m2Ab>CV&ZoWN$!TuQSx81H)U8{ZI|QO0b{=xrB>S^t)>tT2Z2v!(xl^Z91(`b#usLl!!9 gzd{YpS2>#wFfN!@xT|d7nV%~u&DvmmFi&6i4)yUURNDu0p;(guC4$x-orj+#EiTpJmW&)9Z}6g?gh= zEHceTCEdA_aDsQW*GRa=R)T-b`}4j#G&(oiRN=^w5H@e|qiC+Ojj(}y0MF%XjoaMt zkJ%SZ^9bx!HvWp6+py2+5#Z&)ByR;8`}(iv5oBQSVW5nwKTii37fe4qWif+muD(EN z)&}D?S@2C@vtW_kW#jF^CdrJ}j8@bFo6n?lfU&`JQgwAIu<3QvtPRE;k2VbkldF#X z`8tBj^1t z#5o|PLQ2>VbaRm9NP`bp-qPo-wrLNKDMf135vKniEKl{44lpj5t|_|9gKLhuY1Rhg zXL^pu1DmE3DeX=hPbs4;sO#=gIk0K0rUQ%(rlT$;Z3H&)rzp+ZU_9CSwl0*Uta|1f z%f;M#;j;0H12GROS3U*j%1hq-CJ11B?x(Z*-jz1DlzllxA%(uI;!N0AkMe3%W^cK+J&#g&mx0ACBl0j%l6p zt_LyO9uu5V#(fT2?jHj&YsU!73S+oRsT>3A8_N98>EZEZ+Varlpmm49`mSZs0mcQ> zKW(`%0j^o9q%>=T@%+dyy@Ac;=3Tc0W7^DU;ok4VfK6}<9bj$DC~mUzDzIsC(;wKF zk=Ppa1jI~FjP7#U_`sKGugRPR&K;h&yjvjVNH&_#MK1#}cg7H$QO4}h;dAh1#>Wzt z6~=ILTKF@tzN~! zJ4?X1YhEAi6No8NpyYqUz5y}I?F46(G5g*1FMa?ov%*1GRv5!~R2vGw`nGi%<^84Ln&(0(&Dvmm#?!)8?*VA}DZc;! diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329257.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329257.0 deleted file mode 100644 index 87b82da0e1dd9429629ba9cba5011401d74a9ff2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{ldrVVT9Ds{Y?nQ+a=>%=jK+u?4M~%aw9$q6u`D39j7}OPd;qr*k8(N<60YtvibG;x81DJB-f#>RAvM>27T_ISwe*xNsw?|1T@-# zmb@a-q7x^n^z6KRG2QdckmNFil5!`5-2HaKD)|7O%gVE6PQgDL zQ1cQ3`|KDT;WoBZxjh1WyqIJ}BP|y-rXd48y-Gc&KTii37ffe6UfclJjJrT-)&}F@ zjp+w~javK0V>8W*O{@()Pq6dA#&-oBU~DiwFdn@R*u<}-G;4$Lb$hcyz2N)Mjm)bWU2iHs*p)_lQ@h{3W zYG6}wIMLy@@s@INGkV(7e+AeasiOmo4W`Xe%1U69aFo)l4aT>3NkEvmC@6Ui{HQaBfQJ_zyBM6-HF;S9cb~JhhYHj520lw}w@Kn3r}DmKDZulyLJo zSl?(z_m8qM?E)&>OflXVGa7Q*~Wiyi2 z4$cBLPZrPt#s<^5qRhEq3uP@V{fg&G?--Gq7O``*h3#L;i zUbcX1_D#{O4aOT6{dXAHe0R0jDI3#nLxm?bfxxEq8XaJ5Y)EB{nGbAC*D3v;jSZCx zf7gPT18c(WxoxJym+2bc`8hcEt0imh%f#&4h7wCuW8h_uMi87)#%%YXe?5q4TS{0~ z7{k{aLvz9UYN{eS-NN4fF)O#AsFC_%u)eRW=>X$`=}oWpSAlDW@1-+n3aY)E`pd%6NF`jF???IolRhUwW{3@Wn(rN j&@VPA4XiIGm<}*5nAY#<+zqZ75khI!2IF-f_n!VA$Ui8{ diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329258.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc16.1329258.0 deleted file mode 100644 index f850c2ed3f5b4b91a9a612283ad7e9912f239eb6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{lZA?>F7=R0PgL^@Q0)ik#g6j-1WhtQ(loO0YF&afsNhc|7rI0GL$!&4Nk0FaA zA$}1RGhMoYRMD8J@q;XZX>qJp1joiQA#s8IFc>u>h>CL*JhUgLc@Mq)dGov{&v~ER z+kA=dH+>@C&NW1?>zG_Uv^rtFLC~Ab%Dg>Bvn^XysN|R1jApAvFj(|LnbKNaRizhd zlwSxYn^902XQ|S1Q>C%g_e9lww;-9EyJf>?@mznqMv`yF4Nrcs<52tdMI|(b}BYCdOWZmh9 zpC7Y^q!2i0--|a~>V-P5M`Ta{6J0hszOwZF46(U1=A;j_4VMIr!G;N zwZZt-wjMRG$y@I6*#rl$$#tNw8_q%-K}iP~8%$@nZp#5SIq8&UZ7`mZkeCxkt~&bg zt(+T)xOdyzD4gec=^lM$tYl=SR=#Fo6F9X>*?UJMWv&*TJ~dbmQhu08a8_80WFgWT zkaDS-uyb^^sQSsZM__rMzU*+$dIU@H3 zE8YxjzC4=O<+TZva>;pgd#^MO*jznM2N)YnFR9rd43@sTkk;tHbn4K!X4nPRm!+Wtj0>h) zt~XVHYyR)2SsRQiPybK^Y_8m>z9$;f=|Hdk2;C2Ce!oQrSQ`g&W^L>SHkNiu&#`f! zcU{qEK+LkNRJYeg3g67Nhhshl=W0?`+!u*CQiA$iPB)18&qoAjlrj5BO&a{n{-hu* zD~w^`-}O^qeLLzDJzn8J|Ck3qM-ExUeXzcfLv(;~!Suw6jAP)M_HQW7+F;zZ-mnVT zNRw{)Yyy2|o6n%g`JPT-qe-R%j18u1cco7Oo0oo?wZXWeUk78J9V~y~-OW50^Kxc? z1~_-Y+lF3|n365%q@oCcn9Do_XOuA;uF|A~H*?-JVOe1ecL^=!V0{l4e$^)$v$+s` h|F|v%tS>5o4lpj5mR!-bfNP%f)2t1~RWobH-UIV}BV7Oh diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800064.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800064.0 deleted file mode 100644 index 2465a20bb26056efa0858701da3f2159073d90c8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{lYfy|)7=SlLeP5xLCQF-37ZtNiq7jZ7G{v~Z$fenBs}IYzJNAo&MlNAcVnm@P zW1Ask%ouV>aL&vj+@p!kRXuw{2lH~?-6W(K4WdSG6h&`Y=>Vi@;Ocz|UU#cN z$<;;@+>0N`yR8xoy{xR+_n?xnN#ox5bE~EhR?d6!T&hk?>4lH^gg1;Nu+!!b9o%F8 zoi-0oCkG};k*MWD=L$bE(ELgs#%&6s1B?r%Te}X2foFP+rZj7V@!6ff?f@I^Ot;;p zj{}=1GZGhk-2!a>l+yvm2GhH9it>TYj8^3=E1pMKn@y_OrcMjkxR zJ8#hHy=86PTer37;=tCDQ`VajDWlY=(LW>ur0hIRa8_82-c$zM11YbcA?$y;YE@Ay( zqm(1c(C6rf&w$N2jt($3n4X|KnFDMV@RVk4FkW_Xh^vC6-1o&Tm=or7*lnCbLClz4 zzVpCd?Z-d2C1T3dXiV!GBZ%4fjo^$jX62cgF(77IJ7HO24EIIh72tgBQ(xYZj9HCi zzHk-6`6{N<0mcQ>Q<{Wf;F;-BlxA%(zOMBB4q&r%U+P`SnAv5>syzP_*sRT^1B?x( zo315v0ULQ9rCA${FKHMcf|z2qcMY)tF(0IuRdV@AYgnL*4=2?S@9ai8DL z@1BF0YR!p4@XS$GO0zZ? z7sj721U7?$Hr7hUtTv<7Z7Kf1#xP^6(-M#mV&;|y*4b>F z;WslxbL9ZoTXZPWA`!DK5oIKntOhab3kl9BWA^&fU-LlB8OI6B3S+ptuto{aclL8| zy-nEBFJ|^~BtM$l3eH#2P6rqlOfN38z?hcrlxA%(?pfD07}(6uyKJ{{^l!Gb2!E}w)%>s@dbdb`l4aWaI(1(DSzA?J{w%fSCnD;x16=3g=)cp+-F=Y$U z)Db2h5HoQN!5L-Dj`1qL0AfyBOITJI!;AA=ioyBZw`Vs>#w?9T34Z-&fb)6oq63Tz XrXS^m27+gL?4~qpgYjie9xMI<49+J{ diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800065.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800065.0 deleted file mode 100644 index 2ca3d7f3418b8d1c0b17efe05f62ec9994530329..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{lZA_C_6oA{6DDOK&WGli%N^PhNuqG@6VR&hiiFFMSSR&3D3YBV_)Yi2fIF`*0 z-DGs2kb&8vkqOQ6KorrZsB4jO;DYy&BD3O_Thzyf^82K9YadW;|%M*!5q~9Y#9K@s~6U8d_j>nOKN54g;a^p%mz_?&~?+-2)Tlgn8XPam+}^qD5j3U9fNJgiMsU2(t#p8K!SvF;mM_3F+m2G2 zwZVA9%qRzJ{#>K&^4kPU8Sg>K>ik7u)5OsM#s<@&!ynuNHZ44*SsRRN{uvmFAt_yd zhbD4ay4iq@I0?iY(jLwSdryu3+bs}NRE)$aXV!z53ljuqlrd}QIgk!w)=UwW6~^!# zL;4kPzUx_!ZwtokG@z(!^KsyOm$K;q&^N5E~FPdniK=_C^%QdxT@Y zv9Tlp#9S;SIHQdF>>Rm}0b;s}2+Im%`27zzw1e|a_xb!DLBE-uB`D%G!anQAYkSsRQ`y*F9}Y~~Y=^$N!9bR%te?HOQmDv=JbHg2TbBbfmXJK}^|M z!m`2`&VB2;8=SA@uf%(P;b6a*&iyFATc!r*8=s~Fj0>jSt=&oBnM*U2W^FL8nLDos zHn&_i12)0_%{G6AhF&Rm0-NMII>6Xqy7xJ86|m`Dp;;S@pLi0r1;iBRnC|;;69Qw_ z#1@W%y-{VKJP?Q}DnRK)_pX4L(I$d3%9!2e8k2*Vv;VWKFouse+QY&5997PK!I;g3 kX#e+De*x#4tEK~t3#JvjpZdTvFMLdC)&}E}fg6th02fdti2wiq diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800066.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800066.0 deleted file mode 100644 index b47b770ce24d89d515e84a6c1832dfc516da7fdf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{ldr(YK9Dp|^?v2oBJSHJU9<7;Jqs1Yq$z)JQ&0w~+x^c7Za<{_d(KJG36(eB= zVe=R>=pT<+B~%KfF^o5PXS@b&)~lR$&zzm#v3LKx^Zm|z=l9LM8?A8sI(J1M;64}5 zwfqQw7(TRbvZNLD%E)X%pQ#RwQ}Pos1-;QAB^$I-s?wO1k)f5=Dc4A1rXVSWZYnWN z)Cq}>CvBt02Ms6Z#>UJU%(YE+uj)Q`+(6!k_mYGZgWi~^GZ>A&igIsMQh9ZkAHf^0 zln&$4!w5cr@5lFEF6fGp7$Qjv^u3eIt0c?||Sz_?)Ab49{HaLw#hlxA%({ziNq2y8BTMqjbp zxJo&q0v$}B4O2#NbbztJwDGd39@so`(yR@}b8Im)hLDs+Uwa30M`Ai0Hg4lUOp6p( z3C`{L<@Z&Ymx0E%l0XAtxskO2(b1RU=YAyjbStdHb*kHP8 zUdv}-6S9NStPRFP_aCbQF{kGF))5;JGc5H$7C6_saAduF%rgN$JweQ~aRg_Sai30) z3x7e(n+b$vg)zLdTx$jEYi;~!_i*`U)+8a7@FE|qZ$lFuU|cZmf7=!guDPI@(yR@} zue*CS0~=e=+Uv40Ys@Ix+XQ1;g6RNjV@6xs@2P-|FqYE)*_hE=am7>+GyhajgWbj* zzL^DlYydbnVyC)MCT8n06#DqvEfDilDZv?K%uas4e>I3%w}-H-Fou^_Eo9Ds`$!^|Q@6hQ@qsHmV-5j;?yU~GbtxLb{^YfOR4)Kq(Dr{b-6VM0aG zh!L`|O{@w^%w`vlEa51gC}5VjN4iyelOqqeVLh5ndjGY zB(;>=R=BYGeTp+>Z;1pJWk?qRFf=Wtp=T;s%2u%e+~!(X6uw zW`)p4WoUJJA;a^Jg{%8@G&xt5wjh*y8}D1wcW%ruK8XLpEUYpa#SFbk6oX~905nl^ z>t-0iZ#as=xs2HaAHvIdztw`VpFKL>P!>+u%qfq?aG{BW4dMfNE?XzA>W8hWrfE?G z_SxFe&9#o%>-Grr@nVvhh{l(ht0TyO^L|hq7aL6n7#B=GkIw7>*SyeAvo;t%Q&?07 zY!nr}9-9GPY?AG$bdmOVV1v%m0mcT?Vp`~1U~{FPW^FJY`K6;Kgk1G`%ZCW=H)~_9 zyOBT7^S(J*`e51P-kn`rIvc>LjuS0)5-F3_XwQ?jPavhDoZzgm8o3_S&IKt0DhT_X zt{Qb+OHT*O`&nOG-{;{qrA&hoW(fzt^4^%}0ONw`aM8L0T(f5#rCA${YaQi(1Dihv zr8?X;-cm-MMB4+LNxbS1H|n7#orK`%HIjrw>6m#FfN!bYi^AK*IbrDY1RhgLtII65L0W-z9Jbj_av(BvKImyV*wpt zY%qP>FjNU_rW~X+YlHDaZTSumbHJ|PtHcJxd^+k>JUG{&in=BpGxidi0b=&16P!`T zeHMh2$Ag&sa>BC07!FMs8wS?*tg*xG;q}R^&qO`{4qFY@*V;@67#BY`>$n zf%P@MnQ+rB?Cp!0y8<1l`1As-uiHfj7#B>R8ZL0)nz!Fknzg}rVb!C2U{hng=&|wk znza?9u;BIxU~{*C4lp*D4wD&!37X#+9sYY5ILW43OpDi6d|t|crhjA8Bkk5yoOf9}q0mW*jzg0h#(jbMG7 b^XLHMg6VZ}GmF9c^7d1jwZZuHLHeq%2+b_s diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800068.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800068.0 deleted file mode 100644 index 560abc9ad08b0afde6f2f89f99d488427107182b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{lYfO_@7=X)7`94r9$h>g#G8wB-GniW}dH@lFAx2^12$?{k-$-3)w-#B^fsNt? za55V-Xwry4HCi=JOk5PPWw3FXY|~6-a{*_9i(&J!;fN@BXirY}9{crY^Smd|d7qp% zP2&2Do6{^@k$LTv|C0NY=S?fnE7e+gYN?V>AhYa%I=a?y>(@==g-;VFW%!nue)xce$d z>35p2uj%HZi=9K2V0posHv6Q9$CQ!+bb|BW3znxVrUQ%%rc-X_T?W@o(@~nW!FcyxD!I4xF3w_gK3~Oi3=ZG7O%teE@hLvZd>v!Yuh=HN%Dv3=8J5OcJi;EXb6vrfueK}_or z!m`2`{w8w$d9c2+C(%7lVNd^TG;|(exN2eqm0?eVc{T%8M}qBtT2YN9Ywdm`qForu8YQO k$V3;6@ArZAt=dBe7#B?UmfVJKA$BjNSsRRNj~`Ec1x?T=xBvhE diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800069.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc18.1800069.0 deleted file mode 100644 index cf6681b7a70a3c459de3532629ad9d09d68eaf2e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{mdq`7J9Dt`*dT;NMy|cH7)q0A{?BHOUS(J;4u$Y@|xQAMy!v_o}=c?u?2XYUhg06O+8`+PS^4$gfajYBXCcp_0O0=(}~Qawx&8 zuJ0SheVRe=zI<;!WToEZV>_~2^M(;tGx6nUt|OMPQoaw*=?s?TK6uWAy6Gbc9I&ym znX4O}>+$Fl;KxK0gZNoxd&9_p>xFa*Cxp`h#s$-P@tFzWn(aQCwZXVU?`{D$ql!Dc zHi3R@;_N7Fp0Ovek)Ndlj18tO0hzhLrouZh!Oua@BXWtzq1Q{A!1%Q4gNy zgR-^8P|2%~EvI?QT5xLK$=YioDdW^AX_f6VNSRqga8_82IQQ?LAmyZD!v3eLM(yM3 z`-0`kjSgq0hu@TvH1r|R+6D(@?j4 zhpvG2Ma0no#s$;K+y)+8^S+N}Z7|-nDKHb*thDKFipI=2g{;e>zXF@XJLmvogXt8m zX%VoQT0m*m2IGpO$1*_7$2p<5hz*G8vLkAcaOczLF;WdkVG;4$LaY@d*2S zW9FowvdHfT!1}s2(*ec>)6#Bk_{Si)h0?4I##{4MW?if3}LzNBq*fN{a}l@Edq RTr=59Y1RhgH774u{smP{EzK7-J)hz1prFWo_+t%(Ac{Ce|!5GVe`U zn^`<2h{sqJWAo^O2`0pZka;X@Ua@4Hwa1a)(c3?7zTe4reqZiwq{RDc-xXQRMb|}M z{xPFrhCEO!XbdLR?Dcw6MtE4PiXWY!H(AVr)~pe9D$6>fQ6prjG6h41UQp>fsN_^b znm*b4#9DFW-9U0~Ow_C(Zs;2Ss*ZC*`|wgeK+va{O_t;|v&ABl*m|JPDr5Tqg4bRv zQgF@b1n-Qf2Bw!O$w0$%X&5&rfetV(n0_R+rGje)CsLZV!T9d*(cBPQ4pW-7!MJR(Z7{GYm^kN} z$HrI6U8M!*hHU8sTPfy>p%;1vrB+Y$$2`! z*kF2Kp;M+w%K*!;oc;q%ROsnL$NlbXQ#;#%kcuah41LF}3a? zwH}*J@Xb`$I~Rg;_nw_{TO?*<0*bgA9R^|!uOc|3jM+)*)syzY&=0})&}F3bn@3AX7O#qUC(Yh! abbxWe^z}UtjNqDY?38A0Fy5AkO8)?sOem)S diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244601.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244601.0 deleted file mode 100644 index 0d2688ffbc10cdd011b3fea7488f3ca512edc69b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{leN4<@9KbKd{C-|~QLAoTVOAIU;qB_5-#s) z_@y7A&qJl75_It@jeLHZph*o0j+FBOse(qY)g@@-bxCslhP7+sb(`erI#sHmlM4e> zT(U|n#99AI%gxR6BIiad4)fsVsE5=Ioa^VxyYM4)LZViqk5g;)dN)bwXe5i8su)Y~ zhO5SLT#cUK&b%W(e3hUXWcOtRyctJWW#H?HoGO*DF1!=ZC8_j@gK*VBG;}Ck}iC}3mPN@>;x}7<=LKxGOkZ%- zVq+_%cR4z0bguw5b7s&1#s<^1{$~~d8~>SG~uItnX)X^L5df zrUcX(cKIe)pM44)U|cZ$(R0jVa7}wPrCA${Pp;381vZAWi8n-JW|gA~^O^0yru`fp zU~DigTN0WEY;w<2nzh0B%6vmFh-r3lze#LB%w}0Z4>)&Ny7w*dm}BGG_JNq1%>-wZ zai3!8qh=7(V+&zfVGOtbzP}EvFX!t=i-*l8)1*ZG$`8R{eSzI{fN{a}tjvX(~frKC%K}BV9@dSer7$UEfg&Y@Q9$|JjtGs7>2;f|!jF zJ`ENdd-!B_M|bQ3=W0*SY!r#v5{uRserg3Vi!KnHQO4{A*{@CzbNfZYvcedS`O?(} z*4Nx-Ecg94R}lfe3-o#_DMg6R~)!!O{P9j=sSZ7{wrtEv&$9IZNMwXyY@ zEiFVH)k^rqJaC*2FgBQ8zN@Vq*q{@XW^FKj^3T>F5K~B8eaEsJ2N<(xLY@ws``3_t zS0tt+0-1fJ(?QJl*#u{lG0Qy+ZUQkM=Ma_^#_*!9q4i*WqYh=>6OCCKi4OiL2>>yR cE9d~@g6SjQ&prUxlpdorYlHD=FC#qv0iic8RsaA1 diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244602.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244602.0 deleted file mode 100644 index 794c94be8eadad705aafb5f636acd2e50a097b65..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{lYfO_@7=T-E;rmpu6$3b~8Cd|O28`#Z2_@~7DcD= zlE@&7GPH!k!ZM~3LqrD_1*dYE8Sa89TP30>3sZhf;{`pm$3xy@zy2i8`{p_ClhS07 z`!iLQT*vLcoqYM(+M%^e7b*=3txmSCRIM{5#AeC(U{TKqN-XlLdf3Clq7#Ht7ulo^xi06Udqoms8xEMF~3M}G%gWY zz0s==j(G(V{6<$z5La13a0&0jdui0V88*q^4%Q?5u0mcQ>8pn5kfoE3cQJS^E zc+O67Hn15Ro^spF@?eu_Lk*_xQ^4l2lMXO8m^R-V$N@Iz?@^ky!T1A-_HRG()Q1Dl zgE`Iktxi{?1w7BsE>#pQ5jo%NowKsF0c`!~_x>vaDHG+WCX%CMnMM@NZ+&Lc{@9=3lz_?&q77%_CJTv~=uTj>E{@Wy4aP4f-BtyVlt-R= zMR1D)pSf+sQ6Of9*e(Hk3j3#chZ2?*#_)*L z-~i|Qx$y2a!I*X>s{b-A7M!n3O9vPiOfM{*Is~4XzKznX4aTF&`zC?S_6w?R!I+jt zbjUg?0X8{pbbztJwC2>99@vb`(5wx{Bj#4yLCpGQ-yUKEV*2}ByA1X|*%f+SIHsn8 z-v?r@v=E$8#(h?gm(+ro85M+Og)#hVu1E&Xmpd`;^6>c0v=^YrgM;1ReAcIQfN{Ze z@Ufi;JTq~U(yR@}KmTt*53p(awCoqbn06Zq>s{{)Y+_RB0Bd7Ib!YwN0h<+Rlzz{~ zhHA}+-+-8R`onLyY-Yo6X5_d2;b8Bk*7%zOF`aqn(Z=8~5Od@r!5L-DPAKnBfS66~ zgk^;>Z0Vf)2RPq=PsA;ku%};4OD_6tbK4j=--g9>fN{ZeMT2t^JX7LJY1Rhg&EuIi zU~|5?&28i9->mf{k{paL2R0x7NCy}jOlvKb6~N}v49(hLy!t?j1H@FQzUp<|W)6%w z>Ofb)Uh}>qeF8B>nW#6ZT?%4)A0;@WjM&c07COMVV0y>!CNp^EvY#l;+F*R}v%=Q5-ts8B diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244603.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244603.0 deleted file mode 100644 index a25f3e3ea2958e7da7cda8dc76d3fd0612a1587b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{lYfO_@7=R0Grr#$+xs6LiXl4kKr8qL;#N>n~s3qP)O+?Aomdl6WN;_MSprSRR z7>R;bBc$pG6pdzaOlKqOfWoFC1IN^DP&8mbC-a7e2*yQrXirY_9{cf=JnzZ#zE9dJ zC7!=`$CRhJiocbPDaD{7`lDQ9PLW=|V!NO(NljMC`AYmN40XB*eI>6XqS}_$c0d2fAYlHE~J6&}VWYvFeO)up7 z+PmA`jRJX|57?fg3zv+{?D<`05x}ddfA)2Vq)bzz(WG;`KuWEP;HU(51%O|xrkd5`~Vzp_Zd3CxM2G2c%u`nDgBnxtPRHXkylfI zO@7L%PPdJ(lyS{ylchcg*aUq+2N)Yn*UY%q1Dgw8nzh0Bl4|R!D3Y@Jr8I%-oSgR9 z_$7jvwd)^9!Mj4#+b)rq5;eNJJ%Pgb=j0>huK5|&Vn)a_L&DvnRa#+6)*u1dk{VE#M+>8<~y^aMo z%YLE*j18tM7H3ofo23p)vo;vN7+PxqF@FsB_!_YRF?(fW_2Au@^0@2bF;{e^^@5m9 zW`Z-yxKF(O-?JcQ`(DDb!WcgMnH9#=J%8f%@cGGfYSGQFYGc6p9=@alj0>h0oxhv` z)+~EPY1Rhg*Z)g71!A^ll-&@G>9nD$F?$8DsaZn@SQ{I%^j94QHW_Ov{ho~t75sJZ zDu{XQ=Y(#zO#u94O08oq@NU_qWj94)j(mwqKX2O)VrE?-IHQc&te(2%Am)sNu&gkK zqg!uufb$K`OXzV6`})N+XQRDGqg%lF)&WC0AtFs{nNpg zE32w|MPf=aQOJ~e9K;+rLU2YIvl|{q90W1%*ASK!#_)m3fqHO0^Qk?3qA?p(sO5@k fCph1xMmoT_V7lqDax+-7qlwb24aOgv4ypeKF)B6L diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244604.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244604.0 deleted file mode 100644 index f432a9768b3b05e4b4e07605e3d2371a1d21b80b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{lZA?>F7=UlZAKvRa5J5g>E1fYxrrm-g2-6d*7+KUM5aZ^gr5{{2zS?*%0!2}X z(GAh4D8{cr4K*2VZW$_y3!@Bm%nDmF=GI0g#F&L>3~q=TwR&hzj(Ly0{nsMBY_VUcsT)nWMT(rv$$?C931$udNS(!z0 z>MNz9a#7NYlT?0TQK^_8`X*`3K+haX7AF z&h;kJ9Nm{To8WytO&Z=%PVgB*q%iF>(f*2Ukt`YwVa@N}n#;dmN!Vy1O5h8MWZNrv z(S(1mj=-ud_a^ucHtYy`M5)4 zFs}PA^?P9B8w!MMri8Is=0>)=b)N&98^d&fvBC7md)|rzHoHbB&DvnRLSx+#N3Qzs zz+){x)ZO<}u+bZWAVgGHN@Ka<7u&lwyVrwLBmV5~R!F(bgvR_f4@lYVBRDH;La}-w z7o>Fh3HzF^3B|a_Ik3D#$Gbd}9$`~*R+O-UyAPK4MKc{>Trho6)usm5)U{BWwZXVL zvLh4N>`7bI6SN7JGU*gr>bvYzgID)$BIVP?fby`t{2k*#s$-X3~e*GrlW+?tPRGmY2#~w&6;zz%Zf2;Pob;T zn>GTQgB^5$vB7j;WP3b6q(S9p)ZM>e@K>8f(fZAVV4 zLCpDE3C<|vJ~bs`FNmqCAuKD5;lG#vvlFau_t@Q_N7y&hV@9@5njV4mEgYui} zd1q`4tS{jqrCA${R~>9x0&EIaSN@_H)8j_Yw{^b*o6)s&fVFWW@8Pt0z~;tBlzz>| zjRv`_91wGxGr2El69L~$-|fMT;M}OQ8NVvT9L__d!?#T!rmK_Sj521|ogZ%nF+aRS zSXLOr?JbQ4u)gM~T#@ouD_GyL6WjY0 lW4dyY=Dq62V11n~I>5MKI!|mjf@|(QNom#w<8tTWftRg`D_8&k diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244605.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc19.1244605.0 deleted file mode 100644 index dc544a626ce9d449e1404be2512a62ebb839a2c7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{lSxi$|7=SNTC)}$DR>5V2DM7-bRwV%u%n3C!5rG&5F^X7dOTi*-wPle}2S%fb z!#p5rf{I#Ai5Pd|LR1hDm94}j5@ead2f-P|2r8PW=%GEH=0EoKvHAX!@BClxy|J9* zH&PaRgztYCd*g%RsbYqQ+N9DM4Y^TaiE?31uGU~Sn$$*>DMN0~)9Y2HLV1Bn zm#a0&wWCy8rY=XD=J=Pfn+uclDAf7Wf9L$jz|(GN7_UmD1B?r%^;umqa81d2O0zZ? z_l)<90ydpJBMuvh3!4}#3g6P$1Z*mL=>TJc>4apDK4265jMA(P#v||W;Zw;~``bSH z@oBY+ugdY}dcPJ;dwksjPEB}XyCsq`Mv2<=8#s`1RTsfoVI{g!vGoB+ zIj@_rKjW z*8>}M_|n^U8&@f1m1x}2zIb5MvXBliHkfWn30Veg43U&(Z7{xjaB$OflCosbJ(%}O z{^+oA3k5M>bp5>poEtaeQ@uz`PKja`H2Z;=du0S?lrcO1^~Sj%W|==>Sz!zpKian( ztgk<_^NwiDIyKrK(9;aow>X;)FfN$BIaF=~*Bo}ztPRFHT5WN_Ch@YSK{RG@CGzc% zt_C)`t8{>|!E{YuX%?_~mo6Mz%}9xt9xM_rzlcaXFPB z=FIH`XOwZD6GwN=0WpLBAS^44;qoi*3c>m;uV30dTt1m~=_q&0{Q$7O=s`NbxL`V@ z?JxKOYI4%74aPN9nlHena#_KB(U^5sRItV^0@zf?(*f4TiWEa)YsUYU8YJxM$m<^J@4+JsaerH)> z3{QTWa1*R=hgWd3UD(wZvp59}WdE}Qtk3982N)Mj&t5tVVacP3nzbB64FMzYJ<)N74lp*D4pElFn7htWnzh0BhkB3P3(~(@f>n)^v3nG*aHd7t?_duGdNFfie8nh)ff`fwFZM1Q{j$2mN!X! z34Xu1#E<=?Ah?*D#<}iPY3yuqdxFM~u!>;IJhnqgSWnJ_V^h?IWIKH2d}mM~fkoLL zez36{4q81tL=H?8Vd#`1t=gXqJni!gWt%th1C$G<*DqSr4X)|Eg{Ns7jK6OU$N@I- zJ)<_8Ne*n*8jfM)s&4rMN zW~+^(lmV6K;-2AtdIGjc~DCL>2*XV>L|m;wF-r<5@(>5*ImF`ecRmKMgai&NJ=u)e-sPg(?H znv&3R)&9d^eN)o-0m=o_E1Y$3eWiAqw!yfYM^7KHc~!4$6^xl*iK1lf2Z2pd13y67 zVA^$2h5;=8;rj>bfOK!WUqR*5gQOw;SqccoclP>|E_S%pFz>xAf~pE;FL1% zGyR?SOb~PHA;Qwa81C1(=Y#cq|Mkx5;qYae6lkctHxR6E88i5QfO5gK?3m6STvI0E zY1#(kHCA@A+h`bZDT|)3dX(zn*lpL!N!PIhMiaiV%9#J zd*5o~1YhQ`cyIumThOxNfk4dXJ5Y-{_c4eW*G_Os8MC8Dzib0Bea(cWg)uzT8kr5& z*B7{;-74(pA2UB5{n=i!1+339pC6!HFui%(vq5mpoFJa2Z7^Pc{PQ_rQ+BJ)X5;8* zw&FC>HiiENHf|<lUeD+*Az_?(#OBJykJX1H1(yR@}Uqv@1 z0h^@z-`zHUc(F;fBWqPk39#AvkPa|5nAS`7c3>0QMQPRs<4(UD2}<(Ro_l@a{KRu@ zS6z*U2!i0dUT+HFoIke}X*Qh%Tg5xKugRoL)uOEEq`U{DYX$`Y5%N8r@=EXcxcuJL7qjklDMb;wT{mIrLA6X*bAgXvHIC+q+=wTYBwZ7|MvTL*=bl;ynv9C{M0gZnGo1i6hfVHushF3}>uxa(szuDMPS5aD3%{8sx^`59y+UhJr%X)ea^z?nSr20V+eUCk8MBcGGNyu{!xFddaKvlKjY%2-OXHWdnVRzW@YrAff;O diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426000.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426000.0 deleted file mode 100644 index 5b4799c1283f517d623bc1d32b0deb67de194665..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{leN2pT9KfHH)$>s5R^FvzQtY%e=EjduQLAKmnd9oYx;W~xyVATBOGcPtrT3yj zGnv=9l8#7R8k2c{*P6FkYZF3#-TiuQ-(UCKpU*zu-#)+ZXU}s>81wJnktL<MXT{#IiOsk4Z*63mqlXW^bross=GGtyJPw^PFMsN=6RT>NH+ZBCr0%2okyqUymc*06JXO4|m>*6eMle&APCxLCYfBnHm zhwe6cINMk;iIt&+A0B7BlYwV1CH|~3To|BSFfEtML%}tBEHrI{@qnpAZGp|AhkwjA zL#)^=s6vxFRer#x_K`3^*8ru18(WQyA}JRrkfJOl1f(4Okl?hi0_ip-*n^Z$2iQTn3UoNM zT?>|%kymf%_pq9hQKCQ19`C{Om_lKIa>2CSnm!&}v%*5tHW+XGz3v;ZDfJ7wVY0E7 z(xVz(49cklHnsl30A+*efMsFbz$SR6K+`rDAJ^cRHHM@-(B&|dU7Ff$wz2UAF+02p zV!*lc#{Rk~5|dG&AXgtbh*{`Ca7r1oXJjvWK+Hf-!qUPR_Ia{$C|KX64NsdyV;YpG zMV1C*+HDdBC>Ko6(pT>Q*W7ENX&a0iOFc&c8~dyLEzy{}s*y{?$1D)jsZJQ6Y%uK@ z`TQKP$+gh54aO5D8vQ`bn`b4ri4BNZlz;jWIJYvxy;(e_LtbqZh}pY`;FL1%qlo9T zLCp3n!qUPR);|co0@l~u`^n^C_02TIqImChabSJ6%;$jt$_3M}l6G$f*YvOvXxawj zx62Iqz-G>h)H|Xv4OQsNu@d-^EnX!I&^A@5I4mFx*i2h3(1UEMP;X?-eh_osbFUVY zjV*jLT@!sA!MSYH?7Jc{->yX^b2nT8G3OWwPAOwHKR2QS#FVuXmKMhFUge2nvE8kw!!$1b8Q-6Q{HgNY-8;+TTzH2 z%iq5SHr@tdfU?2#>w=E+z~;Jzrfo2O#6JBXi1{up!D!lz9gO+=L9rg3`}f%XR*{%Y z6gu9m9tmPbloFg$#_SpC(N7>|=KxC!V|ZIlXE|73&*h!>MPpVhM=t3(^TGOfy)Zzz XV0v-H);4g>Lv;d8+hF{ALB`4dE95Kt diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426001.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426001.0 deleted file mode 100644 index be113a4bb322db9c2d07178e6e0437f78492e059..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{ldu)qq7{EId+4l|G=^@N0y0~mP+OF;ruTD1RFk);L=3CpXvqg2ZyK!>pBFN&L z*v7_Wn}*g&T+VSjW|mT8wHXUS7z+zwSqyO}uJ77+Z}YtR_0Q$^Jo&xPFJ0p$-p{wP z_-gLy@j3M$V%lPc4bYo)*(OC?zQL3`L!GGLf6FzPEIDRN{xVAs>1?EslGvQR#-s6M}EBlQoikF zf6^6D&E)9~;COq>>s>t_ep5>HsM8tH298%(K?fKYOjmqONCwZW+(T*B2IJwHJ5FFz zt%|$gvGJF3lmnfND>(pc{#Mfg#s<>^gaxO8P4q-cvo;vFMYb1*k(AqC1&!z8^4@rD z0-`|7m0ADp2YYQ{pBhABN(6Mp*nAMgRE;D!ql{TJFR2#9)Q%!7D~w_7i(9+F`C78> zUKEY#(xWF;w-gse`WX*5Yx4V z;EXcvGizLUG>F-~m9VTZh8+>hwu1AOe(dsi`2A+Ow5TZd^%HQu^e=RPal!P$SwaeU z=KtR)&DvnRaNu(Xuu&)FT^5b$vZMHRiwf98Eu#ahjUDYOTH6Y2+&=mz8#`JQBvXQz z!|%(RJT`&wn;HLNZUxvosbOlfNX(8@R95Qf1~EgM3C<{EHqhD>24cEP3&lozuxL|rv{^z^knMq?Q&Dvo6k^S-|U{m8f<+btm zZ`M|cq-PgsfX%UUbbztJbV${NIlxABp3{f$F%xSD%L-$7W@h0QaK6;je_j)fX-h;)pQeU` c^F`Ou0mcQ>hjzT{2cFsJqgfk_m&nn~AC>DPo&W#< diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426002.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426002.0 deleted file mode 100644 index 29a3fdac1b00dde52310aa6a95a0fdd65cea0a7a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{lTTD|)7=RCgF6Vdw>$NVnictiitWYr;GEq@NAh_aPR|_qiDni?&^rV}S0tG%J(~>?6_bhM) z&LptUF4srikhsU~5$NN^ByBOuFIm>9AOl_h%c6KLnGP^6n2u~L*#xdxI!v=R7!ThP z83b%1+6O!~BfQuwsX|?2W$%H_gbq5u*kD@y)X@!W%7$sy2IJPz{cTgoRbRICD){x6 zn{K)rjS&RFH&2^AiR&J;W`)S_fm8eMwEiQJa)}20>dfy0DbL*}I4i6{jem=sAf>K@ zus`T(kZt~?E{sji?jYlHE!191jm zbM%)LPPdJ>lrwDTk9T?-u*r&|1B?x(e{QlB1Di3?lxA%(-W9m9RZdctyz`&ShvmNa z*!YBjn2wyd-Qe70`R4|Sn4AV#<#nS#%pxVh8D-4I9h~e3VlJCOSXLOr+Pz89V0{m^ zKWvnY>CmCw_NxhCeOt5X0ONw``D@Pq1g`00pfqcP@vZmUOMp#Jt@dBZn1wbJqqi%8 zjjE0gFgBQ0tKZmx%}hI`SsRSkc1}DDVir`({v$RZX8qmzGvM4jtD;Fd=EMB~5g?|z zh~SJe?$hz%_9+mv+}q>!@cL#t(vWLPUnyAMvwk|jxL|tuaq%O# z=HUTKvo;u)r{(tno7wAfZ%M{f}7nozVOXV$QsoE&W&wY*dh_LdkcE;GUf}2xv!bvj51~~o&B>E#Qdv;u&gkK z+f#};u)edwb8fqZz5QbrZbmUd)fHfU0mcT?*8Qy)fXy~1rCA${&r_y62QeoU=v&>p@q;mc+w^52 zIQMdS@g0enTnY+}wX6j(4^$ diff --git a/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426003.0 b/monai/brats_mri_2d_0/~/output_brats_mri_2d_gen/tb/events.out.tfevents.1695014637.sc20.1426003.0 deleted file mode 100644 index 48117b3e5384ccfac341bfab392ccf722a1e82ab..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2669 zcmZ{leN4<@9Ke6)Fuz}3a$ZAkm&GiH7@P4ZT}GyMHfu=N9lz?RyZK!&GpsUfd5IZ` zWY>1FEVET!Qt47eiG<0^y!?ULnh>W}kypxv9LCfz>3|S zPJUlhj?2Sa;lU(fDUz%g!u`m=i`S}9UKdRV7#B=GQTxY%Ynoyx&DvnR>3gvm*sN&l zciRl}U=wCXisYoXz^1jG4lp*D_Bno(2R4=`lxA%(-rA(BnnJGn>0yr_UtfOrmaEZd zK@gM~hLlO%yMZiiexnYYTK=G=P9|koJUSQ^a0{etXd^f)9FLx#6YU^n+hfB1ryGw# zBLBPs%TpFqI|e;GrsRz1)K^nKSf0L+4lpj5{`oQYHMnN-IZCrO7?1s3VFWh$^CRnB zHl9)jl%V6^5=($h?gBc%*kF2q3y20bGZ#{twZV8)NFoDdbis~F$cu_ zpf1pZbC>z_-j<2U#iQ;ZFZeFs_!68^#_W}6CsIMo3_rrM!Wic3G<{%wkCHnYWMevv zXhKw8D_Ea%I~`zLFx|l|$_Cfmw}aBG4aOD1Mt@*4qRP-H8#B8EHO9PD0-N2{bbztJ zbg{Gw{z@d*P@1*Dc<@%c?uX2E?5Fra%R58JOjFS3c(L-nc68GIje2&M4zP z+P5L11Cu&gkKyPeZ)V10*rKD#_TzD!2~(iLb`V11>%bbxWe^kP$57`W#4K1#DT z7%yI4mkMmcHl#Pn#&p=x^u_D;02^H_9bj$j=;{xj>%gXCi2l#Uj%IWQ4}h4LUa9Z7 zY?Sb2=7fI90q1%(EV(Ze^IaTLe2=&ZV%nPt&M0H{jHR>&#EfYnEGvxRWi$Pkfc4d= zgPUE#p8hejH=#TGyE4K0w83tPRG;dIgHWrl_{kZR6=@)^-Lp z&UFR?o5VUgz}R5=*yB97bcKV`tPRF<4)ZHPOlM|di)%N-Va&qPfDPc5XHVxokcr9Z zP;9NUuLCjL|FNtvh8KEwsKENJUfbI$8`HKPl~q*S0PE|lq63Tz XrY~OZ9}TWqbd%Do4aT#6J9hpBMR_bj diff --git a/monai/brats_mri_2d_0/brats_mri_2d_diff.isc b/monai/brats_mri_2d_V1/brats_mri_2d_diff.isc similarity index 74% rename from monai/brats_mri_2d_0/brats_mri_2d_diff.isc rename to monai/brats_mri_2d_V1/brats_mri_2d_diff.isc index 9409c35c..39578a95 100644 --- a/monai/brats_mri_2d_0/brats_mri_2d_diff.isc +++ b/monai/brats_mri_2d_V1/brats_mri_2d_diff.isc @@ -3,4 +3,4 @@ gpu_type="24GB VRAM GPU" nnodes = 9 venv_path = "~/.venv/bin/activate" output_path = "~/output_brats_mri_2d_diff" -command="train_cycling_diff.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc, --gen-load-path ~/output_brats_mri_2d_gen/exp_1645/checkpoint.isc" \ No newline at end of file +command="train_cycling_diff.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc, --gen-load-path ~/output_brats_mri_2d_gen/exp_1645/checkpoint.isc --tboard-path ~/output_brats_mri_2d_diff/tb" \ No newline at end of file diff --git a/monai/brats_mri_2d_V1/brats_mri_2d_gen.isc b/monai/brats_mri_2d_V1/brats_mri_2d_gen.isc new file mode 100644 index 00000000..ca081d87 --- /dev/null +++ b/monai/brats_mri_2d_V1/brats_mri_2d_gen.isc @@ -0,0 +1,6 @@ +experiment_name="brats_mri_2d_gen" +gpu_type="24GB VRAM GPU" +nnodes = 9 +venv_path = "~/.venv/bin/activate" +output_path = "~/output_brats_mri_2d_gen" +command="train_cycling_gen.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_brats_mri_2d_gen/tb" \ No newline at end of file diff --git a/monai/brats_mri_2d_0/loops_0.py b/monai/brats_mri_2d_V1/loops.py similarity index 80% rename from monai/brats_mri_2d_0/loops_0.py rename to monai/brats_mri_2d_V1/loops.py index c9c8d804..3d0fc19d 100644 --- a/monai/brats_mri_2d_0/loops_0.py +++ b/monai/brats_mri_2d_V1/loops.py @@ -1,10 +1,9 @@ -from tqdm import tqdm -import torch -from torch.cuda.amp import GradScaler, autocast +import torch, os +from torch.cuda.amp import autocast import torch.nn.functional as F -import torch.distributed as dist import utils from cycling_utils import atomic_torch_save + from torch.utils.tensorboard import SummaryWriter tb_path = "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/output_brats_mri_2d_gen/tb" @@ -18,7 +17,7 @@ def train_generator_one_epoch( # Maybe pull these out into args later kl_weight = 1e-6 - generator_warm_up_n_epochs = 3 + generator_warm_up_n_epochs = 10 perceptual_weight = 0.001 adv_weight = 0.01 @@ -96,14 +95,16 @@ def train_generator_one_epoch( metrics["train"].update({"train_images_seen":len(images), "epoch_loss":recons_loss.item()}) if epoch > generator_warm_up_n_epochs: metrics["train"].update({"gen_epoch_loss":generator_loss.item(), "disc_epoch_loss":discriminator_loss.item()}) - metrics["train"].reduce_and_reset_local() + metrics["train"].reduce() timer.report(f'train batch {train_step} metrics update') - recons_loss = metrics["train"].agg[metrics["train"].map["epoch_loss"]] / metrics["train"].agg[metrics["train"].map["train_images_seen"]] - gen_loss = metrics["train"].agg[metrics["train"].map["gen_epoch_loss"]] / metrics["train"].agg[metrics["train"].map["train_images_seen"]] - disc_loss = metrics["train"].agg[metrics["train"].map["disc_epoch_loss"]] / metrics["train"].agg[metrics["train"].map["train_images_seen"]] - print("Epoch [{}] Step [{}/{}] :: recons_loss: {:,.3f}, gen_loss: {:,.3f}, disc_loss: {:,.3f}".format(epoch, train_step+1, total_steps, recons_loss, gen_loss, disc_loss)) + recons_loss = metrics["train"].local[metrics["train"].map["epoch_loss"]] / metrics["train"].local[metrics["train"].map["train_images_seen"]] + gen_loss = metrics["train"].local[metrics["train"].map["gen_epoch_loss"]] / metrics["train"].local[metrics["train"].map["train_images_seen"]] + disc_loss = metrics["train"].local[metrics["train"].map["disc_epoch_loss"]] / metrics["train"].local[metrics["train"].map["train_images_seen"]] + print("Epoch [{}] Step [{}/{}] :: recons_loss: {:,.3f}, gen_loss: {:,.3f}, disc_loss: {:,.3f}".format(epoch, train_step, total_steps, recons_loss, gen_loss, disc_loss)) + + metrics["train"].reset_local() ## Checkpointing print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") @@ -114,10 +115,10 @@ def train_generator_one_epoch( metrics["train"].end_epoch() if utils.is_main_process() and train_step % 1 == 0: # Checkpointing every batch - writer = SummaryWriter(log_dir=tb_path) - writer.add_scalar("recons_loss", recons_loss, step) - writer.add_scalar("gen_loss", recons_loss, step) - writer.add_scalar("disc_loss", recons_loss, step) + writer = SummaryWriter(log_dir=args.tboard_path) + writer.add_scalar("Train/recons_loss", recons_loss, train_step + epoch * total_steps) + writer.add_scalar("Train/gen_loss", gen_loss, train_step + epoch * total_steps) + writer.add_scalar("Train/disc_loss", disc_loss, train_step + epoch * total_steps) writer.flush() writer.close() checkpoint = { @@ -167,7 +168,8 @@ def evaluate_generator( timer.report(f'eval batch {val_step} recons_loss') metrics["val"].update({"val_images_seen": len(images), "val_loss": recons_loss.item()}) - metrics["val"].reduce_and_reset_local() + metrics["val"].reduce() + metrics["val"].reset_local() timer.report(f'eval batch {val_step} metrics update') @@ -180,7 +182,6 @@ def evaluate_generator( metrics["val"].end_epoch() if utils.is_main_process() and val_step % 1 == 0: # Checkpointing every batch - print(f"Saving checkpoint at epoch {epoch} train batch {val_step}") checkpoint = { # Universals "args": args, @@ -201,8 +202,8 @@ def evaluate_generator( val_loss = metrics["val"].agg[metrics["val"].map["val_loss"]] / metrics["val"].agg[metrics["val"].map["val_images_seen"]] if utils.is_main_process(): - writer = SummaryWriter(log_dir=tb_path) - writer.add_scalar("val", val_loss, epoch) + writer = SummaryWriter(log_dir=args.tboard_path) + writer.add_scalar("Val/loss", val_loss, epoch) writer.flush() writer.close() print(f"Epoch {epoch} val loss: {val_loss:.4f}") @@ -210,12 +211,11 @@ def evaluate_generator( return timer, metrics - ## -- DIFFUSION MODEL - ## def train_diffusion_one_epoch( args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, - train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer + train_sampler, val_sampler, lr_scheduler, device, timer, metrics ): unet.train() @@ -250,50 +250,64 @@ def train_diffusion_one_epoch( scaler_u.scale(loss).backward() scaler_u.step(optimizer_u) scaler_u.update() + lr_scheduler.step() timer.report(f'train batch {train_step} unet backward') - epoch_loss += loss.item() - train_images_seen += len(images) - recons_loss = epoch_loss / train_images_seen - print("Epoch [{}] Step [{}/{}] :: recons_loss: {:,.3f}".format(epoch, train_step+1, total_steps, recons_loss)) + # Reduce metrics accross nodes + metrics["train"].update({"train_images_seen":len(images), "epoch_loss":loss.item()}) + metrics["train"].reduce() + + recons_loss = metrics["train"].local[metrics["train"].map["epoch_loss"]] / metrics["train"].local[metrics["train"].map["train_images_seen"]] + print("Epoch [{}] Step [{}/{}] :: recons_loss: {:,.3f}".format(epoch, train_step, total_steps, recons_loss)) + + metrics["train"].reset_local() + + timer.report(f'train batch {train_step} metrics update') ## Checkpointing print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") train_sampler.advance(len(images)) train_step = train_sampler.progress // train_loader.batch_size + + if train_step == total_steps: + metrics["train"].end_epoch() + if utils.is_main_process() and train_step % 1 == 0: # Checkpointing every batch + + writer = SummaryWriter(log_dir=args.tboard_path) + writer.add_scalar("Train/recons_loss", recons_loss, train_step + epoch * total_steps) + writer.flush() + writer.close() + checkpoint = { # Universals "args": args, "epoch": epoch, - # State variables "unet": unet.module.state_dict(), "optimizer_u": optimizer_u.state_dict(), "scaler_u": scaler_u.state_dict(), "train_sampler": train_sampler.state_dict(), "val_sampler": val_sampler.state_dict(), - - # Evaluation metrics - "train_images_seen": train_images_seen, - "val_images_seen": val_images_seen, - "epoch_loss": epoch_loss, - "val_loss": val_loss, + "lr_scheduler": lr_scheduler.state_dict(), + # Metrics + "metrics": metrics, } timer = atomic_torch_save(checkpoint, args.resume, timer) - return unet, timer + return unet, timer, metrics def evaluate_diffusion( args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, - train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer + train_sampler, val_sampler, lr_scheduler, device, timer, metrics ): unet.eval() val_step = val_sampler.progress // val_loader.batch_size - print(f'\nEvaluating / resuming epoch {epoch} from training step {val_step}\n') + total_steps = int(len(val_sampler) / val_loader.batch_size) + print(f'\nEvaluating / resuming epoch {epoch} from eval step {val_step}\n') with torch.no_grad(): for step, batch in enumerate(val_loader): @@ -316,36 +330,43 @@ def evaluate_diffusion( loss = F.mse_loss(noise_pred.float(), noise.float()) timer.report(f'eval batch {val_step} loss') - val_loss += loss.item() - val_images_seen += len(images) + metrics["val"].update({"val_images_seen": len(images), "val_loss": loss.item()}) + metrics["val"].reduce() + metrics["val"].reset_local() + timer.report(f'eval batch {val_step} metrics update') ## Checkpointing print(f"Saving checkpoint at epoch {epoch} val batch {val_step}") val_sampler.advance(len(images)) + + if val_step == total_steps: + metrics["val"].end_epoch() + if utils.is_main_process() and val_step % 1 == 0: # Checkpointing every batch print(f"Saving checkpoint at epoch {epoch} train batch {val_step}") checkpoint = { # Universals "args": args, "epoch": epoch, - # State variables "unet": unet.module.state_dict(), "optimizer_u": optimizer_u.state_dict(), "scaler_u": scaler_u.state_dict(), "train_sampler": train_sampler.state_dict(), "val_sampler": val_sampler.state_dict(), - - # Evaluation metrics - "train_images_seen": train_images_seen, - "val_images_seen": val_images_seen, - "epoch_loss": epoch_loss, - "val_loss": val_loss, + "lr_scheduler": lr_scheduler.state_dict(), + # Metrics + "metrics": metrics, } timer = atomic_torch_save(checkpoint, args.resume, timer) - val_loss /= val_images_seen + val_loss = metrics["val"].agg[metrics["val"].map["val_loss"]] / metrics["val"].agg[metrics["val"].map["val_images_seen"]] + if utils.is_main_process(): + writer = SummaryWriter(log_dir=args.tboard_path) + writer.add_scalar("Val/loss", val_loss, epoch) + writer.flush() + writer.close() print(f"Epoch {epoch} diff val loss: {val_loss:.4f}") - return timer + return timer, metrics diff --git a/monai/brats_mri_2d_0/prep.py b/monai/brats_mri_2d_V1/prep.py similarity index 100% rename from monai/brats_mri_2d_0/prep.py rename to monai/brats_mri_2d_V1/prep.py diff --git a/monai/brats_mri_2d_0/train_cycling_diff.py b/monai/brats_mri_2d_V1/train_cycling_diff.py similarity index 50% rename from monai/brats_mri_2d_0/train_cycling_diff.py rename to monai/brats_mri_2d_V1/train_cycling_diff.py index c1665320..6e600aab 100644 --- a/monai/brats_mri_2d_0/train_cycling_diff.py +++ b/monai/brats_mri_2d_V1/train_cycling_diff.py @@ -4,59 +4,46 @@ timer.report('importing Timer') import os - -import matplotlib.pyplot as plt -import numpy as np import torch -import torch.nn.functional as F +# import torch.nn.functional as F from monai import transforms from monai.apps import DecathlonDataset -from monai.config import print_config -from monai.data import DataLoader, Dataset +# from monai.config import print_config +from monai.data import DataLoader #, Dataset from monai.utils import first, set_determinism from torch.cuda.amp import GradScaler, autocast from pathlib import Path -from tqdm import tqdm +# from tqdm import tqdm from generative.inferers import LatentDiffusionInferer -from generative.losses.adversarial_loss import PatchAdversarialLoss -from generative.losses.perceptual import PerceptualLoss -from generative.networks.nets import AutoencoderKL, DiffusionModelUNet, PatchDiscriminator +# from generative.losses.adversarial_loss import PatchAdversarialLoss +# from generative.losses.perceptual import PerceptualLoss +from generative.networks.nets import AutoencoderKL, DiffusionModelUNet # , PatchDiscriminator from generative.networks.schedulers import DDPMScheduler -from cycling_utils import InterruptableDistributedSampler, Timer +from cycling_utils import InterruptableDistributedSampler, Timer, MetricsTracker # from loops import train_generator_one_epoch, evaluate_generator from loops import train_diffusion_one_epoch, evaluate_diffusion import utils def get_args_parser(add_help=True): import argparse - parser = argparse.ArgumentParser(description="Latent Diffusion Model Training", add_help=add_help) - parser.add_argument("--resume", type=str, help="path of checkpoint", required=True) # for checkpointing parser.add_argument("--gen-load-path", type=str, help="path of checkpoint", dest="gen_load_path") # for checkpointing parser.add_argument("--prev-resume", default=None, help="path of previous job checkpoint for strong fail resume", dest="prev_resume") # for checkpointing - # parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") + parser.add_argument("--tboard-path", default=None, help="path for saving tensorboard logs", dest="tboard_path") # for checkpointing parser.add_argument("--data-path", default="/mnt/Datasets/Open-Datasets/MONAI", type=str, help="dataset path", dest="data_path") parser.add_argument("--device", default="cuda", type=str, help="device (Use cuda or cpu Default: cuda)") parser.add_argument("--start-epoch", default=0, type=int, metavar="N", help="start epoch") - # parser.add_argument("-b", "--batch-size", default=8, type=int, help="images per gpu, the total batch size is $NGPU x batch_size", dest="batch_size") - # parser.add_argument("--epochs", default=30, type=int, metavar="N", help="number of total epochs to run") - # parser.add_argument("--print-freq", default=1, type=int, help="print frequency") parser.add_argument("--dist-url", default="env://", type=str, help="url used to set up distributed training") parser.add_argument("-j", "--workers", default=16, type=int, metavar="N", help="number of data loading workers (default: 16)") - return parser timer.report('importing everything else') def main(args, timer): - # ## Distributed training prelims - # if args.output_dir: - # utils.mkdir(args.output_dir) - utils.init_distributed_mode(args) # Sets args.distributed among other things assert args.distributed # don't support cycling when not distributed for simplicity @@ -67,31 +54,36 @@ def main(args, timer): # Maybe this will work? set_determinism(42) - channel = 0 # 0 = Flair + channel = 0 # 0 = "Flair" channel assert channel in [0, 1, 2, 3], "Choose a valid channel" - ## NEED TO COME BACK AND ALIGN WITH BRATS CONFIG - train_transforms = transforms.Compose([ - transforms.LoadImaged(keys=["image", "label"], image_only=False), # image_only current default will change soon, so including explicitly - transforms.EnsureChannelFirstd(keys=["image", "label"]), - transforms.Lambdad(keys=["image"], func=lambda x: x[channel, None, :, :, :]), - transforms.EnsureTyped(keys=["image", "label"]), - transforms.Orientationd(keys=["image", "label"], axcodes="RAS"), - transforms.Spacingd(keys=["image", "label"], pixdim=(3.0, 3.0, 2.0), mode=("bilinear", "nearest")), - transforms.CenterSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 44)), - transforms.ScaleIntensityRangePercentilesd(keys="image", lower=0, upper=99.5, b_min=0, b_max=1), - transforms.RandSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 1), random_size=False, random_centre=True), # Eeach of the 44 slices will be randomly sampled. - transforms.Lambdad(keys=["image", "label"], func=lambda x: x.squeeze(-1)), - transforms.CopyItemsd(keys=["label"], times=1, names=["slice_label"]), - transforms.Lambdad(keys=["slice_label"], func=lambda x: 1.0 if x.sum() > 0 else 0.0), + preprocessing_transform = transforms.Compose([ + transforms.LoadImaged(keys="image", image_only=False), # image_only current default will change soon, so including explicitly + transforms.EnsureChannelFirstd(keys="image"), + transforms.Lambdad(keys="image", func=lambda x: x[channel, :, :, :]), + transforms.AddChanneld(keys="image"), + transforms.EnsureTyped(keys="image"), + transforms.Orientationd(keys="image", axcodes="RAS"), + transforms.CenterSpatialCropd(keys="image", roi_size=(240, 240, 100)), + transforms.ScaleIntensityRangePercentilesd(keys="image", lower=0, upper=100, b_min=0, b_max=1), + ]) + + crop_transform = transforms.Compose([ + transforms.DivisiblePadd(keys="image", k=[32,32,1]), + transforms.RandSpatialCropd(keys="image", roi_size=(256, 256, 1), random_size=False), # Each of the 100 slices will be randomly sampled. + transforms.SqueezeDimd(keys="image", dim=3), + # transforms.RandFlipd(keys="image", prob=0.5, spatial_axis=0), + # transforms.RandFlipd(keys="image", prob=0.5, spatial_axis=1), ]) + preprocessing = transforms.Compose([preprocessing_transform, crop_transform]) + train_ds = DecathlonDataset( root_dir=args.data_path, task="Task01_BrainTumour", section="training", cache_rate=0.0, - num_workers=4, download=False, seed=0, transform=train_transforms, + num_workers=8, download=False, seed=0, transform=preprocessing, ) val_ds = DecathlonDataset( root_dir=args.data_path, task="Task01_BrainTumour", section="validation", cache_rate=0.0, - num_workers=4, download=False, seed=0, transform=train_transforms, + num_workers=8, download=False, seed=0, transform=preprocessing, ) timer.report('build datasets') @@ -101,8 +93,9 @@ def main(args, timer): timer.report('build samplers') - train_loader = DataLoader(train_ds, batch_size=2, sampler=train_sampler, num_workers=args.workers) - val_loader = DataLoader(val_ds, batch_size=1, sampler=val_sampler, num_workers=args.workers) + # Original trainer had batch size = 2 * 50. Using 10 nodes x batch size 10 => eff batch size = 100 + train_loader = DataLoader(train_ds, batch_size=10, sampler=train_sampler, num_workers=1) + val_loader = DataLoader(val_ds, batch_size=1, sampler=val_sampler, num_workers=1) check_data = first(train_loader) # Used later timer.report('build dataloaders') @@ -120,69 +113,39 @@ def main(args, timer): timer.report('generator to device') - # # Discriminator definition - # discriminator = PatchDiscriminator( - # spatial_dims=2, num_layers_d=3, num_channels=64, - # in_channels=1, out_channels=1 - # ) - # discriminator = discriminator.to(device) - - # timer.report('discriminator to device') - # Diffusion model (unet) unet = DiffusionModelUNet( - spatial_dims=2, in_channels=3, out_channels=3, num_res_blocks=2, - num_channels=(128, 256, 512),attention_levels=(False, True, True), - num_head_channels=(0, 256, 512), + spatial_dims=2, in_channels=1, out_channels=1, num_res_blocks=2, + num_channels=(32, 64, 128, 256), attention_levels=(False, True, True, True), + num_head_channels=(0, 32, 32, 32), ) unet = unet.to(device) timer.report('unet to device') - # # Autoencoder loss functions - # adv_loss = PatchAdversarialLoss(criterion="least_squares") - # perceptual_loss = PerceptualLoss(spatial_dims=2, network_type="alex") - # perceptual_loss.to(device) - - # timer.report('loss functions') - # Prepare for distributed training - # generator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) - # discriminator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) unet = torch.nn.SyncBatchNorm.convert_sync_batchnorm(unet) - # generator_without_ddp = generator - # discriminator_without_ddp = discriminator unet_without_ddp = unet if args.distributed: - # generator = torch.nn.parallel.DistributedDataParallel(generator, device_ids=[args.gpu], find_unused_parameters=True) - # discriminator = torch.nn.parallel.DistributedDataParallel(discriminator, device_ids=[args.gpu], find_unused_parameters=True) unet = torch.nn.parallel.DistributedDataParallel(unet, device_ids=[args.gpu], find_unused_parameters=True) - # generator_without_ddp = generator.module - # discriminator_without_ddp = discriminator.module unet_without_ddp = unet.module timer.report('unet prepped for distribution') # Optimizers - # optimizer_g = torch.optim.Adam(generator_without_ddp.parameters(), lr=1e-4) - # optimizer_d = torch.optim.Adam(discriminator_without_ddp.parameters(), lr=5e-4) - optimizer_u = torch.optim.Adam(unet_without_ddp.parameters(), lr=1e-4) - - timer.report('optimizers') + optimizer_u = torch.optim.Adam(unet_without_ddp.parameters(), lr=5e-5) + lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer_u, milestones=[1000], gamma=0.1) # For mixed precision training - # scaler_g = GradScaler() - # scaler_d = GradScaler() scaler_u = GradScaler() - timer.report('grad scalers') + timer.report('optimizer, lr_scheduler and grad scaler') - # Init tracking metrics - train_images_seen = 0 - val_images_seen = 0 - epoch_loss = 0 - val_loss = 0 + # Init metric tracker + train_metrics = MetricsTracker(["train_images_seen", "epoch_loss"]) + val_metrics = MetricsTracker(["val_images_seen", "val_loss"]) + metrics = {'train': train_metrics, 'val': val_metrics} # RETRIEVE GENERATOR CHECKPOINT FROM PREVIOUS JOB @@ -200,54 +163,21 @@ def main(args, timer): scaler_u.load_state_dict(checkpoint["scaler_u"]) train_sampler.load_state_dict(checkpoint["train_sampler"]) val_sampler.load_state_dict(checkpoint["val_sampler"]) - train_images_seen = checkpoint["train_images_seen"] - val_images_seen = checkpoint["val_images_seen"] + lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) # Metrics - train_images_seen = checkpoint["train_images_seen"] - val_images_seen = checkpoint["val_images_seen"] - epoch_loss = checkpoint["epoch_loss"] - val_loss = checkpoint["val_loss"] + metrics = checkpoint["metrics"] + metrics["train"].to(device) + metrics["val"].to(device) timer.report('checkpoint retrieval') - # ## -- TRAINING THE AUTO-ENCODER - ## - - # n_gen_epochs = 100 - # gen_val_interval = 1 - - # for epoch in range(args.start_epoch, n_gen_epochs): - - # print('\n') - # print(f"EPOCH :: {epoch}") - # print('\n') - - # with train_sampler.in_epoch(epoch): - # timer = Timer("Start training") - # generator, timer = train_generator_one_epoch( - # args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, - # scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, - # train_images_seen, val_images_seen, epoch_loss, gen_epoch_loss, disc_epoch_loss, val_loss - # ) - # timer.report(f'training generator for epoch {epoch}') - - # if epoch % gen_val_interval == 0: # Eval every epoch - # with val_sampler.in_epoch(epoch): - # timer = Timer("Start evaluation") - # timer = evaluate_generator( - # args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, - # scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, - # train_images_seen, val_images_seen, epoch_loss, gen_epoch_loss, disc_epoch_loss, val_loss - # ) - # timer.report(f'evaluating generator for epoch {epoch}') - - ## -- TRAINING THE DIFFUSION MODEL - ## n_diff_epochs = 200 diff_val_interval = 1 # Prepare LatentDiffusionInferer - scheduler = DDPMScheduler(num_train_timesteps=1000, schedule="linear_beta", beta_start=0.0015, beta_end=0.0195) + scheduler = DDPMScheduler(num_train_timesteps=1000, schedule="scaled_linear_beta", beta_start=0.0015, beta_end=0.0195) with torch.no_grad(): with autocast(enabled=True): z = generator.encode_stage_2_inputs(check_data["image"].to(device)) @@ -264,18 +194,18 @@ def main(args, timer): with train_sampler.in_epoch(epoch): timer = Timer("Start training") - unet, timer = train_diffusion_one_epoch( + unet, timer, metrics = train_diffusion_one_epoch( args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, - train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer + train_sampler, val_sampler, lr_scheduler, device, timer, metrics ) timer.report(f'training unet for epoch {epoch}') if epoch % diff_val_interval == 0: with val_sampler.in_epoch(epoch): timer = Timer("Start evaluation") - timer = evaluate_diffusion( + timer, metrics = evaluate_diffusion( args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, - train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer + train_sampler, val_sampler, lr_scheduler, device, timer, metrics ) timer.report(f'evaluating unet for epoch {epoch}') diff --git a/monai/brats_mri_2d_0/train_cycling_gen_0.py b/monai/brats_mri_2d_V1/train_cycling_gen.py similarity index 73% rename from monai/brats_mri_2d_0/train_cycling_gen_0.py rename to monai/brats_mri_2d_V1/train_cycling_gen.py index d8db2b91..6a7d85fe 100644 --- a/monai/brats_mri_2d_0/train_cycling_gen_0.py +++ b/monai/brats_mri_2d_V1/train_cycling_gen.py @@ -4,97 +4,50 @@ timer.report('importing Timer') import os - -# import matplotlib.pyplot as plt -# import numpy as np import torch -import torch.distributed as dist -# import torch.nn.functional as F from monai import transforms from monai.apps import DecathlonDataset -# from monai.config import print_config -from monai.data import DataLoader# , Dataset -from monai.utils import first, set_determinism -from torch.cuda.amp import GradScaler# , autocast +from monai.data import DataLoader +from monai.utils import set_determinism +from torch.cuda.amp import GradScaler from pathlib import Path -# from tqdm import tqdm -# from generative.inferers import LatentDiffusionInferer from generative.losses.adversarial_loss import PatchAdversarialLoss from generative.losses.perceptual import PerceptualLoss -from generative.networks.nets import AutoencoderKL, PatchDiscriminator # , DiffusionModelUNet -# from generative.networks.schedulers import DDPMScheduler +from generative.networks.nets import AutoencoderKL, PatchDiscriminator -from cycling_utils import InterruptableDistributedSampler, Timer -from loops_0 import train_generator_one_epoch, evaluate_generator -# from loops import train_diffusion_one_epoch, evaluate_diffusion +from cycling_utils import InterruptableDistributedSampler, Timer, MetricsTracker +from loops import train_generator_one_epoch, evaluate_generator import utils def get_args_parser(add_help=True): import argparse - parser = argparse.ArgumentParser(description="Latent Diffusion Model Training", add_help=add_help) - parser.add_argument("--resume", type=str, help="path of checkpoint", required=True) # for checkpointing parser.add_argument("--prev-resume", default=None, help="path of previous job checkpoint for strong fail resume", dest="prev_resume") # for checkpointing - # parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") + parser.add_argument("--tboard-path", default=None, help="path for saving tensorboard logs", dest="tboard_path") # for checkpointing parser.add_argument("--data-path", default="/mnt/Datasets/Open-Datasets/MONAI", type=str, help="dataset path", dest="data_path") parser.add_argument("--device", default="cuda", type=str, help="device (Use cuda or cpu Default: cuda)") parser.add_argument("--start-epoch", default=0, type=int, metavar="N", help="start epoch") - # parser.add_argument("-b", "--batch-size", default=8, type=int, help="images per gpu, the total batch size is $NGPU x batch_size", dest="batch_size") - # parser.add_argument("--epochs", default=30, type=int, metavar="N", help="number of total epochs to run") - # parser.add_argument("--print-freq", default=1, type=int, help="print frequency") parser.add_argument("--dist-url", default="env://", type=str, help="url used to set up distributed training") parser.add_argument("-j", "--workers", default=16, type=int, metavar="N", help="number of data loading workers (default: 16)") - return parser timer.report('importing everything else') -class MetricsTracker: - def __init__(self, metric_names): - self.map = {n:i for i,n in enumerate(metric_names)} - self.local = torch.zeros(len(metric_names), dtype=torch.float16, requires_grad=False, device='cuda') - self.agg = torch.zeros(len(metric_names), dtype=torch.float16, requires_grad=False, device='cuda') - self.epoch_reports = [] - - def update(self, metrics: dict): - for n,v in metrics.items(): - self.local[self.map[n]] += v - - def reduce_and_reset_local(self): - # Reduce over all nodes, add that to local store, and reset local - dist.all_reduce(self.local, op=dist.ReduceOp.SUM) - self.agg += self.local - self.local = torch.zeros(len(self.local), dtype=torch.float16, requires_grad=False, device='cuda') - - def end_epoch(self): - self.epoch_reports.append(self.agg) - self.local = torch.zeros(len(self.local), dtype=torch.float16, requires_grad=False, device='cuda') - self.agg = torch.zeros(len(self.local), dtype=torch.float16, requires_grad=False, device='cuda') - - def to(self, device): - self.local = self.local.to(device) - self.agg = self.agg.to(device) - - def main(args, timer): - # ## Distributed training prelims - # if args.output_dir: - # utils.mkdir(args.output_dir) - utils.init_distributed_mode(args) # Sets args.distributed among other things assert args.distributed # don't support cycling when not distributed for simplicity device = torch.device(args.device) - timer.report('preliminaries') - # Maybe this will work? set_determinism(42) - channel = 0 # 0 = Flair + timer.report('preliminaries') + + channel = 0 # 0 = "Flair" channel assert channel in [0, 1, 2, 3], "Choose a valid channel" preprocessing_transform = transforms.Compose([ transforms.LoadImaged(keys="image", image_only=False), # image_only current default will change soon, so including explicitly @@ -109,7 +62,6 @@ def main(args, timer): crop_transform = transforms.Compose([ transforms.DivisiblePadd(keys="image", k=[4,4,1]), - # transforms.RandSpatialCropSamplesd(keys="image", roi_size=(240, 240, 1), random_size=False, num_samples=26), transforms.RandSpatialCropd(keys="image", roi_size=(240, 240, 1), random_size=False), # Each of the 100 slices will be randomly sampled. transforms.SqueezeDimd(keys="image", dim=3), transforms.RandFlipd(keys="image", prob=0.5, spatial_axis=0), @@ -137,7 +89,6 @@ def main(args, timer): # Original trainer had batch size = 26. Using 9 nodes x batch size 3 = eff batch size = 27 train_loader = DataLoader(train_ds, batch_size=3, sampler=train_sampler, num_workers=1) val_loader = DataLoader(val_ds, batch_size=1, sampler=val_sampler, num_workers=1) - # check_data = first(train_loader) # Used later timer.report('build dataloaders') @@ -161,16 +112,6 @@ def main(args, timer): timer.report('discriminator to device') - # # Diffusion model (unet) - # unet = DiffusionModelUNet( - # spatial_dims=2, in_channels=3, out_channels=3, num_res_blocks=2, - # num_channels=(128, 256, 512),attention_levels=(False, True, True), - # num_head_channels=(0, 256, 512), - # ) - # unet = unet.to(device) - - # timer.report('unet to device') - # Autoencoder loss functions adv_loss = PatchAdversarialLoss(criterion="least_squares") perceptual_loss = PerceptualLoss( @@ -183,32 +124,26 @@ def main(args, timer): # Prepare for distributed training generator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) discriminator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) - # unet = torch.nn.SyncBatchNorm.convert_sync_batchnorm(unet) generator_without_ddp = generator discriminator_without_ddp = discriminator - # unet_without_ddp = unet if args.distributed: generator = torch.nn.parallel.DistributedDataParallel(generator, device_ids=[args.gpu], find_unused_parameters=True) # find_unused_parameters necessary for monai training discriminator = torch.nn.parallel.DistributedDataParallel(discriminator, device_ids=[args.gpu], find_unused_parameters=True) # find_unused_parameters necessary for monai training - # unet = torch.nn.parallel.DistributedDataParallel(unet, device_ids=[args.gpu], find_unused_parameters=True) generator_without_ddp = generator.module discriminator_without_ddp = discriminator.module - # unet_without_ddp = unet.module timer.report('models prepped for distribution') # Optimizers optimizer_g = torch.optim.Adam(generator_without_ddp.parameters(), lr=5e-5) optimizer_d = torch.optim.Adam(discriminator_without_ddp.parameters(), lr=5e-5) - # optimizer_u = torch.optim.Adam(unet_without_ddp.parameters(), lr=1e-4) timer.report('optimizers') # For mixed precision training scaler_g = GradScaler() scaler_d = GradScaler() - # scaler_u = GradScaler() timer.report('grad scalers') diff --git a/monai/brats_mri_2d/utils.py b/monai/brats_mri_2d_V1/utils.py similarity index 100% rename from monai/brats_mri_2d/utils.py rename to monai/brats_mri_2d_V1/utils.py diff --git a/monai/brats_mri_2d/brats_mri_2d_diff.isc b/monai/brats_mri_2d_v0/brats_mri_2d_diff.isc similarity index 100% rename from monai/brats_mri_2d/brats_mri_2d_diff.isc rename to monai/brats_mri_2d_v0/brats_mri_2d_diff.isc diff --git a/monai/brats_mri_2d/brats_mri_2d_gen.isc b/monai/brats_mri_2d_v0/brats_mri_2d_gen.isc similarity index 100% rename from monai/brats_mri_2d/brats_mri_2d_gen.isc rename to monai/brats_mri_2d_v0/brats_mri_2d_gen.isc diff --git a/monai/brats_mri_2d/loops.py b/monai/brats_mri_2d_v0/loops.py similarity index 100% rename from monai/brats_mri_2d/loops.py rename to monai/brats_mri_2d_v0/loops.py diff --git a/monai/brats_mri_2d/prep.py b/monai/brats_mri_2d_v0/prep.py similarity index 100% rename from monai/brats_mri_2d/prep.py rename to monai/brats_mri_2d_v0/prep.py diff --git a/monai/brats_mri_2d/train_cycling_diff.py b/monai/brats_mri_2d_v0/train_cycling_diff.py similarity index 100% rename from monai/brats_mri_2d/train_cycling_diff.py rename to monai/brats_mri_2d_v0/train_cycling_diff.py diff --git a/monai/brats_mri_2d/train_cycling_gen.py b/monai/brats_mri_2d_v0/train_cycling_gen.py similarity index 100% rename from monai/brats_mri_2d/train_cycling_gen.py rename to monai/brats_mri_2d_v0/train_cycling_gen.py diff --git a/monai/brats_mri_2d_0/utils.py b/monai/brats_mri_2d_v0/utils.py similarity index 100% rename from monai/brats_mri_2d_0/utils.py rename to monai/brats_mri_2d_v0/utils.py diff --git a/monai/brats_mri_2d/backup/LICENSE b/monai/monai.bundle/LICENSE similarity index 100% rename from monai/brats_mri_2d/backup/LICENSE rename to monai/monai.bundle/LICENSE diff --git a/monai/brats_mri_2d/backup/brats_mri_2d.isc b/monai/monai.bundle/brats_mri_2d.isc similarity index 100% rename from monai/brats_mri_2d/backup/brats_mri_2d.isc rename to monai/monai.bundle/brats_mri_2d.isc diff --git a/monai/brats_mri_2d/backup/configs/inference.json b/monai/monai.bundle/configs/inference.json similarity index 100% rename from monai/brats_mri_2d/backup/configs/inference.json rename to monai/monai.bundle/configs/inference.json diff --git a/monai/brats_mri_2d/backup/configs/inference_autoencoder.json b/monai/monai.bundle/configs/inference_autoencoder.json similarity index 100% rename from monai/brats_mri_2d/backup/configs/inference_autoencoder.json rename to monai/monai.bundle/configs/inference_autoencoder.json diff --git a/monai/brats_mri_2d/backup/configs/logging.conf b/monai/monai.bundle/configs/logging.conf similarity index 100% rename from monai/brats_mri_2d/backup/configs/logging.conf rename to monai/monai.bundle/configs/logging.conf diff --git a/monai/brats_mri_2d/backup/configs/metadata.json b/monai/monai.bundle/configs/metadata.json similarity index 100% rename from monai/brats_mri_2d/backup/configs/metadata.json rename to monai/monai.bundle/configs/metadata.json diff --git a/monai/brats_mri_2d/backup/configs/multi_gpu_train_autoencoder.json b/monai/monai.bundle/configs/multi_gpu_train_autoencoder.json similarity index 100% rename from monai/brats_mri_2d/backup/configs/multi_gpu_train_autoencoder.json rename to monai/monai.bundle/configs/multi_gpu_train_autoencoder.json diff --git a/monai/brats_mri_2d/backup/configs/multi_gpu_train_diffusion.json b/monai/monai.bundle/configs/multi_gpu_train_diffusion.json similarity index 100% rename from monai/brats_mri_2d/backup/configs/multi_gpu_train_diffusion.json rename to monai/monai.bundle/configs/multi_gpu_train_diffusion.json diff --git a/monai/brats_mri_2d/backup/configs/train_autoencoder.json b/monai/monai.bundle/configs/train_autoencoder.json similarity index 100% rename from monai/brats_mri_2d/backup/configs/train_autoencoder.json rename to monai/monai.bundle/configs/train_autoencoder.json diff --git a/monai/brats_mri_2d/backup/configs/train_diffusion.json b/monai/monai.bundle/configs/train_diffusion.json similarity index 100% rename from monai/brats_mri_2d/backup/configs/train_diffusion.json rename to monai/monai.bundle/configs/train_diffusion.json diff --git a/monai/brats_mri_2d/backup/docs/README.md b/monai/monai.bundle/docs/README.md similarity index 100% rename from monai/brats_mri_2d/backup/docs/README.md rename to monai/monai.bundle/docs/README.md diff --git a/monai/brats_mri_2d/backup/docs/data_license.txt b/monai/monai.bundle/docs/data_license.txt similarity index 100% rename from monai/brats_mri_2d/backup/docs/data_license.txt rename to monai/monai.bundle/docs/data_license.txt diff --git a/monai/brats_mri_2d/backup/scripts/__init__.py b/monai/monai.bundle/scripts/__init__.py similarity index 100% rename from monai/brats_mri_2d/backup/scripts/__init__.py rename to monai/monai.bundle/scripts/__init__.py diff --git a/monai/brats_mri_2d/backup/scripts/ldm_sampler.py b/monai/monai.bundle/scripts/ldm_sampler.py similarity index 100% rename from monai/brats_mri_2d/backup/scripts/ldm_sampler.py rename to monai/monai.bundle/scripts/ldm_sampler.py diff --git a/monai/brats_mri_2d/backup/scripts/ldm_trainer.py b/monai/monai.bundle/scripts/ldm_trainer.py similarity index 100% rename from monai/brats_mri_2d/backup/scripts/ldm_trainer.py rename to monai/monai.bundle/scripts/ldm_trainer.py diff --git a/monai/brats_mri_2d/backup/scripts/losses.py b/monai/monai.bundle/scripts/losses.py similarity index 100% rename from monai/brats_mri_2d/backup/scripts/losses.py rename to monai/monai.bundle/scripts/losses.py diff --git a/monai/brats_mri_2d/backup/scripts/utils.py b/monai/monai.bundle/scripts/utils.py similarity index 100% rename from monai/brats_mri_2d/backup/scripts/utils.py rename to monai/monai.bundle/scripts/utils.py diff --git a/tv-detection/engine.py b/tv-detection/engine.py index 87f8ecc1..e3b1112d 100644 --- a/tv-detection/engine.py +++ b/tv-detection/engine.py @@ -10,7 +10,6 @@ from cycling_utils import atomic_torch_save from torch.utils.tensorboard import SummaryWriter -tb_path = "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/output_maskrcnn_resnet50_fpn/tb" def train_one_epoch( model, optimizer, data_loader_train, train_sampler, test_sampler, @@ -68,14 +67,16 @@ def train_one_epoch( train_metrics.update({"images_seen": len(images) ,"loss": loss_value}) train_metrics.update({k:v.item() for k,v in loss_dict_reduced.items()}) - train_metrics.reduce_and_reset_local() - + train_metrics.reduce() # Gather results from all nodes + report_metrics = ["loss", "loss_box_reg", "loss_classifier", "loss_mask", "loss_objectness", "loss_rpn_box_reg"] - norm = train_metrics.agg[train_metrics.map["images_seen"]] - vals = [train_metrics.agg[train_metrics.map[k]]/norm for k in report_metrics] + norm = train_metrics.local[train_metrics.map["images_seen"]] + vals = [train_metrics.local[train_metrics.map[k]]/norm for k in report_metrics] rpt = ", ".join([f"{k}: {v:,.3f}" for k,v in zip(report_metrics, vals)]) print(f"EPOCH: [{epoch}], BATCH: [{train_sampler.progress}/{len(train_sampler)}], "+rpt) + train_metrics.reset_local() + # metric_logger.update(loss=losses_reduced, **loss_dict_reduced) # metric_logger.update(lr=optimizer.param_groups[0]["lr"]) # timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: updating metric logger') @@ -92,7 +93,7 @@ def train_one_epoch( if utils.is_main_process() and train_sampler.progress % 1 == 0: # Checkpointing every batch - writer = SummaryWriter(log_dir=tb_path) + writer = SummaryWriter(log_dir=args.tboard_path) for metric,val in zip(report_metrics, vals): writer.add_scalar("Train/"+metric, val, train_sampler.progress + epoch * len(train_sampler)) writer.flush() @@ -120,7 +121,6 @@ def train_one_epoch( # return metric_logger, timer return model, timer, train_metrics - def _get_iou_types(model): model_without_ddp = model if isinstance(model, torch.nn.parallel.DistributedDataParallel): @@ -132,7 +132,6 @@ def _get_iou_types(model): iou_types.append("keypoints") return iou_types - @torch.inference_mode() def evaluate( model, data_loader_test, epoch, test_sampler, args, coco_evaluator, @@ -181,12 +180,11 @@ def evaluate( # metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) timer.report(f'Epoch {epoch} batch: {test_step} update evaluator') + print(f"Saving checkpoint at epoch {epoch} eval batch {test_step}") test_sampler.advance(len(images)) - test_step = test_sampler.progress // data_loader_test.batch_size if utils.is_main_process() and test_step % 1 == 0: # Checkpointing every batch - print(f"Saving checkpoint at epoch {epoch} eval batch {test_step}") checkpoint = { "args": args, "epoch": epoch, @@ -215,7 +213,7 @@ def evaluate( results = coco_evaluator.summarize() if utils.is_main_process(): - writer = SummaryWriter(log_dir=tb_path) + writer = SummaryWriter(log_dir=args.tboard_path) for i,val in enumerate(results): writer.add_scalar(f"Eval/F{i}", val, test_step + epoch * total_steps) writer.flush() diff --git a/tv-detection/maskrcnn_resnet50_fpn.isc b/tv-detection/maskrcnn_resnet50_fpn.isc index c44bd096..4cc8c061 100644 --- a/tv-detection/maskrcnn_resnet50_fpn.isc +++ b/tv-detection/maskrcnn_resnet50_fpn.isc @@ -3,4 +3,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_maskrcnn_resnet50_fpn" -command="train_cycling.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file +command="train_cycling.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_maskrcnn_resnet50_fpn/tb" \ No newline at end of file diff --git a/tv-detection/maskrcnn_resnet50_fpn_small.isc b/tv-detection/maskrcnn_resnet50_fpn_small.isc deleted file mode 100644 index 505b6dd9..00000000 --- a/tv-detection/maskrcnn_resnet50_fpn_small.isc +++ /dev/null @@ -1,6 +0,0 @@ -experiment_name="maskrcnn_resnet50_fpn_small" -gpu_type="24GB VRAM GPU" -nnodes = 11 -venv_path = "~/.venv/bin/activate" -output_path = "~/output_maskrcnn_resnet50_fpn" -command="train_cycling_small.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/tv-detection/retinanet_resnet50_fpn.isc b/tv-detection/retinanet_resnet50_fpn.isc index c2e03e44..f97baa3a 100644 --- a/tv-detection/retinanet_resnet50_fpn.isc +++ b/tv-detection/retinanet_resnet50_fpn.isc @@ -3,5 +3,5 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_retinanet_resnet50_fpn" -command="train_cycling.py --dataset coco --model retinanet_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --lr 0.01 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --prev-resume ~/output_retinanet_resnet50_fpn/exp_1639/checkpoint.isc" +command="train_cycling.py --dataset coco --model retinanet_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --lr 0.01 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_retinanet_resnet50_fpn/tb" diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index 74ed8ee0..220ce63a 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -39,7 +39,7 @@ from torchvision.transforms import InterpolationMode from transforms import SimpleCopyPaste -from cycling_utils import InterruptableDistributedSampler, InterruptableDistributedGroupedBatchSampler +from cycling_utils import InterruptableDistributedSampler, InterruptableDistributedGroupedBatchSampler, MetricsTracker timer.report('importing everything else') @@ -97,8 +97,8 @@ def main(args, timer): if args.dataset == "coco_kp" and args.use_v2: raise ValueError("KeyPoint detection doesn't support V2 transforms yet") - if args.output_dir: - utils.mkdir(args.output_dir) + # if args.output_dir: + # utils.mkdir(args.output_dir) utils.init_distributed_mode(args) print(args) @@ -232,7 +232,7 @@ def main(args, timer): timer.report('init coco evaluator') - train_metrics = utils.MetricsTracker(["images_seen", "loss", "loss_box_reg", "loss_classifier", "loss_mask", "loss_objectness", "loss_rpn_box_reg"]) + train_metrics = MetricsTracker(["images_seen", "loss", "loss_box_reg", "loss_classifier", "loss_mask", "loss_objectness", "loss_rpn_box_reg"]) # RETRIEVE CHECKPOINT Path(args.resume).parent.mkdir(parents=True, exist_ok=True) @@ -283,6 +283,7 @@ def main(args, timer): timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch coco_evaluator, timer, train_metrics = evaluate(model, data_loader_test, epoch, test_sampler, args, coco_evaluator, optimizer, lr_scheduler, warmup_lr_scheduler, train_sampler, device, scaler, timer, train_metrics) + def get_args_parser(add_help=True): import argparse @@ -306,8 +307,12 @@ def get_args_parser(add_help=True): parser.add_argument("--lr-gamma", default=0.1, type=float, help="decrease lr by a factor of lr-gamma (multisteplr scheduler only)") parser.add_argument("--print-freq", default=1, type=int, help="print frequency") parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") + parser.add_argument("--resume", default="", type=str, help="path of checkpoint") parser.add_argument("--prev-resume", default=None, help="path of previous job checkpoint for strong fail resume", dest="prev_resume") # for checkpointing + parser.add_argument("--tboard-path", default=None, help="path for saving tensorboard logs", dest="tboard_path") # for checkpointing + + parser.add_argument("--start_epoch", default=0, type=int, help="start epoch") parser.add_argument("--aspect-ratio-group-factor", default=3, type=int) parser.add_argument("--rpn-score-thresh", default=None, type=float, help="rpn score threshold for faster-rcnn") diff --git a/tv-detection/train_cycling_small.py b/tv-detection/train_cycling_small.py deleted file mode 100644 index 4fb6f148..00000000 --- a/tv-detection/train_cycling_small.py +++ /dev/null @@ -1,335 +0,0 @@ -r"""PyTorch Detection Training. - -To run in a multi-gpu environment, use the distributed launcher:: - - python -m torch.distributed.launch --nproc_per_node=$NGPU --use_env \ - train.py ... --world-size $NGPU - -The default hyperparameters are tuned for training on 8 gpus and 2 images per gpu. - --lr 0.02 --batch-size 2 --world-size 8 -If you use different number of gpus, the learning rate should be changed to 0.02/8*$NGPU. - -On top of that, for training Faster/Mask R-CNN, the default hyperparameters are - --epochs 26 --lr-steps 16 22 --aspect-ratio-group-factor 3 - -Also, if you train Keypoint R-CNN, the default hyperparameters are - --epochs 46 --lr-steps 36 43 --aspect-ratio-group-factor 3 -Because the number of images is smaller in the person keypoint subset of COCO, -the number of epochs should be adapted so that we have the same number of iterations. -""" -from cycling_utils import Timer - -timer = Timer() -timer.report('importing Timer') - -import datetime -import os -import time -# import warnings - -from pathlib import Path -import presets -import torch -import torch.utils.data -import torchvision -import utils -from coco_utils import get_coco - -import torchvision.models.detection -import torchvision.models.detection.mask_rcnn -from engine import evaluate, train_one_epoch -from group_by_aspect_ratio import create_aspect_ratio_groups, GroupedBatchSampler -from torchvision.transforms import InterpolationMode -from transforms import SimpleCopyPaste - -from cycling_utils import InterruptableDistributedSampler, InterruptableDistributedGroupedBatchSampler, atomic_torch_save - -timer.report('importing everything else') - -def copypaste_collate_fn(batch): - copypaste = SimpleCopyPaste(blending=True, resize_interpolation=InterpolationMode.BILINEAR) - return copypaste(*utils.collate_fn(batch)) - -def get_dataset(is_train, args): - image_set = "train" if is_train else "val" - num_classes, mode = {"coco": (91, "instances"), "coco_kp": (2, "person_keypoints")}[args.dataset] - with_masks = "mask" in args.model - ds = get_coco( - root=args.data_path, - image_set=image_set, - transforms=get_transform(is_train, args), - mode=mode, - use_v2=args.use_v2, - with_masks=with_masks, - ) - return ds, num_classes - -def get_transform(is_train, args): - if is_train: - return presets.DetectionPresetTrain( - data_augmentation=args.data_augmentation, backend=args.backend, use_v2=args.use_v2 - ) - elif args.weights and args.test_only: - weights = torchvision.models.get_weight(args.weights) - trans = weights.transforms() - return lambda img, target: (trans(img), target) - else: - return presets.DetectionPresetEval(backend=args.backend, use_v2=args.use_v2) - -def _get_iou_types(model): # intersection over union (iou) types - model_without_ddp = model - if isinstance(model, torch.nn.parallel.DistributedDataParallel): - model_without_ddp = model.module - iou_types = ["bbox"] - if isinstance(model_without_ddp, torchvision.models.detection.MaskRCNN): - iou_types.append("segm") - if isinstance(model_without_ddp, torchvision.models.detection.KeypointRCNN): - iou_types.append("keypoints") - return iou_types - -timer.report('defined other functions') - -def main(args, timer): - - if args.backend.lower() == "tv_tensor" and not args.use_v2: - raise ValueError("Use --use-v2 if you want to use the tv_tensor backend.") - if args.dataset not in ("coco", "coco_kp"): - raise ValueError(f"Dataset should be coco or coco_kp, got {args.dataset}") - if "keypoint" in args.model and args.dataset != "coco_kp": - raise ValueError("Oops, if you want Keypoint detection, set --dataset coco_kp") - if args.dataset == "coco_kp" and args.use_v2: - raise ValueError("KeyPoint detection doesn't support V2 transforms yet") - - if args.output_dir: - utils.mkdir(args.output_dir) - - utils.init_distributed_mode(args) - print(args) - - device = torch.device(args.device) - - if args.use_deterministic_algorithms: - torch.use_deterministic_algorithms(True) - - timer.report('main preliminaries') - - # Data loading code - dataset_train, num_classes = get_dataset(is_train=True, args=args) - dataset_test, _ = get_dataset(is_train=False, args=args) - - ## SUBSET FOR TESTING - dataset_train = torch.utils.data.Subset(dataset_train, torch.arange(2*11*5)) # batch_size x nodes x iterations - dataset_test = torch.utils.data.Subset(dataset_test, torch.arange(1*11*2)) - - timer.report('loading data') - - # if args.distributed: - # train_sampler = torch.utils.data.distributed.DistributedSampler(dataset) - # test_sampler = torch.utils.data.distributed.DistributedSampler(dataset_test, shuffle=False) - # else: - # train_sampler = torch.utils.data.RandomSampler(dataset) - # test_sampler = torch.utils.data.SequentialSampler(dataset_test) - - # if args.aspect_ratio_group_factor >= 0: # default == 3 - # group_ids = create_aspect_ratio_groups(dataset_train, k=args.aspect_ratio_group_factor) - # train_batch_sampler = GroupedBatchSampler(train_sampler, group_ids, args.batch_size) - # else: - # train_batch_sampler = torch.utils.data.BatchSampler(train_sampler, args.batch_size, drop_last=True) - - group_ids = create_aspect_ratio_groups(dataset_train, k=args.aspect_ratio_group_factor) - train_sampler = InterruptableDistributedGroupedBatchSampler(dataset_train, group_ids, args.batch_size) - test_sampler = InterruptableDistributedSampler(dataset_test) - - timer.report('creating data samplers') - - train_collate_fn = utils.collate_fn - if args.use_copypaste: - if args.data_augmentation != "lsj": - raise RuntimeError("SimpleCopyPaste algorithm currently only supports the 'lsj' data augmentation policies") - print("Using copypaste_collate_fn for train_collate_fn") - train_collate_fn = copypaste_collate_fn - - data_loader_train = torch.utils.data.DataLoader( - dataset_train, batch_sampler=train_sampler, num_workers=args.workers, collate_fn=train_collate_fn - ) - data_loader_test = torch.utils.data.DataLoader( - dataset_test, batch_size=1, sampler=test_sampler, num_workers=args.workers, collate_fn=utils.collate_fn - ) - - timer.report('creating data loaders') - - kwargs = {"trainable_backbone_layers": args.trainable_backbone_layers} - if args.data_augmentation in ["multiscale", "lsj"]: - kwargs["_skip_resize"] = True - if "rcnn" in args.model: - if args.rpn_score_thresh is not None: - kwargs["rpn_score_thresh"] = args.rpn_score_thresh - - model = torchvision.models.get_model( - args.model, weights=args.weights, weights_backbone=args.weights_backbone, num_classes=num_classes, **kwargs - ) - model.to(device) - - timer.report('creating model and .to(device)') - - if args.distributed and args.sync_bn: - model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model) - - model_without_ddp = model - if args.distributed: - model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.gpu]) - model_without_ddp = model.module - - timer.report('preparing model for distributed training') - - if args.norm_weight_decay is None: - parameters = [p for p in model.parameters() if p.requires_grad] - else: - param_groups = torchvision.ops._utils.split_normalization_params(model) - wd_groups = [args.norm_weight_decay, args.weight_decay] - parameters = [{"params": p, "weight_decay": w} for p, w in zip(param_groups, wd_groups) if p] - - opt_name = args.opt.lower() - if opt_name.startswith("sgd"): - optimizer = torch.optim.SGD( - parameters, - lr=args.lr, - momentum=args.momentum, - weight_decay=args.weight_decay, - nesterov="nesterov" in opt_name, - ) - elif opt_name == "adamw": - optimizer = torch.optim.AdamW(parameters, lr=args.lr, weight_decay=args.weight_decay) - else: - raise RuntimeError(f"Invalid optimizer {args.opt}. Only SGD and AdamW are supported.") - - scaler = torch.cuda.amp.GradScaler() if args.amp else None - - timer.report('optimizer and scaler') - - ## OUTER LR_SCHEDULER - args.lr_scheduler = args.lr_scheduler.lower() - if args.lr_scheduler == "multisteplr": - lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=args.lr_steps, gamma=args.lr_gamma) - elif args.lr_scheduler == "cosineannealinglr": - lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=args.epochs) - else: - raise RuntimeError( - f"Invalid lr scheduler '{args.lr_scheduler}'. Only MultiStepLR and CosineAnnealingLR are supported." - ) - - ## WARMUP LR_SCHEDULER - warmup_factor = 1.0 / 1000 - warmup_iters = min(1000, len(data_loader_train) - 1) - warmup_lr_scheduler = torch.optim.lr_scheduler.LinearLR( - optimizer, start_factor=warmup_factor, total_iters=warmup_iters - ) - - timer.report('learning rate schedulers') - - from coco_eval import CocoEvaluator - from coco_utils import get_coco_api_from_dataset - coco = get_coco_api_from_dataset(data_loader_test.dataset) - iou_types = _get_iou_types(model) - coco_evaluator = CocoEvaluator(coco, iou_types) - - timer.report('init coco evaluator') - - Path(args.resume).parent.mkdir(parents=True, exist_ok=True) - if args.resume and os.path.isfile(args.resume): - - checkpoint = torch.load(args.resume, map_location="cpu") - model_without_ddp.load_state_dict(checkpoint["model"]) - args.start_epoch = checkpoint["epoch"] - - optimizer.load_state_dict(checkpoint["optimizer"]) - lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) - warmup_lr_scheduler.load_state_dict(checkpoint["warmup_lr_scheduler"]) - train_sampler.load_state_dict(checkpoint["train_sampler"]) - if args.amp: - scaler.load_state_dict(checkpoint["scaler"]) - - test_sampler.load_state_dict(checkpoint["test_sampler"]) - - # Evaluator state variables - coco_evaluator.img_ids = checkpoint["img_ids"] - coco_evaluator.eval_imgs = checkpoint["eval_imgs"] - - timer.report('retrieving checkpoint') - - if args.test_only: - # We disable the cudnn benchmarking because it can noticeably affect the accuracy - torch.backends.cudnn.benchmark = False - torch.backends.cudnn.deterministic = True - coco_evaluator, timer = evaluate(model, data_loader_test, device, timer) - return - - for epoch in range(args.start_epoch, args.epochs): - - print('\n') - print(f"EPOCH :: {epoch}") - print('\n') - - with train_sampler.in_epoch(epoch): - timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch - metric_logger, timer = train_one_epoch(model, optimizer, data_loader_train, train_sampler, test_sampler, lr_scheduler, warmup_lr_scheduler, args, device, coco_evaluator, epoch, scaler, timer) - - # NEST THE TEST SAMPLER IN TRAIN SAMPLER CONTEXT TO AVOID EPOCH RESTART? - with test_sampler.in_epoch(epoch): - timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch - coco_evaluator, timer = evaluate(model, data_loader_test, epoch, test_sampler, args, coco_evaluator, optimizer, lr_scheduler, warmup_lr_scheduler, train_sampler, device, scaler, timer) - -def get_args_parser(add_help=True): - import argparse - - parser = argparse.ArgumentParser(description="PyTorch Detection Training", add_help=add_help) - - parser.add_argument("--data-path", default="/datasets01/COCO/022719/", type=str, help="dataset path") - parser.add_argument("--dataset",default="coco",type=str,help="dataset name. Use coco for object detection and instance segmentation and coco_kp for Keypoint detection",) - parser.add_argument("--model", default="maskrcnn_resnet50_fpn", type=str, help="model name") - parser.add_argument("--device", default="cuda", type=str, help="device (Use cuda or cpu Default: cuda)") - parser.add_argument("-b", "--batch-size", default=2, type=int, help="images per gpu, the total batch size is $NGPU x batch_size") - parser.add_argument("--epochs", default=26, type=int, metavar="N", help="number of total epochs to run") - parser.add_argument("-j", "--workers", default=4, type=int, metavar="N", help="number of data loading workers (default: 4)") - parser.add_argument("--opt", default="sgd", type=str, help="optimizer") - parser.add_argument("--lr",default=0.02,type=float,help="initial learning rate, 0.02 is the default value for training on 8 gpus and 2 images_per_gpu") - parser.add_argument("--momentum", default=0.9, type=float, metavar="M", help="momentum") - parser.add_argument("--wd","--weight-decay",default=1e-4,type=float,metavar="W",help="weight decay (default: 1e-4)",dest="weight_decay",) - parser.add_argument("--norm-weight-decay",default=None,type=float,help="weight decay for Normalization layers (default: None, same value as --wd)") - parser.add_argument("--lr-scheduler", default="multisteplr", type=str, help="name of lr scheduler (default: multisteplr)") - parser.add_argument("--lr-step-size", default=8, type=int, help="decrease lr every step-size epochs (multisteplr scheduler only)") - parser.add_argument("--lr-steps",default=[16, 22],nargs="+",type=int,help="decrease lr every step-size epochs (multisteplr scheduler only)") - parser.add_argument("--lr-gamma", default=0.1, type=float, help="decrease lr by a factor of lr-gamma (multisteplr scheduler only)") - parser.add_argument("--print-freq", default=1, type=int, help="print frequency") - parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") - parser.add_argument("--resume", default="", type=str, help="path of checkpoint") - parser.add_argument("--start_epoch", default=0, type=int, help="start epoch") - parser.add_argument("--aspect-ratio-group-factor", default=3, type=int) - parser.add_argument("--rpn-score-thresh", default=None, type=float, help="rpn score threshold for faster-rcnn") - parser.add_argument("--trainable-backbone-layers", default=None, type=int, help="number of trainable layers of backbone") - parser.add_argument("--data-augmentation", default="hflip", type=str, help="data augmentation policy (default: hflip)") - parser.add_argument("--sync-bn",dest="sync_bn",help="Use sync batch norm",action="store_true") - parser.add_argument("--test-only",dest="test_only",help="Only test the model",action="store_true") - parser.add_argument("--use-deterministic-algorithms", action="store_true", help="Forces the use of deterministic algorithms only.") - - # distributed training parameters - parser.add_argument("--world-size", default=1, type=int, help="number of distributed processes") - parser.add_argument("--dist-url", default="env://", type=str, help="url used to set up distributed training") - parser.add_argument("--weights", default=None, type=str, help="the weights enum name to load") - parser.add_argument("--weights-backbone", default=None, type=str, help="the backbone weights enum name to load") - - # Mixed precision training parameters - parser.add_argument("--amp", action="store_true", help="Use torch.cuda.amp for mixed precision training") - - # Use CopyPaste augmentation training parameter - parser.add_argument("--use-copypaste",action="store_true",help="Use CopyPaste data augmentation. Works only with data-augmentation='lsj'.",) - - parser.add_argument("--backend", default="PIL", type=str.lower, help="PIL or tensor - case insensitive") - parser.add_argument("--use-v2", action="store_true", help="Use V2 transforms") - - return parser - - -if __name__ == "__main__": - args = get_args_parser().parse_args() - main(args, timer) diff --git a/tv-detection/utils.py b/tv-detection/utils.py index 4d09685d..dc4298ab 100644 --- a/tv-detection/utils.py +++ b/tv-detection/utils.py @@ -5,38 +5,7 @@ from collections import defaultdict, deque import torch -import torch.distributed as dist - -class MetricsTracker: - def __init__(self, metric_names): - self.metric_names = metric_names - self.map = {n:i for i,n in enumerate(metric_names)} - self.local = torch.zeros(len(metric_names), dtype=torch.float16, requires_grad=False, device='cuda') - self.agg = torch.zeros(len(metric_names), dtype=torch.float16, requires_grad=False, device='cuda') - self.epoch_reports = [] - - def update(self, metrics: dict): - for n,v in metrics.items(): - self.local[self.map[n]] += v - - def reduce_and_reset_local(self): - # Reduce over all nodes, add that to local store, and reset local - dist.all_reduce(self.local, op=dist.ReduceOp.SUM) - self.agg += self.local - self.local = torch.zeros(len(self.local), dtype=torch.float16, requires_grad=False, device='cuda') - - def end_epoch(self): - self.epoch_reports.append(self.agg) - self.local = torch.zeros(len(self.local), dtype=torch.float16, requires_grad=False, device='cuda') - self.agg = torch.zeros(len(self.local), dtype=torch.float16, requires_grad=False, device='cuda') - - def to(self, device): - self.local = self.local.to(device) - self.agg = self.agg.to(device) - - # def report(self): - # return ", ".join([f"{k}: {v:,.3f}" for k,v in zip(self.metric_names, self.agg)]) - +import torch.distributed as dist class SmoothedValue: """Track a series of values and provide access to smoothed values over a diff --git a/tv-segmentation/deeplabv3_mobilenet_v3_large.isc b/tv-segmentation/deeplabv3_mobilenet_v3_large.isc index 3a76f734..e02a8c63 100644 --- a/tv-segmentation/deeplabv3_mobilenet_v3_large.isc +++ b/tv-segmentation/deeplabv3_mobilenet_v3_large.isc @@ -4,4 +4,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_deeplabv3_mobilenet_v3_large" -command="train_cycling.py --dataset coco -b 2 --model deeplabv3_mobilenet_v3_large --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --prev-resume ~/output_deeplabv3_mobilenet_v3_large/exp_1654/checkpoint.isc" +command="train_cycling.py --dataset coco -b 2 --model deeplabv3_mobilenet_v3_large --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_deeplabv3_mobilenet_v3_large/tb" diff --git a/tv-segmentation/fcn_resnet101.isc b/tv-segmentation/fcn_resnet101.isc index bfdbb7c8..8b148024 100644 --- a/tv-segmentation/fcn_resnet101.isc +++ b/tv-segmentation/fcn_resnet101.isc @@ -4,4 +4,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_fcn_resnet101" -command="train_cycling.py --lr 0.02 --dataset coco -b 4 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --prev-resume ~/output_fcn_resnet101/exp_1664/checkpoint.isc" \ No newline at end of file +command="train_cycling.py --lr 0.02 --dataset coco -b 4 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_fcn_resnet101/tb" \ No newline at end of file diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index 3d198b1d..a4e81e07 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -3,10 +3,7 @@ timer = Timer() timer.report('importing Timer') -import datetime -import os -import time -import warnings +import os, warnings from pathlib import Path import presets @@ -18,7 +15,9 @@ from torch import nn from torch.optim.lr_scheduler import PolynomialLR from torchvision.transforms import functional as F, InterpolationMode -from cycling_utils import InterruptableDistributedSampler, atomic_torch_save, Timer +from cycling_utils import InterruptableDistributedSampler, atomic_torch_save, Timer, MetricsTracker + +from torch.utils.tensorboard import SummaryWriter timer.report('importing everything else') @@ -57,32 +56,113 @@ def criterion(inputs, target): return losses["out"] return losses["out"] + 0.5 * losses["aux"] +def train_one_epoch( + args, model, criterion, optimizer, data_loader_train, + train_sampler, test_sampler, confmat, lr_scheduler, + device, epoch, scaler=None, timer=None, train_metrics=None + ): + + model.train() + # metric_logger = utils.MetricLogger(delimiter=" ") + # metric_logger.add_meter("lr", utils.SmoothedValue(window_size=1, fmt="{value}")) + # header = f"Epoch: [{epoch}]" + + # Running this before starting the training loop assists reporting on progress after resuming - train_step == batch count + # Also means when resuming during evaluation, the training phase is skipped as train_sampler progress == 100%. + train_step = train_sampler.progress // data_loader_train.batch_size + total_steps = len(train_sampler) // data_loader_train.batch_size + print(f'\nTraining / resuming epoch {epoch} from training step {train_step}\n') + timer.report('launch training routine') + + for images, target in data_loader_train: + # for image, target in metric_logger.log_every(data_loader_train, train_step, print_freq, header): + + images, target = images.to(device), target.to(device) + timer.report(f'Epoch: {epoch} batch {train_step}: moving batch data to device') + + optimizer.zero_grad() + + with torch.cuda.amp.autocast(enabled=scaler is not None): + output = model(images) + loss = criterion(output, target) + timer.report(f'Epoch: {epoch} batch {train_step}: forward pass') + + if scaler is not None: + scaler.scale(loss).backward() + scaler.step(optimizer) + scaler.update() + else: + loss.backward() + optimizer.step() + lr_scheduler.step() + + timer.report(f'Epoch: {epoch} batch {train_step}: backward pass') + + train_metrics.update({"images_seen": len(images), "loss": loss.item()}) + train_metrics.reduce() # Reduce to sync metrics between nodes for this batch + batch_loss = train_metrics.local[train_metrics.map["loss"]] / train_metrics.local[train_metrics.map["images_seen"]] + print(f"EPOCH: [{epoch}], BATCH: [{train_step}/{total_steps}], loss: {batch_loss}") + train_metrics.reset_local() + + # metric_logger.update(loss=loss.item(), lr=optimizer.param_groups[0]["lr"]) + print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") + train_sampler.advance(len(images)) + + train_step = train_sampler.progress // data_loader_train.batch_size + if train_step == total_steps: + train_metrics.end_epoch() + + if utils.is_main_process() and train_step % 1 == 0: # Checkpointing every batch + + writer = SummaryWriter(log_dir=args.tboard_path) + writer.add_scalar("Train/loss", batch_loss, train_step + epoch * total_steps) + writer.flush() + writer.close() + + checkpoint = { + "args": args, + "epoch": epoch, + "model": model.module.state_dict(), + "optimizer": optimizer.state_dict(), + "lr_scheduler": lr_scheduler.state_dict(), + "train_sampler": train_sampler.state_dict(), + "test_sampler": test_sampler.state_dict(), + "confmat": confmat.mat, + "confmat_temp": confmat.temp_mat, + "train_metrics": train_metrics, + } + if args.amp: + checkpoint["scaler"] = scaler.state_dict() + timer = atomic_torch_save(checkpoint, args.resume, timer) + + # return metric_logger, timer + return model, timer, train_metrics + def evaluate( - model, data_loader_test, num_classes, confmat, - optimizer, lr_scheduler, - train_sampler: InterruptableDistributedSampler, - test_sampler: InterruptableDistributedSampler, - device, epoch, print_freq, scaler=None, timer=None + args, model, data_loader_test, num_classes, confmat, + optimizer, lr_scheduler, train_sampler, test_sampler, + device, epoch, scaler=None, timer=None, train_metrics=None, ): model.eval() - metric_logger = utils.MetricLogger(delimiter=" ") - header = "Test:" - num_processed_samples = 0 - timer.report(f'evaluation preliminaries') + # metric_logger = utils.MetricLogger(delimiter=" ") + # header = "Test:" + # num_processed_samples = 0 test_step = test_sampler.progress // data_loader_test.batch_size + total_steps = len(test_sampler) // data_loader_test.batch_size print(f'\nEvaluating / resuming epoch {epoch} from eval step {test_step}\n') timer.report('launch evaluation routine') with torch.inference_mode(): - for image, target in metric_logger.log_every(data_loader_test, test_step, print_freq, header): + for images, target in data_loader_test: + # for images, target in metric_logger.log_every(data_loader_test, test_step, print_freq, header): - image, target = image.to(device), target.to(device) + images, target = images.to(device), target.to(device) timer.report(f'Epoch {epoch} batch: {test_step} moving to device') - output = model(image) + output = model(images) output = output["out"] timer.report(f'Epoch {epoch} batch: {test_step} forward through model') @@ -91,15 +171,14 @@ def evaluate( # FIXME need to take into account that the datasets # could have been padded in distributed setup - num_processed_samples += image.shape[0] + # num_processed_samples += images.shape[0] timer.report(f'Epoch {epoch} batch: {test_step} confmat update') - test_sampler.advance(len(image)) - + print(f"Saving checkpoint at epoch {epoch} eval batch {test_step}") + test_sampler.advance(len(images)) test_step = test_sampler.progress // data_loader_test.batch_size - if utils.is_main_process() and test_step % 1 == 0: # Checkpointing every batch - print(f"Saving checkpoint at epoch {epoch} eval batch {test_step}") + if utils.is_main_process() and test_step % 1 == 0: # Checkpointing every batch checkpoint = { "args": args, "epoch": epoch, @@ -108,105 +187,53 @@ def evaluate( "lr_scheduler": lr_scheduler.state_dict(), "train_sampler": train_sampler.state_dict(), "test_sampler": test_sampler.state_dict(), - "confmat": confmat.mat, # For storing eval metric "confmat_temp": confmat.temp_mat, # For storing eval metric + "train_metrics": train_metrics, } if args.amp: checkpoint["scaler"] = scaler.state_dict() timer = atomic_torch_save(checkpoint, args.resume, timer) - print(confmat) + # Report key performance metrics + acc_global, acc, iu = confmat.compute() + acc = acc_global.item() * 100 + mean_iou = iu.mean().item() * 100 + print(f"EPOCH: [{epoch}] EVAL :: acc: {acc:.2f}, mean_iou: {mean_iou:.2f}") confmat.reset() - num_processed_samples = utils.reduce_across_processes(num_processed_samples) - if ( - hasattr(data_loader_test.dataset, "__len__") - and len(data_loader_test.dataset) != num_processed_samples - and torch.distributed.get_rank() == 0 - ): - # See FIXME above - warnings.warn( - f"It looks like the dataset has {len(data_loader_test.dataset)} samples, but {num_processed_samples} " - "samples were used for the validation, which might bias the results. " - "Try adjusting the batch size and / or the world size. " - "Setting the world size to 1 is always a safe bet." - ) + if utils.is_main_process(): + writer = SummaryWriter(log_dir=args.tboard_path) + writer.add_scalar("Val/acc", acc, epoch) + writer.add_scalar("Val/mean_iou", mean_iou, epoch) + writer.flush() + writer.close() + + # num_processed_samples = utils.reduce_across_processes(num_processed_samples) + + # if ( + # hasattr(data_loader_test.dataset, "__len__") + # and len(data_loader_test.dataset) != num_processed_samples + # and torch.distributed.get_rank() == 0 + # ): + # # See FIXME above + # warnings.warn( + # f"It looks like the dataset has {len(data_loader_test.dataset)} samples, but {num_processed_samples} " + # "samples were used for the validation, which might bias the results. " + # "Try adjusting the batch size and / or the world size. " + # "Setting the world size to 1 is always a safe bet." + # ) return confmat, timer +timer.report('defined other functions') -def train_one_epoch( - model, criterion, optimizer, data_loader_train, - train_sampler: InterruptableDistributedSampler, - test_sampler: InterruptableDistributedSampler, confmat, - lr_scheduler, device, epoch, print_freq, scaler=None, timer=None - ): - - model.train() - metric_logger = utils.MetricLogger(delimiter=" ") - metric_logger.add_meter("lr", utils.SmoothedValue(window_size=1, fmt="{value}")) - header = f"Epoch: [{epoch}]" - - timer.report('training preliminaries') - - # Running this before starting the training loop assists reporting on progress after resuming - train_step == batch count - # Also means when resuming during evaluation, the training phase is skipped as train_sampler progress == 100%. - train_step = train_sampler.progress // data_loader_train.batch_size - print(f'\nTraining / resuming epoch {epoch} from training step {train_step}\n') - - for image, target in metric_logger.log_every(data_loader_train, train_step, print_freq, header): - - image, target = image.to(device), target.to(device) - timer.report(f'Epoch: {epoch} batch {train_step}: moving batch data to device') - - with torch.cuda.amp.autocast(enabled=scaler is not None): - output = model(image) - loss = criterion(output, target) - timer.report(f'Epoch: {epoch} batch {train_step}: forward pass') - - optimizer.zero_grad() - if scaler is not None: - scaler.scale(loss).backward() - scaler.step(optimizer) - scaler.update() - else: - loss.backward() - optimizer.step() - timer.report(f'Epoch: {epoch} batch {train_step}: backward pass') - - lr_scheduler.step() - metric_logger.update(loss=loss.item(), lr=optimizer.param_groups[0]["lr"]) - train_sampler.advance(len(image)) - timer.report(f'Epoch: {epoch} batch {train_step}: updating metric logger') - - train_step = train_sampler.progress // data_loader_train.batch_size - if utils.is_main_process() and train_step % 1 == 0: # Checkpointing every batch - print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") - checkpoint = { - "args": args, - "epoch": epoch, - "model": model.module.state_dict(), - "optimizer": optimizer.state_dict(), - "lr_scheduler": lr_scheduler.state_dict(), - "train_sampler": train_sampler.state_dict(), - "test_sampler": test_sampler.state_dict(), - - "confmat": confmat.mat, # For storing eval metric - "confmat_temp": confmat.temp_mat, # For storing eval metric - } - if args.amp: - checkpoint["scaler"] = scaler.state_dict() - timer = atomic_torch_save(checkpoint, args.resume, timer) - - return metric_logger, timer -timer.report('defined other functions') def main(args, timer): - if args.output_dir: - utils.mkdir(args.output_dir) + # if args.output_dir: + # utils.mkdir(args.output_dir) utils.init_distributed_mode(args) print(args) @@ -314,6 +341,9 @@ def main(args, timer): confmat = utils.ConfusionMatrix(num_classes) timer.report('init confmat') + # Init general purpose metrics tracker + train_metrics = MetricsTracker(["images_seen", "loss"]) + # RETRIEVE CHECKPOINT Path(args.resume).parent.mkdir(parents=True, exist_ok=True) checkpoint = None @@ -325,28 +355,29 @@ def main(args, timer): checkpoint = torch.load(args.prev_resume, map_location="cpu") if checkpoint is not None: - model_without_ddp.load_state_dict(checkpoint["model"], strict=not args.test_only) args.start_epoch = checkpoint["epoch"] - + model_without_ddp.load_state_dict(checkpoint["model"], strict=not args.test_only) + optimizer.load_state_dict(checkpoint["optimizer"]) lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) train_sampler.load_state_dict(checkpoint["train_sampler"]) + test_sampler.load_state_dict(checkpoint["test_sampler"]) if args.amp: # Could align this syntactically... scaler.load_state_dict(checkpoint["scaler"]) - - test_sampler.load_state_dict(checkpoint["test_sampler"]) confmat.mat = checkpoint["confmat"] confmat.temp_mat = checkpoint["confmat_temp"] + train_metrics = checkpoint["train_metrics"] + train_metrics.to(device) timer.report('retrieving checkpoint') - if args.test_only: - # We disable the cudnn benchmarking because it can noticeably affect the accuracy - torch.backends.cudnn.benchmark = False - torch.backends.cudnn.deterministic = True - confmat, timer = evaluate(model, data_loader_test, num_classes, confmat, test_sampler, device, 0, args.print_freq, timer) - print(confmat) - return + # if args.test_only: + # # We disable the cudnn benchmarking because it can noticeably affect the accuracy + # torch.backends.cudnn.benchmark = False + # torch.backends.cudnn.deterministic = True + # confmat, timer = evaluate(model, data_loader_test, num_classes, confmat, test_sampler, device, 0, timer) + # print(confmat) + # return for epoch in range(args.start_epoch, args.epochs): @@ -356,13 +387,21 @@ def main(args, timer): with train_sampler.in_epoch(epoch): timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch - metric_logger, timer = train_one_epoch(model, criterion, optimizer, data_loader_train, train_sampler, test_sampler, confmat, lr_scheduler, device, epoch, args.print_freq, scaler, timer) + model, timer, train_metrics = train_one_epoch( + args, model, criterion, optimizer, data_loader_train, + train_sampler, test_sampler, confmat, lr_scheduler, + device, epoch, scaler, timer, train_metrics + ) timer.report(f'training for epoch {epoch}') # NEST TEST SAMPLER IN TRAIN SAMPLER CONTEXT TO AVOID EPOCH RESTART? with test_sampler.in_epoch(epoch): timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch - confmat, timer = evaluate(model, data_loader_test, num_classes, confmat, optimizer, lr_scheduler, train_sampler, test_sampler, device, epoch, args.print_freq, scaler, timer) + confmat, timer = evaluate( + args, model, data_loader_test, num_classes, confmat, + optimizer, lr_scheduler, train_sampler, test_sampler, + device, epoch, scaler, timer, train_metrics, + ) timer.report(f'evaluation for epoch {epoch}') @@ -397,6 +436,7 @@ def get_args_parser(add_help=True): parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") parser.add_argument("--resume", type=str, help="path of checkpoint", required=True) parser.add_argument("--prev-resume", default=None, help="path of previous job checkpoint for strong fail resume", dest="prev_resume") # for checkpointing + parser.add_argument("--tboard-path", default=None, help="path for saving tensorboard logs", dest="tboard_path") # for checkpointing parser.add_argument("--start-epoch", default=0, type=int, metavar="N", help="start epoch") parser.add_argument( "--test-only", diff --git a/tv-segmentation/utils.py b/tv-segmentation/utils.py index 21255602..899a9bad 100644 --- a/tv-segmentation/utils.py +++ b/tv-segmentation/utils.py @@ -7,7 +7,6 @@ import torch import torch.distributed as dist - class SmoothedValue: """Track a series of values and provide access to smoothed values over a window or the global series average. From 5d7a69be57706171185bfe3cea8c1c0a9070ed48 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Tue, 19 Sep 2023 17:39:13 +1000 Subject: [PATCH 31/44] updates, updates, updates --- cycling_utils/cycling_utils/saving.py | 32 +++++++++--------- .../brats_mri_2d_diff.isc | 0 .../brats_mri_2d_gen.isc | 0 .../loops.py | 33 +++++++++++-------- .../prep.py | 0 .../train_cycling_diff.py | 10 +++--- .../train_cycling_gen.py | 10 +++--- .../utils.py | 0 tv-detection/engine.py | 8 +++-- tv-detection/train_cycling.py | 5 +-- tv-segmentation/train_cycling.py | 8 +++-- 11 files changed, 57 insertions(+), 49 deletions(-) rename monai/{brats_mri_2d_V1 => brats_mri_2d_v1}/brats_mri_2d_diff.isc (100%) rename monai/{brats_mri_2d_V1 => brats_mri_2d_v1}/brats_mri_2d_gen.isc (100%) rename monai/{brats_mri_2d_V1 => brats_mri_2d_v1}/loops.py (89%) rename monai/{brats_mri_2d_V1 => brats_mri_2d_v1}/prep.py (100%) rename monai/{brats_mri_2d_V1 => brats_mri_2d_v1}/train_cycling_diff.py (97%) rename monai/{brats_mri_2d_V1 => brats_mri_2d_v1}/train_cycling_gen.py (96%) rename monai/{brats_mri_2d_V1 => brats_mri_2d_v1}/utils.py (100%) diff --git a/cycling_utils/cycling_utils/saving.py b/cycling_utils/cycling_utils/saving.py index 40db3ff5..c29d57d7 100644 --- a/cycling_utils/cycling_utils/saving.py +++ b/cycling_utils/cycling_utils/saving.py @@ -2,6 +2,7 @@ import os import torch import torch.distributed as dist +from collections import defaultdict def atomic_torch_save(obj, f: str | Path, timer=None, **kwargs): f = str(f) @@ -15,35 +16,32 @@ def atomic_torch_save(obj, f: str | Path, timer=None, **kwargs): return timer else: return - + class MetricsTracker: - def __init__(self, metric_names): - self.metric_names = metric_names - self.map = {n:i for i,n in enumerate(metric_names)} - self.local = torch.zeros(len(metric_names), dtype=torch.float16, requires_grad=False, device='cuda') - self.agg = torch.zeros(len(metric_names), dtype=torch.float16, requires_grad=False, device='cuda') + def __init__(self): + self.local = defaultdict(float) + self.agg = defaultdict(float) self.epoch_reports = [] def update(self, metrics: dict): for n,v in metrics.items(): - self.local[self.map[n]] += v + self.local[n] += v def reduce(self): - # Reduce local over all nodes, add that to local store - dist.all_reduce(self.local, op=dist.ReduceOp.SUM) - self.agg += self.local + names, local = zip(*self.local.items()) + local = torch.tensor(local, dtype=torch.float16, requires_grad=False, device='cuda') + dist.all_reduce(local, op=dist.ReduceOp.SUM) + self.local = defaultdict(float, zip(names, local.cpu().numpy())) + for k in self.local: + self.agg[k] += self.local[k] def reset_local(self): - self.local = torch.zeros(len(self.local), dtype=torch.float16, requires_grad=False, device='cuda') + self.local = defaultdict(float) def end_epoch(self): self.epoch_reports.append(self.agg) - self.local = torch.zeros(len(self.local), dtype=torch.float16, requires_grad=False, device='cuda') - self.agg = torch.zeros(len(self.local), dtype=torch.float16, requires_grad=False, device='cuda') - - def to(self, device): - self.local = self.local.to(device) - self.agg = self.agg.to(device) + self.local = defaultdict(float) + self.agg = defaultdict(float) # ## ENABLING ACTIVE PROGRESS TRACKING diff --git a/monai/brats_mri_2d_V1/brats_mri_2d_diff.isc b/monai/brats_mri_2d_v1/brats_mri_2d_diff.isc similarity index 100% rename from monai/brats_mri_2d_V1/brats_mri_2d_diff.isc rename to monai/brats_mri_2d_v1/brats_mri_2d_diff.isc diff --git a/monai/brats_mri_2d_V1/brats_mri_2d_gen.isc b/monai/brats_mri_2d_v1/brats_mri_2d_gen.isc similarity index 100% rename from monai/brats_mri_2d_V1/brats_mri_2d_gen.isc rename to monai/brats_mri_2d_v1/brats_mri_2d_gen.isc diff --git a/monai/brats_mri_2d_V1/loops.py b/monai/brats_mri_2d_v1/loops.py similarity index 89% rename from monai/brats_mri_2d_V1/loops.py rename to monai/brats_mri_2d_v1/loops.py index 3d0fc19d..d7daf1ef 100644 --- a/monai/brats_mri_2d_V1/loops.py +++ b/monai/brats_mri_2d_v1/loops.py @@ -99,9 +99,12 @@ def train_generator_one_epoch( timer.report(f'train batch {train_step} metrics update') - recons_loss = metrics["train"].local[metrics["train"].map["epoch_loss"]] / metrics["train"].local[metrics["train"].map["train_images_seen"]] - gen_loss = metrics["train"].local[metrics["train"].map["gen_epoch_loss"]] / metrics["train"].local[metrics["train"].map["train_images_seen"]] - disc_loss = metrics["train"].local[metrics["train"].map["disc_epoch_loss"]] / metrics["train"].local[metrics["train"].map["train_images_seen"]] + # recons_loss = metrics["train"].local[metrics["train"].map["epoch_loss"]] / metrics["train"].local[metrics["train"].map["train_images_seen"]] + # gen_loss = metrics["train"].local[metrics["train"].map["gen_epoch_loss"]] / metrics["train"].local[metrics["train"].map["train_images_seen"]] + # disc_loss = metrics["train"].local[metrics["train"].map["disc_epoch_loss"]] / metrics["train"].local[metrics["train"].map["train_images_seen"]] + recons_loss = metrics["train"].local["epoch_loss"] / metrics["train"].local["train_images_seen"] + gen_loss = metrics["train"].local["gen_epoch_loss"] / metrics["train"].local["train_images_seen"] + disc_loss = metrics["train"].local["disc_epoch_loss"] / metrics["train"].local["train_images_seen"] print("Epoch [{}] Step [{}/{}] :: recons_loss: {:,.3f}, gen_loss: {:,.3f}, disc_loss: {:,.3f}".format(epoch, train_step, total_steps, recons_loss, gen_loss, disc_loss)) metrics["train"].reset_local() @@ -179,7 +182,15 @@ def evaluate_generator( val_step = val_sampler.progress // val_loader.batch_size if val_step == total_steps: - metrics["val"].end_epoch() + # val_loss = metrics["val"].agg[metrics["val"].map["val_loss"]] / metrics["val"].agg[metrics["val"].map["val_images_seen"]] + val_loss = metrics["val"].agg["val_loss"] / metrics["val"].agg["val_images_seen"] + if utils.is_main_process(): + writer = SummaryWriter(log_dir=args.tboard_path) + writer.add_scalar("Val/loss", val_loss, epoch) + writer.flush() + writer.close() + print(f"Epoch {epoch} val loss: {val_loss:.4f}") + metrics["val"].end_epoch() if utils.is_main_process() and val_step % 1 == 0: # Checkpointing every batch checkpoint = { @@ -200,14 +211,6 @@ def evaluate_generator( } timer = atomic_torch_save(checkpoint, args.resume, timer) - val_loss = metrics["val"].agg[metrics["val"].map["val_loss"]] / metrics["val"].agg[metrics["val"].map["val_images_seen"]] - if utils.is_main_process(): - writer = SummaryWriter(log_dir=args.tboard_path) - writer.add_scalar("Val/loss", val_loss, epoch) - writer.flush() - writer.close() - print(f"Epoch {epoch} val loss: {val_loss:.4f}") - return timer, metrics @@ -257,7 +260,8 @@ def train_diffusion_one_epoch( metrics["train"].update({"train_images_seen":len(images), "epoch_loss":loss.item()}) metrics["train"].reduce() - recons_loss = metrics["train"].local[metrics["train"].map["epoch_loss"]] / metrics["train"].local[metrics["train"].map["train_images_seen"]] + # recons_loss = metrics["train"].local[metrics["train"].map["epoch_loss"]] / metrics["train"].local[metrics["train"].map["train_images_seen"]] + recons_loss = metrics["train"].local["epoch_loss"] / metrics["train"].local["train_images_seen"] print("Epoch [{}] Step [{}/{}] :: recons_loss: {:,.3f}".format(epoch, train_step, total_steps, recons_loss)) metrics["train"].reset_local() @@ -361,7 +365,8 @@ def evaluate_diffusion( } timer = atomic_torch_save(checkpoint, args.resume, timer) - val_loss = metrics["val"].agg[metrics["val"].map["val_loss"]] / metrics["val"].agg[metrics["val"].map["val_images_seen"]] + # val_loss = metrics["val"].agg[metrics["val"].map["val_loss"]] / metrics["val"].agg[metrics["val"].map["val_images_seen"]] + val_loss = metrics["val"].agg["val_loss"] / metrics["val"].agg["val_images_seen"] if utils.is_main_process(): writer = SummaryWriter(log_dir=args.tboard_path) writer.add_scalar("Val/loss", val_loss, epoch) diff --git a/monai/brats_mri_2d_V1/prep.py b/monai/brats_mri_2d_v1/prep.py similarity index 100% rename from monai/brats_mri_2d_V1/prep.py rename to monai/brats_mri_2d_v1/prep.py diff --git a/monai/brats_mri_2d_V1/train_cycling_diff.py b/monai/brats_mri_2d_v1/train_cycling_diff.py similarity index 97% rename from monai/brats_mri_2d_V1/train_cycling_diff.py rename to monai/brats_mri_2d_v1/train_cycling_diff.py index 6e600aab..f40e7deb 100644 --- a/monai/brats_mri_2d_V1/train_cycling_diff.py +++ b/monai/brats_mri_2d_v1/train_cycling_diff.py @@ -143,9 +143,9 @@ def main(args, timer): timer.report('optimizer, lr_scheduler and grad scaler') # Init metric tracker - train_metrics = MetricsTracker(["train_images_seen", "epoch_loss"]) - val_metrics = MetricsTracker(["val_images_seen", "val_loss"]) - metrics = {'train': train_metrics, 'val': val_metrics} + # train_metrics = MetricsTracker(["train_images_seen", "epoch_loss"]) + # val_metrics = MetricsTracker(["val_images_seen", "val_loss"]) + metrics = {'train': MetricsTracker(), 'val': MetricsTracker()} # RETRIEVE GENERATOR CHECKPOINT FROM PREVIOUS JOB @@ -166,8 +166,8 @@ def main(args, timer): lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) # Metrics metrics = checkpoint["metrics"] - metrics["train"].to(device) - metrics["val"].to(device) + # metrics["train"].to(device) + # metrics["val"].to(device) timer.report('checkpoint retrieval') diff --git a/monai/brats_mri_2d_V1/train_cycling_gen.py b/monai/brats_mri_2d_v1/train_cycling_gen.py similarity index 96% rename from monai/brats_mri_2d_V1/train_cycling_gen.py rename to monai/brats_mri_2d_v1/train_cycling_gen.py index 6a7d85fe..45ceefc7 100644 --- a/monai/brats_mri_2d_V1/train_cycling_gen.py +++ b/monai/brats_mri_2d_v1/train_cycling_gen.py @@ -148,9 +148,9 @@ def main(args, timer): timer.report('grad scalers') # Init metric tracker - train_metrics = MetricsTracker(["train_images_seen", "epoch_loss", "gen_epoch_loss", "disc_epoch_loss"]) - val_metrics = MetricsTracker(["val_images_seen", "val_loss"]) - metrics = {'train': train_metrics, 'val': val_metrics} + # train_metrics = MetricsTracker(["train_images_seen", "epoch_loss", "gen_epoch_loss", "disc_epoch_loss"]) + # val_metrics = MetricsTracker(["val_images_seen", "val_loss"]) + metrics = {'train': MetricsTracker(), 'val': MetricsTracker()} # RETRIEVE CHECKPOINT Path(args.resume).parent.mkdir(parents=True, exist_ok=True) @@ -171,8 +171,8 @@ def main(args, timer): val_sampler.load_state_dict(checkpoint["val_sampler"]) # Metrics metrics = checkpoint["metrics"] - metrics["train"].to(device) - metrics["val"].to(device) + # metrics["train"].to(device) + # metrics["val"].to(device) timer.report('checkpoint retrieval') diff --git a/monai/brats_mri_2d_V1/utils.py b/monai/brats_mri_2d_v1/utils.py similarity index 100% rename from monai/brats_mri_2d_V1/utils.py rename to monai/brats_mri_2d_v1/utils.py diff --git a/tv-detection/engine.py b/tv-detection/engine.py index e3b1112d..fb03657a 100644 --- a/tv-detection/engine.py +++ b/tv-detection/engine.py @@ -69,9 +69,11 @@ def train_one_epoch( train_metrics.update({k:v.item() for k,v in loss_dict_reduced.items()}) train_metrics.reduce() # Gather results from all nodes - report_metrics = ["loss", "loss_box_reg", "loss_classifier", "loss_mask", "loss_objectness", "loss_rpn_box_reg"] - norm = train_metrics.local[train_metrics.map["images_seen"]] - vals = [train_metrics.local[train_metrics.map[k]]/norm for k in report_metrics] + report_metrics = ["loss", "loss_box_reg", "loss_classifier", "loss_mask", "loss_objectness", "loss_rpn_box_reg", "bbox_regression"] + # norm = train_metrics.local[train_metrics.map["images_seen"]] + norm = train_metrics.local["images_seen"] + # vals = [train_metrics.local[train_metrics.map[k]]/norm for k in report_metrics] + vals = [train_metrics.local[k]/norm for k in report_metrics] rpt = ", ".join([f"{k}: {v:,.3f}" for k,v in zip(report_metrics, vals)]) print(f"EPOCH: [{epoch}], BATCH: [{train_sampler.progress}/{len(train_sampler)}], "+rpt) diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index 220ce63a..88b9ab1d 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -232,7 +232,8 @@ def main(args, timer): timer.report('init coco evaluator') - train_metrics = MetricsTracker(["images_seen", "loss", "loss_box_reg", "loss_classifier", "loss_mask", "loss_objectness", "loss_rpn_box_reg"]) + # train_metrics = MetricsTracker(["images_seen", "loss", "loss_box_reg", "loss_classifier", "loss_mask", "loss_objectness", "loss_rpn_box_reg", "bbox_regression"]) + train_metrics = MetricsTracker() # RETRIEVE CHECKPOINT Path(args.resume).parent.mkdir(parents=True, exist_ok=True) @@ -257,7 +258,7 @@ def main(args, timer): coco_evaluator.img_ids = checkpoint["img_ids"] coco_evaluator.eval_imgs = checkpoint["eval_imgs"] train_metrics = checkpoint["train_metrics"] - train_metrics.to(device) + # train_metrics.to(device) timer.report('retrieving checkpoint') diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index a4e81e07..714212f4 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -100,7 +100,8 @@ def train_one_epoch( train_metrics.update({"images_seen": len(images), "loss": loss.item()}) train_metrics.reduce() # Reduce to sync metrics between nodes for this batch - batch_loss = train_metrics.local[train_metrics.map["loss"]] / train_metrics.local[train_metrics.map["images_seen"]] + # batch_loss = train_metrics.local[train_metrics.map["loss"]] / train_metrics.local[train_metrics.map["images_seen"]] + batch_loss = train_metrics.local["loss"] / train_metrics.local["images_seen"] print(f"EPOCH: [{epoch}], BATCH: [{train_step}/{total_steps}], loss: {batch_loss}") train_metrics.reset_local() @@ -342,7 +343,8 @@ def main(args, timer): timer.report('init confmat') # Init general purpose metrics tracker - train_metrics = MetricsTracker(["images_seen", "loss"]) + # train_metrics = MetricsTracker(["images_seen", "loss"]) + train_metrics = MetricsTracker() # RETRIEVE CHECKPOINT Path(args.resume).parent.mkdir(parents=True, exist_ok=True) @@ -367,7 +369,7 @@ def main(args, timer): confmat.mat = checkpoint["confmat"] confmat.temp_mat = checkpoint["confmat_temp"] train_metrics = checkpoint["train_metrics"] - train_metrics.to(device) + # train_metrics.to(device) timer.report('retrieving checkpoint') From 89bf0990b567d086ad3ad7ac921e12ce42e17a19 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Tue, 19 Sep 2023 20:19:54 +1000 Subject: [PATCH 32/44] code tidier --- monai/brats_mri_2d_v1/loops.py | 35 +++++++++++++++++++-- monai/brats_mri_2d_v1/train_cycling_diff.py | 21 ++++++++++--- tv-detection/engine.py | 33 +++---------------- tv-detection/train_cycling.py | 6 ---- tv-segmentation/train_cycling.py | 22 ++----------- 5 files changed, 54 insertions(+), 63 deletions(-) diff --git a/monai/brats_mri_2d_v1/loops.py b/monai/brats_mri_2d_v1/loops.py index d7daf1ef..467e7c1f 100644 --- a/monai/brats_mri_2d_v1/loops.py +++ b/monai/brats_mri_2d_v1/loops.py @@ -9,17 +9,46 @@ ## -- AUTO-ENCODER - ## +# def compute_kl_loss(z_mu, z_sigma): +# kl_loss = 0.5 * torch.sum( +# z_mu.pow(2) + z_sigma.pow(2) - torch.log(z_sigma.pow(2)) - 1, dim=list(range(1, len(z_sigma.shape))) +# ) +# return torch.sum(kl_loss) / kl_loss.shape[0] + +# def generator_loss(gen_images, real_images, z_mu, z_sigma, disc_net, loss_perceptual): +# recons_loss = intensity_loss(gen_images, real_images) +# kl_loss = compute_kl_loss(z_mu, z_sigma) +# p_loss = loss_perceptual(gen_images.float(), real_images.float()) +# loss_g = recons_loss + kl_weight * kl_loss + perceptual_weight * p_loss + +# logits_fake = disc_net(gen_images)[-1] +# generator_loss = adv_loss(logits_fake, target_is_real=True, for_discriminator=False) +# loss_g = loss_g + adv_weight * generator_loss + +# return loss_g + +# def discriminator_loss(gen_images, real_images, disc_net): +# logits_fake = disc_net(gen_images.contiguous().detach())[-1] +# loss_d_fake = adv_loss(logits_fake, target_is_real=False, for_discriminator=True) +# logits_real = disc_net(real_images.contiguous().detach())[-1] +# loss_d_real = adv_loss(logits_real, target_is_real=True, for_discriminator=True) +# discriminator_loss = (loss_d_fake + loss_d_real) * 0.5 +# loss_d = adv_weight * discriminator_loss +# return loss_d + + def train_generator_one_epoch( args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, metrics ): - # Maybe pull these out into args later + # Obtained from scripts.losses.generator_loss kl_weight = 1e-6 + perceptual_weight = 1.0 + adv_weight = 0.5 + # Retained from tutorial generator_warm_up_n_epochs = 10 - perceptual_weight = 0.001 - adv_weight = 0.01 generator.train() discriminator.train() diff --git a/monai/brats_mri_2d_v1/train_cycling_diff.py b/monai/brats_mri_2d_v1/train_cycling_diff.py index f40e7deb..93a1747c 100644 --- a/monai/brats_mri_2d_v1/train_cycling_diff.py +++ b/monai/brats_mri_2d_v1/train_cycling_diff.py @@ -40,6 +40,13 @@ def get_args_parser(add_help=True): parser.add_argument("-j", "--workers", default=16, type=int, metavar="N", help="number of data loading workers (default: 16)") return parser +def compute_scale_factor(autoencoder, train_loader, device): + with torch.no_grad(): + check_data = first(train_loader) + z = autoencoder.encode_stage_2_inputs(check_data["image"].to(device)) + scale_factor = 1 / torch.std(z) + return scale_factor.item() + timer.report('importing everything else') def main(args, timer): @@ -96,7 +103,7 @@ def main(args, timer): # Original trainer had batch size = 2 * 50. Using 10 nodes x batch size 10 => eff batch size = 100 train_loader = DataLoader(train_ds, batch_size=10, sampler=train_sampler, num_workers=1) val_loader = DataLoader(val_ds, batch_size=1, sampler=val_sampler, num_workers=1) - check_data = first(train_loader) # Used later + # check_data = first(train_loader) # Used later timer.report('build dataloaders') @@ -177,11 +184,15 @@ def main(args, timer): diff_val_interval = 1 # Prepare LatentDiffusionInferer + + # with torch.no_grad(): + # with autocast(enabled=True): + # z = generator.encode_stage_2_inputs(check_data["image"].to(device)) + # scale_factor = 1 / torch.std(z) + + scale_factor = compute_scale_factor(generator, train_loader, device) + scheduler = DDPMScheduler(num_train_timesteps=1000, schedule="scaled_linear_beta", beta_start=0.0015, beta_end=0.0195) - with torch.no_grad(): - with autocast(enabled=True): - z = generator.encode_stage_2_inputs(check_data["image"].to(device)) - scale_factor = 1 / torch.std(z) inferer = LatentDiffusionInferer(scheduler, scale_factor=scale_factor) timer.report('building inferer') diff --git a/tv-detection/engine.py b/tv-detection/engine.py index fb03657a..f3409591 100644 --- a/tv-detection/engine.py +++ b/tv-detection/engine.py @@ -18,20 +18,13 @@ def train_one_epoch( ): model.train() - # metric_logger = utils.MetricLogger(delimiter=" ") - # metric_logger.add_meter("lr", utils.SmoothedValue(window_size=1, fmt="{value}")) - # header = f"Epoch: [{epoch}]" timer.report('training preliminaries') - # Running this before starting the training loop assists reporting on progress after resuming - step == batch count - # train_step = train_sampler.progress // args.batch_size print(f'\nTraining / resuming epoch {epoch} from training step {train_sampler.progress}\n') for images, targets in data_loader_train: - # for images, targets in metric_logger.log_every(data_loader_train, train_sampler.progress, args.print_freq, header): ## EDITED THIS - ARGS.BATCH_SIZE == DATALOADER.BATCH_SIZE? GROUPEDBATCHSAMPLER AT PLAY - images = list(image.to(device) for image in images) targets = [{k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in t.items()} for t in targets] timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: moving batch data to device') @@ -70,23 +63,13 @@ def train_one_epoch( train_metrics.reduce() # Gather results from all nodes report_metrics = ["loss", "loss_box_reg", "loss_classifier", "loss_mask", "loss_objectness", "loss_rpn_box_reg", "bbox_regression"] - # norm = train_metrics.local[train_metrics.map["images_seen"]] norm = train_metrics.local["images_seen"] - # vals = [train_metrics.local[train_metrics.map[k]]/norm for k in report_metrics] vals = [train_metrics.local[k]/norm for k in report_metrics] rpt = ", ".join([f"{k}: {v:,.3f}" for k,v in zip(report_metrics, vals)]) print(f"EPOCH: [{epoch}], BATCH: [{train_sampler.progress}/{len(train_sampler)}], "+rpt) train_metrics.reset_local() - # metric_logger.update(loss=losses_reduced, **loss_dict_reduced) - # metric_logger.update(lr=optimizer.param_groups[0]["lr"]) - # timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: updating metric logger') - - # print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") - # train_sampler.advance() # counted in batches, no args to pass - # train_step = train_sampler.progress // data_loader_train.batch_size - print(f"Saving checkpoint at epoch {epoch} train batch {train_sampler.progress}") train_sampler.advance() @@ -120,7 +103,6 @@ def train_one_epoch( timer = atomic_torch_save(checkpoint, args.resume, timer) lr_scheduler.step() # OUTER LR_SCHEDULER STEP EACH EPOCH - # return metric_logger, timer return model, timer, train_metrics def _get_iou_types(model): @@ -148,38 +130,33 @@ def evaluate( torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() - # metric_logger = utils.MetricLogger(delimiter=" ") - # header = "Test:" timer.report(f'evaluation preliminaries') test_step = test_sampler.progress // data_loader_test.batch_size total_steps = len(test_sampler) // data_loader_test.batch_size print(f'\nEvaluating / resuming epoch {epoch} from eval step {test_step}\n') - # timer.report('launch evaluation routine') for images, targets in data_loader_test: - # for images, targets in metric_logger.log_every(data_loader_test, test_sampler.progress, args.print_freq, header): images = list(img.to(device) for img in images) timer.report(f'Epoch {epoch} batch: {test_step} moving to device') if torch.cuda.is_available(): torch.cuda.synchronize() - # model_time = time.time() + outputs = model(images) timer.report(f'Epoch {epoch} batch: {test_step} forward through model') outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] - # model_time = time.time() - model_time + timer.report(f'Epoch {epoch} batch: {test_step} outputs back to cpu') res = {target["image_id"]: output for target, output in zip(targets, outputs)} # res = {img_id: {'boxes': T, 'labels': T, 'scores': T, 'masks': T}, ...} - # evaluator_time = time.time() + coco_evaluator.update(res) - # evaluator_time = time.time() - evaluator_time - # metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) + timer.report(f'Epoch {epoch} batch: {test_step} update evaluator') print(f"Saving checkpoint at epoch {epoch} eval batch {test_step}") @@ -206,8 +183,6 @@ def evaluate( timer = atomic_torch_save(checkpoint, args.resume, timer) # gather the stats from all processes - # metric_logger.synchronize_between_processes() - # print("Averaged stats:", metric_logger) coco_evaluator.synchronize_between_processes() # accumulate predictions from all images diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index 88b9ab1d..15326047 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -23,7 +23,6 @@ timer.report('importing Timer') import os - from pathlib import Path import presets import torch @@ -97,9 +96,6 @@ def main(args, timer): if args.dataset == "coco_kp" and args.use_v2: raise ValueError("KeyPoint detection doesn't support V2 transforms yet") - # if args.output_dir: - # utils.mkdir(args.output_dir) - utils.init_distributed_mode(args) print(args) @@ -232,7 +228,6 @@ def main(args, timer): timer.report('init coco evaluator') - # train_metrics = MetricsTracker(["images_seen", "loss", "loss_box_reg", "loss_classifier", "loss_mask", "loss_objectness", "loss_rpn_box_reg", "bbox_regression"]) train_metrics = MetricsTracker() # RETRIEVE CHECKPOINT @@ -258,7 +253,6 @@ def main(args, timer): coco_evaluator.img_ids = checkpoint["img_ids"] coco_evaluator.eval_imgs = checkpoint["eval_imgs"] train_metrics = checkpoint["train_metrics"] - # train_metrics.to(device) timer.report('retrieving checkpoint') diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index 714212f4..e15ce96e 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -3,8 +3,7 @@ timer = Timer() timer.report('importing Timer') -import os, warnings - +import os from pathlib import Path import presets import torch @@ -63,19 +62,13 @@ def train_one_epoch( ): model.train() - # metric_logger = utils.MetricLogger(delimiter=" ") - # metric_logger.add_meter("lr", utils.SmoothedValue(window_size=1, fmt="{value}")) - # header = f"Epoch: [{epoch}]" - # Running this before starting the training loop assists reporting on progress after resuming - train_step == batch count - # Also means when resuming during evaluation, the training phase is skipped as train_sampler progress == 100%. train_step = train_sampler.progress // data_loader_train.batch_size total_steps = len(train_sampler) // data_loader_train.batch_size print(f'\nTraining / resuming epoch {epoch} from training step {train_step}\n') timer.report('launch training routine') for images, target in data_loader_train: - # for image, target in metric_logger.log_every(data_loader_train, train_step, print_freq, header): images, target = images.to(device), target.to(device) timer.report(f'Epoch: {epoch} batch {train_step}: moving batch data to device') @@ -100,12 +93,10 @@ def train_one_epoch( train_metrics.update({"images_seen": len(images), "loss": loss.item()}) train_metrics.reduce() # Reduce to sync metrics between nodes for this batch - # batch_loss = train_metrics.local[train_metrics.map["loss"]] / train_metrics.local[train_metrics.map["images_seen"]] batch_loss = train_metrics.local["loss"] / train_metrics.local["images_seen"] print(f"EPOCH: [{epoch}], BATCH: [{train_step}/{total_steps}], loss: {batch_loss}") train_metrics.reset_local() - # metric_logger.update(loss=loss.item(), lr=optimizer.param_groups[0]["lr"]) print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") train_sampler.advance(len(images)) @@ -136,7 +127,6 @@ def train_one_epoch( checkpoint["scaler"] = scaler.state_dict() timer = atomic_torch_save(checkpoint, args.resume, timer) - # return metric_logger, timer return model, timer, train_metrics def evaluate( @@ -146,9 +136,6 @@ def evaluate( ): model.eval() - # metric_logger = utils.MetricLogger(delimiter=" ") - # header = "Test:" - # num_processed_samples = 0 test_step = test_sampler.progress // data_loader_test.batch_size total_steps = len(test_sampler) // data_loader_test.batch_size @@ -158,7 +145,6 @@ def evaluate( with torch.inference_mode(): for images, target in data_loader_test: - # for images, target in metric_logger.log_every(data_loader_test, test_step, print_freq, header): images, target = images.to(device), target.to(device) timer.report(f'Epoch {epoch} batch: {test_step} moving to device') @@ -233,9 +219,6 @@ def evaluate( def main(args, timer): - # if args.output_dir: - # utils.mkdir(args.output_dir) - utils.init_distributed_mode(args) print(args) @@ -265,6 +248,7 @@ def main(args, timer): # else: # train_sampler = torch.utils.data.RandomSampler(dataset) # test_sampler = torch.utils.data.SequentialSampler(dataset_test) + train_sampler = InterruptableDistributedSampler(dataset_train) test_sampler = InterruptableDistributedSampler(dataset_test) @@ -343,7 +327,6 @@ def main(args, timer): timer.report('init confmat') # Init general purpose metrics tracker - # train_metrics = MetricsTracker(["images_seen", "loss"]) train_metrics = MetricsTracker() # RETRIEVE CHECKPOINT @@ -369,7 +352,6 @@ def main(args, timer): confmat.mat = checkpoint["confmat"] confmat.temp_mat = checkpoint["confmat_temp"] train_metrics = checkpoint["train_metrics"] - # train_metrics.to(device) timer.report('retrieving checkpoint') From fcd13870db0063b0b2856c85cb6529c24a4621d9 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Wed, 20 Sep 2023 14:28:24 +1000 Subject: [PATCH 33/44] Updates and experiments aligned with literature benchmarks --- cycling_utils/cycling_utils/saving.py | 6 +- monai/brats_mri_2d_v1/brats_mri_2d_diff.isc | 2 +- monai/brats_mri_2d_v1/brats_mri_2d_gen.isc | 2 +- monai/brats_mri_2d_v1/loops.py | 149 ++++++++---------- monai/brats_mri_2d_v1/train_cycling_diff.py | 4 - monai/brats_mri_2d_v1/train_cycling_gen.py | 4 - .../{ => SS}/maskrcnn_resnet50_fpn.isc | 0 .../{ => SS}/retinanet_resnet50_fpn.isc | 0 tv-detection/engine.py | 48 +++--- tv-detection/maskrcnn_resnet101_fpn.isc | 6 + tv-detection/prep.py | 6 +- tv-detection/retinanet_resnet101_fpn.isc | 7 + tv-detection/train_cycling.py | 40 +++-- tv-segmentation/train_cycling.py | 36 +++-- 14 files changed, 160 insertions(+), 150 deletions(-) rename tv-detection/{ => SS}/maskrcnn_resnet50_fpn.isc (100%) rename tv-detection/{ => SS}/retinanet_resnet50_fpn.isc (100%) create mode 100644 tv-detection/maskrcnn_resnet101_fpn.isc create mode 100644 tv-detection/retinanet_resnet101_fpn.isc diff --git a/cycling_utils/cycling_utils/saving.py b/cycling_utils/cycling_utils/saving.py index c29d57d7..ddfda40d 100644 --- a/cycling_utils/cycling_utils/saving.py +++ b/cycling_utils/cycling_utils/saving.py @@ -24,8 +24,8 @@ def __init__(self): self.epoch_reports = [] def update(self, metrics: dict): - for n,v in metrics.items(): - self.local[n] += v + for m,v in metrics.items(): + self.local[m] += v def reduce(self): names, local = zip(*self.local.items()) @@ -39,7 +39,7 @@ def reset_local(self): self.local = defaultdict(float) def end_epoch(self): - self.epoch_reports.append(self.agg) + self.epoch_reports.append(dict(self.agg)) self.local = defaultdict(float) self.agg = defaultdict(float) diff --git a/monai/brats_mri_2d_v1/brats_mri_2d_diff.isc b/monai/brats_mri_2d_v1/brats_mri_2d_diff.isc index 39578a95..9b90b185 100644 --- a/monai/brats_mri_2d_v1/brats_mri_2d_diff.isc +++ b/monai/brats_mri_2d_v1/brats_mri_2d_diff.isc @@ -1,6 +1,6 @@ experiment_name="brats_mri_2d_diff" gpu_type="24GB VRAM GPU" -nnodes = 9 +nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_brats_mri_2d_diff" command="train_cycling_diff.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc, --gen-load-path ~/output_brats_mri_2d_gen/exp_1645/checkpoint.isc --tboard-path ~/output_brats_mri_2d_diff/tb" \ No newline at end of file diff --git a/monai/brats_mri_2d_v1/brats_mri_2d_gen.isc b/monai/brats_mri_2d_v1/brats_mri_2d_gen.isc index ca081d87..df84c2f3 100644 --- a/monai/brats_mri_2d_v1/brats_mri_2d_gen.isc +++ b/monai/brats_mri_2d_v1/brats_mri_2d_gen.isc @@ -1,6 +1,6 @@ experiment_name="brats_mri_2d_gen" gpu_type="24GB VRAM GPU" -nnodes = 9 +nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_brats_mri_2d_gen" command="train_cycling_gen.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_brats_mri_2d_gen/tb" \ No newline at end of file diff --git a/monai/brats_mri_2d_v1/loops.py b/monai/brats_mri_2d_v1/loops.py index 467e7c1f..99a2c4d6 100644 --- a/monai/brats_mri_2d_v1/loops.py +++ b/monai/brats_mri_2d_v1/loops.py @@ -1,40 +1,45 @@ -import torch, os +import torch from torch.cuda.amp import autocast import torch.nn.functional as F import utils from cycling_utils import atomic_torch_save +from generative.losses.adversarial_loss import PatchAdversarialLoss + from torch.utils.tensorboard import SummaryWriter -tb_path = "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/output_brats_mri_2d_gen/tb" ## -- AUTO-ENCODER - ## -# def compute_kl_loss(z_mu, z_sigma): -# kl_loss = 0.5 * torch.sum( -# z_mu.pow(2) + z_sigma.pow(2) - torch.log(z_sigma.pow(2)) - 1, dim=list(range(1, len(z_sigma.shape))) -# ) -# return torch.sum(kl_loss) / kl_loss.shape[0] - -# def generator_loss(gen_images, real_images, z_mu, z_sigma, disc_net, loss_perceptual): -# recons_loss = intensity_loss(gen_images, real_images) -# kl_loss = compute_kl_loss(z_mu, z_sigma) -# p_loss = loss_perceptual(gen_images.float(), real_images.float()) -# loss_g = recons_loss + kl_weight * kl_loss + perceptual_weight * p_loss - -# logits_fake = disc_net(gen_images)[-1] -# generator_loss = adv_loss(logits_fake, target_is_real=True, for_discriminator=False) -# loss_g = loss_g + adv_weight * generator_loss - -# return loss_g - -# def discriminator_loss(gen_images, real_images, disc_net): -# logits_fake = disc_net(gen_images.contiguous().detach())[-1] -# loss_d_fake = adv_loss(logits_fake, target_is_real=False, for_discriminator=True) -# logits_real = disc_net(real_images.contiguous().detach())[-1] -# loss_d_real = adv_loss(logits_real, target_is_real=True, for_discriminator=True) -# discriminator_loss = (loss_d_fake + loss_d_real) * 0.5 -# loss_d = adv_weight * discriminator_loss -# return loss_d +def compute_kl_loss(z_mu, z_sigma): + kl_loss = 0.5 * torch.sum( + z_mu.pow(2) + z_sigma.pow(2) - torch.log(z_sigma.pow(2)) - 1, + dim=list(range(1, len(z_sigma.shape))) + ) + return torch.sum(kl_loss) / kl_loss.shape[0] + +intensity_loss = torch.nn.L1Loss() +adv_loss = PatchAdversarialLoss(criterion="least_squares") + +def generator_loss(gen_images, real_images, z_mu, z_sigma, disc_net, perceptual_loss, kl_weight, perceptual_weight, adv_weight): + # Image intrinsic qualities + recons_loss = intensity_loss(gen_images, real_images) + kl_loss = compute_kl_loss(z_mu, z_sigma) + p_loss = perceptual_loss(gen_images.float(), real_images.float()) + loss_g = recons_loss + (kl_weight * kl_loss) + (perceptual_weight * p_loss) + # Discrimnator-based loss + logits_fake = disc_net(gen_images)[-1] + generator_loss = adv_loss(logits_fake, target_is_real=True, for_discriminator=False) + loss_g = loss_g + (adv_weight * generator_loss) + return loss_g + +def discriminator_loss(gen_images, real_images, disc_net, adv_weight): + logits_fake = disc_net(gen_images.contiguous().detach())[-1] + loss_d_fake = adv_loss(logits_fake, target_is_real=False, for_discriminator=True) + logits_real = disc_net(real_images.contiguous().detach())[-1] + loss_d_real = adv_loss(logits_real, target_is_real=True, for_discriminator=True) + discriminator_loss = (loss_d_fake + loss_d_real) * 0.5 + loss_d = adv_weight * discriminator_loss + return loss_d def train_generator_one_epoch( @@ -47,8 +52,8 @@ def train_generator_one_epoch( kl_weight = 1e-6 perceptual_weight = 1.0 adv_weight = 0.5 - # Retained from tutorial - generator_warm_up_n_epochs = 10 + # From tutorial ? + # generator_warm_up_n_epochs = 10 generator.train() discriminator.train() @@ -70,24 +75,12 @@ def train_generator_one_epoch( reconstruction, z_mu, z_sigma = generator(images) timer.report(f'train batch {train_step} generator forward') - recons_loss = F.l1_loss(reconstruction.float(), images.float()) - timer.report(f'train batch {train_step} recons_loss') - p_loss = perceptual_loss(reconstruction.float(), images.float()) - timer.report(f'train batch {train_step} p_loss') - kl_loss = 0.5 * torch.sum(z_mu.pow(2) + z_sigma.pow(2) - torch.log(z_sigma.pow(2)) - 1, dim=[1, 2, 3]) - kl_loss = torch.sum(kl_loss) / kl_loss.shape[0] - timer.report(f'train batch {train_step} kl_loss') - loss_g = recons_loss + (kl_weight * kl_loss) + (perceptual_weight * p_loss) - timer.report(f'train batch {train_step} loss_g (1)') - - if epoch > generator_warm_up_n_epochs: # Train generator for n epochs on reconstruction, KL, and perceptual loss before introducing discriminator loss - - logits_fake = discriminator(reconstruction.contiguous().float())[-1] - timer.report(f'train batch {train_step} logits_fake from discriminator') - generator_loss = adv_loss(logits_fake, target_is_real=True, for_discriminator=False) - timer.report(f'train batch {train_step} generator_loss (adv_loss)') - loss_g += adv_weight * generator_loss - timer.report(f'train batch {train_step} loss_g (2)') + + loss_g = generator_loss( + reconstruction, images, z_mu, z_sigma, discriminator, perceptual_loss, + kl_weight, perceptual_weight, adv_weight + ) + timer.report(f'train batch {train_step} generator loss: {loss_g.item():.3f}') scaler_g.scale(loss_g).backward() scaler_g.step(optimizer_g) @@ -96,48 +89,32 @@ def train_generator_one_epoch( # TRAIN DISCRIMINATOR - if epoch > generator_warm_up_n_epochs: # Train generator for n epochs before starting discriminator training + optimizer_d.zero_grad(set_to_none=True) - optimizer_d.zero_grad(set_to_none=True) + with autocast(enabled=True): - with autocast(enabled=True): + loss_d = discriminator_loss( + reconstruction, images, discriminator, adv_weight + ) + timer.report(f'train batch {train_step} discriminator loss {loss_d.item():.3f}') - logits_fake = discriminator(reconstruction.contiguous().detach())[-1] - timer.report(f'train batch {train_step} discriminator forward (fake)') - loss_d_fake = adv_loss(logits_fake, target_is_real=False, for_discriminator=True) - timer.report(f'train batch {train_step} loss_d_fake') - logits_real = discriminator(images.contiguous().detach())[-1] - timer.report(f'train batch {train_step} discriminator forward (real)') - loss_d_real = adv_loss(logits_real, target_is_real=True, for_discriminator=True) - timer.report(f'train batch {train_step} loss_d_real') - discriminator_loss = (loss_d_fake + loss_d_real) * 0.5 - timer.report(f'train batch {train_step} discriminator_loss') - loss_d = adv_weight * discriminator_loss - timer.report(f'train batch {train_step} loss_d') - - scaler_d.scale(loss_d).backward() - scaler_d.step(optimizer_d) - scaler_d.update() - timer.report(f'train batch {train_step} discriminator backward') + scaler_d.scale(loss_d).backward() + scaler_d.step(optimizer_d) + scaler_d.update() + timer.report(f'train batch {train_step} discriminator backward') # Reduce metrics accross nodes - metrics["train"].update({"train_images_seen":len(images), "epoch_loss":recons_loss.item()}) - if epoch > generator_warm_up_n_epochs: - metrics["train"].update({"gen_epoch_loss":generator_loss.item(), "disc_epoch_loss":discriminator_loss.item()}) + metrics["train"].update({"train_images_seen":len(images), "loss_g":loss_g.item(), "loss_d": loss_d.item()}) metrics["train"].reduce() - timer.report(f'train batch {train_step} metrics update') - - # recons_loss = metrics["train"].local[metrics["train"].map["epoch_loss"]] / metrics["train"].local[metrics["train"].map["train_images_seen"]] - # gen_loss = metrics["train"].local[metrics["train"].map["gen_epoch_loss"]] / metrics["train"].local[metrics["train"].map["train_images_seen"]] - # disc_loss = metrics["train"].local[metrics["train"].map["disc_epoch_loss"]] / metrics["train"].local[metrics["train"].map["train_images_seen"]] - recons_loss = metrics["train"].local["epoch_loss"] / metrics["train"].local["train_images_seen"] - gen_loss = metrics["train"].local["gen_epoch_loss"] / metrics["train"].local["train_images_seen"] - disc_loss = metrics["train"].local["disc_epoch_loss"] / metrics["train"].local["train_images_seen"] - print("Epoch [{}] Step [{}/{}] :: recons_loss: {:,.3f}, gen_loss: {:,.3f}, disc_loss: {:,.3f}".format(epoch, train_step, total_steps, recons_loss, gen_loss, disc_loss)) + gen_loss = metrics["train"].local["loss_g"] / metrics["train"].local["train_images_seen"] + disc_loss = metrics["train"].local["loss_d"] / metrics["train"].local["train_images_seen"] + print("Epoch [{}] Step [{}/{}], gen_loss: {:.3f}, disc_loss: {:.3f}".format(epoch, train_step, total_steps, gen_loss, disc_loss)) metrics["train"].reset_local() + timer.report(f'train batch {train_step} metrics update') + ## Checkpointing print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") train_sampler.advance(len(images)) @@ -148,7 +125,6 @@ def train_generator_one_epoch( if utils.is_main_process() and train_step % 1 == 0: # Checkpointing every batch writer = SummaryWriter(log_dir=args.tboard_path) - writer.add_scalar("Train/recons_loss", recons_loss, train_step + epoch * total_steps) writer.add_scalar("Train/gen_loss", gen_loss, train_step + epoch * total_steps) writer.add_scalar("Train/disc_loss", disc_loss, train_step + epoch * total_steps) writer.flush() @@ -171,6 +147,9 @@ def train_generator_one_epoch( } timer = atomic_torch_save(checkpoint, args.resume, timer) + gen_loss = metrics["train"].epoch_reports[-1]["loss_g"] / metrics["train"].epoch_reports[-1]["train_images_seen"] + disc_loss = metrics["train"].epoch_reports[-1]["loss_d"] / metrics["train"].epoch_reports[-1]["train_images_seen"] + print("Epoch [{}] :: gen_loss: {:,.3f}, disc_loss: {:,.3f}".format(epoch, gen_loss, disc_loss)) return generator, timer, metrics @@ -211,7 +190,6 @@ def evaluate_generator( val_step = val_sampler.progress // val_loader.batch_size if val_step == total_steps: - # val_loss = metrics["val"].agg[metrics["val"].map["val_loss"]] / metrics["val"].agg[metrics["val"].map["val_images_seen"]] val_loss = metrics["val"].agg["val_loss"] / metrics["val"].agg["val_images_seen"] if utils.is_main_process(): writer = SummaryWriter(log_dir=args.tboard_path) @@ -289,7 +267,6 @@ def train_diffusion_one_epoch( metrics["train"].update({"train_images_seen":len(images), "epoch_loss":loss.item()}) metrics["train"].reduce() - # recons_loss = metrics["train"].local[metrics["train"].map["epoch_loss"]] / metrics["train"].local[metrics["train"].map["train_images_seen"]] recons_loss = metrics["train"].local["epoch_loss"] / metrics["train"].local["train_images_seen"] print("Epoch [{}] Step [{}/{}] :: recons_loss: {:,.3f}".format(epoch, train_step, total_steps, recons_loss)) @@ -328,6 +305,8 @@ def train_diffusion_one_epoch( } timer = atomic_torch_save(checkpoint, args.resume, timer) + train_loss = metrics["train"].epoch_reports[-1]["epoch_loss"] / metrics["train"].epoch_reports[-1]["train_images_seen"] + print("Epoch [{}] :: epoch_loss: {:,.3f}".format(epoch, train_loss)) return unet, timer, metrics @@ -395,12 +374,12 @@ def evaluate_diffusion( timer = atomic_torch_save(checkpoint, args.resume, timer) # val_loss = metrics["val"].agg[metrics["val"].map["val_loss"]] / metrics["val"].agg[metrics["val"].map["val_images_seen"]] - val_loss = metrics["val"].agg["val_loss"] / metrics["val"].agg["val_images_seen"] + val_loss = metrics["val"].epoch_reports[-1]["val_loss"] / metrics["val"].epoch_reports[-1]["val_images_seen"] if utils.is_main_process(): writer = SummaryWriter(log_dir=args.tboard_path) writer.add_scalar("Val/loss", val_loss, epoch) writer.flush() writer.close() - print(f"Epoch {epoch} diff val loss: {val_loss:.4f}") + print(f"Epoch [{epoch}] :: diff val loss: {val_loss:.4f}") return timer, metrics diff --git a/monai/brats_mri_2d_v1/train_cycling_diff.py b/monai/brats_mri_2d_v1/train_cycling_diff.py index 93a1747c..7d57f684 100644 --- a/monai/brats_mri_2d_v1/train_cycling_diff.py +++ b/monai/brats_mri_2d_v1/train_cycling_diff.py @@ -150,8 +150,6 @@ def main(args, timer): timer.report('optimizer, lr_scheduler and grad scaler') # Init metric tracker - # train_metrics = MetricsTracker(["train_images_seen", "epoch_loss"]) - # val_metrics = MetricsTracker(["val_images_seen", "val_loss"]) metrics = {'train': MetricsTracker(), 'val': MetricsTracker()} # RETRIEVE GENERATOR CHECKPOINT FROM PREVIOUS JOB @@ -173,8 +171,6 @@ def main(args, timer): lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) # Metrics metrics = checkpoint["metrics"] - # metrics["train"].to(device) - # metrics["val"].to(device) timer.report('checkpoint retrieval') diff --git a/monai/brats_mri_2d_v1/train_cycling_gen.py b/monai/brats_mri_2d_v1/train_cycling_gen.py index 45ceefc7..2a55c1f6 100644 --- a/monai/brats_mri_2d_v1/train_cycling_gen.py +++ b/monai/brats_mri_2d_v1/train_cycling_gen.py @@ -148,8 +148,6 @@ def main(args, timer): timer.report('grad scalers') # Init metric tracker - # train_metrics = MetricsTracker(["train_images_seen", "epoch_loss", "gen_epoch_loss", "disc_epoch_loss"]) - # val_metrics = MetricsTracker(["val_images_seen", "val_loss"]) metrics = {'train': MetricsTracker(), 'val': MetricsTracker()} # RETRIEVE CHECKPOINT @@ -171,8 +169,6 @@ def main(args, timer): val_sampler.load_state_dict(checkpoint["val_sampler"]) # Metrics metrics = checkpoint["metrics"] - # metrics["train"].to(device) - # metrics["val"].to(device) timer.report('checkpoint retrieval') diff --git a/tv-detection/maskrcnn_resnet50_fpn.isc b/tv-detection/SS/maskrcnn_resnet50_fpn.isc similarity index 100% rename from tv-detection/maskrcnn_resnet50_fpn.isc rename to tv-detection/SS/maskrcnn_resnet50_fpn.isc diff --git a/tv-detection/retinanet_resnet50_fpn.isc b/tv-detection/SS/retinanet_resnet50_fpn.isc similarity index 100% rename from tv-detection/retinanet_resnet50_fpn.isc rename to tv-detection/SS/retinanet_resnet50_fpn.isc diff --git a/tv-detection/engine.py b/tv-detection/engine.py index f3409591..dca3fa14 100644 --- a/tv-detection/engine.py +++ b/tv-detection/engine.py @@ -1,6 +1,6 @@ import math import sys -import time +from itertools import product import torch import torchvision.models.detection.mask_rcnn @@ -14,7 +14,7 @@ def train_one_epoch( model, optimizer, data_loader_train, train_sampler, test_sampler, lr_scheduler, warmup_lr_scheduler, args, device, coco_evaluator, - epoch, scaler=None, timer=None, train_metrics=None, + epoch, scaler=None, timer=None, metrics=None, ): model.train() @@ -28,6 +28,7 @@ def train_one_epoch( images = list(image.to(device) for image in images) targets = [{k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in t.items()} for t in targets] timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: moving batch data to device') + print(f"First 2 image shapes: {images[0].shape}, {images[1].shape}") optimizer.zero_grad() @@ -58,23 +59,23 @@ def train_one_epoch( print(loss_dict_reduced) sys.exit(1) - train_metrics.update({"images_seen": len(images) ,"loss": loss_value}) - train_metrics.update({k:v.item() for k,v in loss_dict_reduced.items()}) - train_metrics.reduce() # Gather results from all nodes + metrics["train"].update({"images_seen": len(images) ,"loss": loss_value}) + metrics["train"].update({k:v.item() for k,v in loss_dict_reduced.items()}) + metrics["train"].reduce() # Gather results from all nodes - report_metrics = ["loss", "loss_box_reg", "loss_classifier", "loss_mask", "loss_objectness", "loss_rpn_box_reg", "bbox_regression"] - norm = train_metrics.local["images_seen"] - vals = [train_metrics.local[k]/norm for k in report_metrics] - rpt = ", ".join([f"{k}: {v:,.3f}" for k,v in zip(report_metrics, vals)]) + report_metrics = [m for m in metrics["train"].local if m != "images_seen"] + images_seen = metrics["train"].local["images_seen"] + vals = [metrics["train"].local[m]/images_seen for m in report_metrics] + rpt = ", ".join([f"{m}: {v:,.3f}" for m,v in zip(report_metrics, vals)]) print(f"EPOCH: [{epoch}], BATCH: [{train_sampler.progress}/{len(train_sampler)}], "+rpt) - train_metrics.reset_local() + metrics["train"].reset_local() print(f"Saving checkpoint at epoch {epoch} train batch {train_sampler.progress}") train_sampler.advance() if train_sampler.progress == len(train_sampler): - train_metrics.end_epoch() + metrics["train"].end_epoch() if utils.is_main_process() and train_sampler.progress % 1 == 0: # Checkpointing every batch @@ -96,14 +97,14 @@ def train_one_epoch( # Evaluator state variables "img_ids": coco_evaluator.img_ids, # catalogue of images seen already "eval_imgs": coco_evaluator.eval_imgs, # image evaluations - "train_metrics": train_metrics, + "metrics": metrics, } if args.amp: checkpoint["scaler"] = scaler.state_dict() timer = atomic_torch_save(checkpoint, args.resume, timer) lr_scheduler.step() # OUTER LR_SCHEDULER STEP EACH EPOCH - return model, timer, train_metrics + return model, timer, metrics def _get_iou_types(model): model_without_ddp = model @@ -120,7 +121,7 @@ def _get_iou_types(model): def evaluate( model, data_loader_test, epoch, test_sampler, args, coco_evaluator, optimizer, lr_scheduler, warmup_lr_scheduler, train_sampler, - device, scaler=None, timer=None, train_metrics=None, + device, scaler=None, timer=None, metrics=None, ): timer.report('starting evaluation routine') @@ -149,14 +150,11 @@ def evaluate( timer.report(f'Epoch {epoch} batch: {test_step} forward through model') outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] - timer.report(f'Epoch {epoch} batch: {test_step} outputs back to cpu') res = {target["image_id"]: output for target, output in zip(targets, outputs)} # res = {img_id: {'boxes': T, 'labels': T, 'scores': T, 'masks': T}, ...} - coco_evaluator.update(res) - timer.report(f'Epoch {epoch} batch: {test_step} update evaluator') print(f"Saving checkpoint at epoch {epoch} eval batch {test_step}") @@ -176,7 +174,7 @@ def evaluate( # Evaluator state variables "img_ids": coco_evaluator.img_ids, # catalogue of images seen already "eval_imgs": coco_evaluator.eval_imgs, # image evaluations - "train_metrics": train_metrics, + "metrics": metrics, } if args.amp: checkpoint["scaler"] = scaler.state_dict() @@ -189,10 +187,18 @@ def evaluate( coco_evaluator.accumulate() results = coco_evaluator.summarize() + metric_A = ["bbox-", "segm-"] + metric_B = ["AP", "AR"] + metric_C = ["", "50", "75", "-S", "-M", "-L"] + metric_names = ["".join(t) for t in product(metric_A, metric_B, metric_C)] + metrics["val"].update({name: val for name,val in zip(metric_names, results)}) + metrics["val"].reduce() + metrics["val"].end_epoch() + if utils.is_main_process(): writer = SummaryWriter(log_dir=args.tboard_path) - for i,val in enumerate(results): - writer.add_scalar(f"Eval/F{i}", val, test_step + epoch * total_steps) + for name,val in metrics["val"].epoch_reports[-1]: + writer.add_scalar(name, val, epoch) writer.flush() writer.close() @@ -205,4 +211,4 @@ def evaluate( timer.report(f'evaluator accumulation, summarization, and reset') - return coco_evaluator, timer, train_metrics + return coco_evaluator, timer, metrics diff --git a/tv-detection/maskrcnn_resnet101_fpn.isc b/tv-detection/maskrcnn_resnet101_fpn.isc new file mode 100644 index 00000000..adcb255b --- /dev/null +++ b/tv-detection/maskrcnn_resnet101_fpn.isc @@ -0,0 +1,6 @@ +experiment_name="maskrcnn_resnet101_fpn" +gpu_type="24GB VRAM GPU" +nnodes = 11 +venv_path = "~/.venv/bin/activate" +output_path = "~/output_maskrcnn_resnet101_fpn" +command="train_cycling.py --dataset coco --model maskrcnn_resnet101_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_maskrcnn_resnet101_fpn/tb" \ No newline at end of file diff --git a/tv-detection/prep.py b/tv-detection/prep.py index e123c416..4a58ea61 100644 --- a/tv-detection/prep.py +++ b/tv-detection/prep.py @@ -3,4 +3,8 @@ # weights = torchvision.models.get_weight('ResNet50_Weights.IMAGENET1K_V1') _ = resnet50(weights=ResNet50_Weights.IMAGENET1K_V1) -# weights = torchvision.models.get_weight(...) \ No newline at end of file +# weights = torchvision.models.get_weight(...) + +from torchvision.models.detection import MaskRCNN +from torchvision.models.detection.backbone_utils import resnet_fpn_backbone +_ = resnet_fpn_backbone(backbone_name="resnet101", weights="ResNet101_Weights.IMAGENET1K_V1") \ No newline at end of file diff --git a/tv-detection/retinanet_resnet101_fpn.isc b/tv-detection/retinanet_resnet101_fpn.isc new file mode 100644 index 00000000..a1d3f65d --- /dev/null +++ b/tv-detection/retinanet_resnet101_fpn.isc @@ -0,0 +1,7 @@ +experiment_name="retinanet_resnet101_fpn" +gpu_type="24GB VRAM GPU" +nnodes = 11 +venv_path = "~/.venv/bin/activate" +output_path = "~/output_retinanet_resnet101_fpn" +command="train_cycling.py --dataset coco --model retinanet_resnet101_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --lr 0.01 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_retinanet_resnet101_fpn/tb" + diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index 15326047..74dabf92 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -33,6 +33,8 @@ import torchvision.models.detection import torchvision.models.detection.mask_rcnn +from torchvision.models.detection import MaskRCNN, RetinaNet +from torchvision.models.detection.backbone_utils import resnet_fpn_backbone from engine import evaluate, train_one_epoch from group_by_aspect_ratio import create_aspect_ratio_groups from torchvision.transforms import InterpolationMode @@ -158,9 +160,15 @@ def main(args, timer): if args.rpn_score_thresh is not None: kwargs["rpn_score_thresh"] = args.rpn_score_thresh - model = torchvision.models.get_model( - args.model, weights=args.weights, weights_backbone=args.weights_backbone, num_classes=num_classes, **kwargs - ) + if args.model == "maskrcnn_resnet101_fpn": + backbone = resnet_fpn_backbone(backbone_name="resnet101", weights="ResNet101_Weights.IMAGENET1K_V1") + model = MaskRCNN(backbone=backbone, num_classes=num_classes) + elif args.model == "retinanet_resnet101_fpn": + backbone = resnet_fpn_backbone(backbone_name="resnet101", weights="ResNet101_Weights.IMAGENET1K_V1") + model = RetinaNet(backbone=backbone, num_classes=num_classes) + # model = torchvision.models.get_model( + # args.model, weights=args.weights, weights_backbone=args.weights_backbone, num_classes=num_classes, **kwargs + # ) model.to(device) timer.report('creating model and .to(device)') @@ -228,7 +236,7 @@ def main(args, timer): timer.report('init coco evaluator') - train_metrics = MetricsTracker() + metrics = {"train": MetricsTracker(), "val": MetricsTracker()} # RETRIEVE CHECKPOINT Path(args.resume).parent.mkdir(parents=True, exist_ok=True) @@ -252,16 +260,16 @@ def main(args, timer): # Evaluator state variables coco_evaluator.img_ids = checkpoint["img_ids"] coco_evaluator.eval_imgs = checkpoint["eval_imgs"] - train_metrics = checkpoint["train_metrics"] + metrics = checkpoint["metrics"] timer.report('retrieving checkpoint') - if args.test_only: - # We disable the cudnn benchmarking because it can noticeably affect the accuracy - torch.backends.cudnn.benchmark = False - torch.backends.cudnn.deterministic = True - coco_evaluator, timer = evaluate(model, data_loader_test, device, timer) - return + # if args.test_only: + # # We disable the cudnn benchmarking because it can noticeably affect the accuracy + # torch.backends.cudnn.benchmark = False + # torch.backends.cudnn.deterministic = True + # coco_evaluator, timer = evaluate(model, data_loader_test, device, timer) + # return for epoch in range(args.start_epoch, args.epochs): @@ -271,12 +279,18 @@ def main(args, timer): with train_sampler.in_epoch(epoch): timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch - model, timer, train_metrics = train_one_epoch(model, optimizer, data_loader_train, train_sampler, test_sampler, lr_scheduler, warmup_lr_scheduler, args, device, coco_evaluator, epoch, scaler, timer, train_metrics) + model, timer, metrics = train_one_epoch( + model, optimizer, data_loader_train, train_sampler, test_sampler, lr_scheduler, warmup_lr_scheduler, + args, device, coco_evaluator, epoch, scaler, timer, metrics + ) # NEST THE TEST SAMPLER IN TRAIN SAMPLER CONTEXT TO AVOID EPOCH RESTART? with test_sampler.in_epoch(epoch): timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch - coco_evaluator, timer, train_metrics = evaluate(model, data_loader_test, epoch, test_sampler, args, coco_evaluator, optimizer, lr_scheduler, warmup_lr_scheduler, train_sampler, device, scaler, timer, train_metrics) + coco_evaluator, timer, metrics = evaluate( + model, data_loader_test, epoch, test_sampler, args, coco_evaluator, optimizer, lr_scheduler, warmup_lr_scheduler, + train_sampler, device, scaler, timer, metrics + ) def get_args_parser(add_help=True): diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index e15ce96e..558c483d 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -58,7 +58,7 @@ def criterion(inputs, target): def train_one_epoch( args, model, criterion, optimizer, data_loader_train, train_sampler, test_sampler, confmat, lr_scheduler, - device, epoch, scaler=None, timer=None, train_metrics=None + device, epoch, scaler=None, timer=None, metrics=None ): model.train() @@ -91,18 +91,18 @@ def train_one_epoch( timer.report(f'Epoch: {epoch} batch {train_step}: backward pass') - train_metrics.update({"images_seen": len(images), "loss": loss.item()}) - train_metrics.reduce() # Reduce to sync metrics between nodes for this batch - batch_loss = train_metrics.local["loss"] / train_metrics.local["images_seen"] + metrics["train"].update({"images_seen": len(images), "loss": loss.item()}) + metrics["train"].reduce() # Reduce to sync metrics between nodes for this batch + batch_loss = metrics["train"].local["loss"] / metrics["train"].local["images_seen"] print(f"EPOCH: [{epoch}], BATCH: [{train_step}/{total_steps}], loss: {batch_loss}") - train_metrics.reset_local() + metrics["train"].reset_local() print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") train_sampler.advance(len(images)) train_step = train_sampler.progress // data_loader_train.batch_size if train_step == total_steps: - train_metrics.end_epoch() + metrics["train"].end_epoch() if utils.is_main_process() and train_step % 1 == 0: # Checkpointing every batch @@ -121,18 +121,18 @@ def train_one_epoch( "test_sampler": test_sampler.state_dict(), "confmat": confmat.mat, "confmat_temp": confmat.temp_mat, - "train_metrics": train_metrics, + "metrics": metrics, } if args.amp: checkpoint["scaler"] = scaler.state_dict() timer = atomic_torch_save(checkpoint, args.resume, timer) - return model, timer, train_metrics + return model, timer, metrics def evaluate( args, model, data_loader_test, num_classes, confmat, optimizer, lr_scheduler, train_sampler, test_sampler, - device, epoch, scaler=None, timer=None, train_metrics=None, + device, epoch, scaler=None, timer=None, metrics=None, ): model.eval() @@ -186,6 +186,9 @@ def evaluate( acc_global, acc, iu = confmat.compute() acc = acc_global.item() * 100 mean_iou = iu.mean().item() * 100 + metrics["val"].update({"acc": acc, "mean_iou": mean_iou}) + metrics["val"].reduce() + metrics["val"].reset_local() print(f"EPOCH: [{epoch}] EVAL :: acc: {acc:.2f}, mean_iou: {mean_iou:.2f}") confmat.reset() @@ -211,12 +214,11 @@ def evaluate( # "Setting the world size to 1 is always a safe bet." # ) - return confmat, timer + return confmat, timer, metrics timer.report('defined other functions') - def main(args, timer): utils.init_distributed_mode(args) @@ -327,7 +329,7 @@ def main(args, timer): timer.report('init confmat') # Init general purpose metrics tracker - train_metrics = MetricsTracker() + metrics = {"train": MetricsTracker(), "val": MetricsTracker()} # RETRIEVE CHECKPOINT Path(args.resume).parent.mkdir(parents=True, exist_ok=True) @@ -351,7 +353,7 @@ def main(args, timer): scaler.load_state_dict(checkpoint["scaler"]) confmat.mat = checkpoint["confmat"] confmat.temp_mat = checkpoint["confmat_temp"] - train_metrics = checkpoint["train_metrics"] + metrics = checkpoint["metrics"] timer.report('retrieving checkpoint') @@ -371,20 +373,20 @@ def main(args, timer): with train_sampler.in_epoch(epoch): timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch - model, timer, train_metrics = train_one_epoch( + model, timer, metrics = train_one_epoch( args, model, criterion, optimizer, data_loader_train, train_sampler, test_sampler, confmat, lr_scheduler, - device, epoch, scaler, timer, train_metrics + device, epoch, scaler, timer, metrics ) timer.report(f'training for epoch {epoch}') # NEST TEST SAMPLER IN TRAIN SAMPLER CONTEXT TO AVOID EPOCH RESTART? with test_sampler.in_epoch(epoch): timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch - confmat, timer = evaluate( + confmat, timer, metrics = evaluate( args, model, data_loader_test, num_classes, confmat, optimizer, lr_scheduler, train_sampler, test_sampler, - device, epoch, scaler, timer, train_metrics, + device, epoch, scaler, timer, metrics, ) timer.report(f'evaluation for epoch {epoch}') From 953cdc4a44de1f4310193c828ffcafc23ae9d55a Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Thu, 21 Sep 2023 08:49:36 +1000 Subject: [PATCH 34/44] Fixed reporting issue with mask/retina --- tv-detection/engine.py | 4 ++-- tv-detection/maskrcnn_resnet101_fpn.isc | 6 +++--- tv-detection/retinanet_resnet50_fpn.isc | 7 +++++++ tv-detection/train_cycling.py | 3 +++ 4 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 tv-detection/retinanet_resnet50_fpn.isc diff --git a/tv-detection/engine.py b/tv-detection/engine.py index dca3fa14..625b8b1e 100644 --- a/tv-detection/engine.py +++ b/tv-detection/engine.py @@ -197,8 +197,8 @@ def evaluate( if utils.is_main_process(): writer = SummaryWriter(log_dir=args.tboard_path) - for name,val in metrics["val"].epoch_reports[-1]: - writer.add_scalar(name, val, epoch) + for name,val in metrics["val"].epoch_reports[-1].items(): + writer.add_scalar("Val/"+name, val, epoch) writer.flush() writer.close() diff --git a/tv-detection/maskrcnn_resnet101_fpn.isc b/tv-detection/maskrcnn_resnet101_fpn.isc index adcb255b..ad665820 100644 --- a/tv-detection/maskrcnn_resnet101_fpn.isc +++ b/tv-detection/maskrcnn_resnet101_fpn.isc @@ -1,6 +1,6 @@ -experiment_name="maskrcnn_resnet101_fpn" -gpu_type="24GB VRAM GPU" +experiment_name = "maskrcnn_resnet101_fpn" +gpu_type = "24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_maskrcnn_resnet101_fpn" -command="train_cycling.py --dataset coco --model maskrcnn_resnet101_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_maskrcnn_resnet101_fpn/tb" \ No newline at end of file +command = "train_cycling.py --dataset coco --model maskrcnn_resnet101_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_maskrcnn_resnet101_fpn/tb" \ No newline at end of file diff --git a/tv-detection/retinanet_resnet50_fpn.isc b/tv-detection/retinanet_resnet50_fpn.isc new file mode 100644 index 00000000..c1165d81 --- /dev/null +++ b/tv-detection/retinanet_resnet50_fpn.isc @@ -0,0 +1,7 @@ +experiment_name="retinanet_resnet50_fpn" +gpu_type="24GB VRAM GPU" +nnodes = 11 +venv_path = "~/.venv/bin/activate" +output_path = "~/output_retinanet_resnet50_fpn" +command="train_cycling.py --dataset coco --model retinanet_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --lr 0.01 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_retinanet_resnet50_fpn/tb" + diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index 74dabf92..181cc80e 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -163,6 +163,9 @@ def main(args, timer): if args.model == "maskrcnn_resnet101_fpn": backbone = resnet_fpn_backbone(backbone_name="resnet101", weights="ResNet101_Weights.IMAGENET1K_V1") model = MaskRCNN(backbone=backbone, num_classes=num_classes) + elif args.model == "retinanet_resnet50_fpn": + backbone = resnet_fpn_backbone(backbone_name="resnet50", weights="ResNet50_Weights.IMAGENET1K_V1") + model = RetinaNet(backbone=backbone, num_classes=num_classes) elif args.model == "retinanet_resnet101_fpn": backbone = resnet_fpn_backbone(backbone_name="resnet101", weights="ResNet101_Weights.IMAGENET1K_V1") model = RetinaNet(backbone=backbone, num_classes=num_classes) From ec3760aa9cabf1d422fc51d1568da28dc8450bd8 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Thu, 21 Sep 2023 15:12:33 +1000 Subject: [PATCH 35/44] tidy up and update of timer etc. --- cycling_utils/cycling_utils/__init__.py | 4 +- cycling_utils/cycling_utils/sampler.py | 32 +- cycling_utils/cycling_utils/saving.py | 128 ++---- cycling_utils/cycling_utils/timer.py | 68 +++- hello_world/hello_world.isc | 7 - hello_world/hello_world.py | 6 - llama2-qlora/requirements.txt | 8 - monai/brats_mri_2d_v0/brats_mri_2d_diff.isc | 6 - monai/brats_mri_2d_v0/brats_mri_2d_gen.isc | 6 - monai/brats_mri_2d_v0/loops.py | 339 ---------------- monai/brats_mri_2d_v0/prep.py | 19 - monai/brats_mri_2d_v0/train_cycling_diff.py | 285 ------------- monai/brats_mri_2d_v0/train_cycling_gen.py | 277 ------------- monai/brats_mri_2d_v1/utils.py | 72 ---- monai/monai.bundle/LICENSE | 201 --------- monai/monai.bundle/brats_mri_2d.isc | 10 - monai/monai.bundle/configs/inference.json | 103 ----- .../configs/inference_autoencoder.json | 156 ------- monai/monai.bundle/configs/logging.conf | 21 - monai/monai.bundle/configs/metadata.json | 110 ----- .../configs/multi_gpu_train_autoencoder.json | 43 -- .../configs/multi_gpu_train_diffusion.json | 16 - .../configs/train_autoencoder.json | 151 ------- .../monai.bundle/configs/train_diffusion.json | 174 -------- monai/monai.bundle/docs/README.md | 166 -------- monai/monai.bundle/docs/data_license.txt | 49 --- monai/monai.bundle/scripts/__init__.py | 12 - monai/monai.bundle/scripts/ldm_sampler.py | 60 --- monai/monai.bundle/scripts/ldm_trainer.py | 380 ------------------ monai/monai.bundle/scripts/losses.py | 52 --- monai/monai.bundle/scripts/utils.py | 50 --- .../brats_mri_2d_diff.isc | 2 +- .../brats_mri_2d_gen.isc | 0 .../loops.py | 17 +- .../prep.py | 0 .../train_cycling_diff.py | 19 +- .../train_cycling_gen.py | 0 .../utils.py | 0 tv-detection/engine.py | 16 +- tv-detection/retinanet_resnet50_fpn.isc | 7 - tv-detection/train_cycling.py | 10 +- tv-segmentation/train_cycling.py | 10 +- 42 files changed, 152 insertions(+), 2940 deletions(-) delete mode 100644 hello_world/hello_world.isc delete mode 100644 hello_world/hello_world.py delete mode 100644 llama2-qlora/requirements.txt delete mode 100644 monai/brats_mri_2d_v0/brats_mri_2d_diff.isc delete mode 100644 monai/brats_mri_2d_v0/brats_mri_2d_gen.isc delete mode 100644 monai/brats_mri_2d_v0/loops.py delete mode 100644 monai/brats_mri_2d_v0/prep.py delete mode 100644 monai/brats_mri_2d_v0/train_cycling_diff.py delete mode 100644 monai/brats_mri_2d_v0/train_cycling_gen.py delete mode 100644 monai/brats_mri_2d_v1/utils.py delete mode 100644 monai/monai.bundle/LICENSE delete mode 100644 monai/monai.bundle/brats_mri_2d.isc delete mode 100644 monai/monai.bundle/configs/inference.json delete mode 100644 monai/monai.bundle/configs/inference_autoencoder.json delete mode 100644 monai/monai.bundle/configs/logging.conf delete mode 100644 monai/monai.bundle/configs/metadata.json delete mode 100644 monai/monai.bundle/configs/multi_gpu_train_autoencoder.json delete mode 100644 monai/monai.bundle/configs/multi_gpu_train_diffusion.json delete mode 100644 monai/monai.bundle/configs/train_autoencoder.json delete mode 100644 monai/monai.bundle/configs/train_diffusion.json delete mode 100644 monai/monai.bundle/docs/README.md delete mode 100644 monai/monai.bundle/docs/data_license.txt delete mode 100644 monai/monai.bundle/scripts/__init__.py delete mode 100644 monai/monai.bundle/scripts/ldm_sampler.py delete mode 100644 monai/monai.bundle/scripts/ldm_trainer.py delete mode 100644 monai/monai.bundle/scripts/losses.py delete mode 100644 monai/monai.bundle/scripts/utils.py rename {monai/brats_mri_2d_v1 => monai_brats_mri_2d}/brats_mri_2d_diff.isc (82%) rename {monai/brats_mri_2d_v1 => monai_brats_mri_2d}/brats_mri_2d_gen.isc (100%) rename {monai/brats_mri_2d_v1 => monai_brats_mri_2d}/loops.py (95%) rename {monai/brats_mri_2d_v1 => monai_brats_mri_2d}/prep.py (100%) rename {monai/brats_mri_2d_v1 => monai_brats_mri_2d}/train_cycling_diff.py (93%) rename {monai/brats_mri_2d_v1 => monai_brats_mri_2d}/train_cycling_gen.py (100%) rename {monai/brats_mri_2d_v0 => monai_brats_mri_2d}/utils.py (100%) delete mode 100644 tv-detection/retinanet_resnet50_fpn.isc diff --git a/cycling_utils/cycling_utils/__init__.py b/cycling_utils/cycling_utils/__init__.py index 22b215e8..c0866ab5 100644 --- a/cycling_utils/cycling_utils/__init__.py +++ b/cycling_utils/cycling_utils/__init__.py @@ -1,5 +1,5 @@ -from .timer import Timer +from .timer import Timer, TimestampedTimer from .saving import atomic_torch_save, MetricsTracker from .sampler import InterruptableDistributedSampler, InterruptableDistributedGroupedBatchSampler -__all__ = ["InterruptableDistributedSampler", "InterruptableDistributedGroupedBatchSampler", "atomic_torch_save", "Timer"] \ No newline at end of file +__all__ = ["InterruptableDistributedSampler", "InterruptableDistributedGroupedBatchSampler", "atomic_torch_save", "Timer", "TimestampedTimer"] \ No newline at end of file diff --git a/cycling_utils/cycling_utils/sampler.py b/cycling_utils/cycling_utils/sampler.py index 02cfcbf1..3aec4ddf 100644 --- a/cycling_utils/cycling_utils/sampler.py +++ b/cycling_utils/cycling_utils/sampler.py @@ -119,10 +119,6 @@ def in_epoch(self, epoch): yield self._reset_progress() - - -## FOR tv-detection, require grouped batches - def _repeat_to_at_least(iterable, n): repeat_times = math.ceil(n / len(iterable)) repeated = chain.from_iterable(repeat(iterable, repeat_times)) @@ -143,21 +139,25 @@ def __init__( """ This is a DistributedSampler that can be suspended and resumed. - This works by keeping track of the epoch and progress within the epoch. - The progress is the number of samples that have been returned by the - sampler. The epoch is the number of times the sampler has been iterated - over. + This works by keeping track of the sample batches that have already been + dispatched. This InterruptableDistributedGroupedBatchSampler also + reproduces the sampling strategy exhibited in the torch vision detection + reference wherein batches are created from images from within the same + 'group', defined in the torchvision example by similarity of image + aspect ratio. - The epoch is incremented at the start of each epoch. The epoch is set - to 0 at initialization. + https://github.com/pytorch/vision/tree/main/references/detection - The progress is incremented by the number of samples returned by the - sampler. The progress is reset to 0 at the end of each epoch. + For this reason, InterruptableDistributedGroupedBatchSampler progress is + tracked in units of batches, not samples. This is an important + distinction from the InterruptableDistributedSampler which tracks progress + in units of samples. The progress is reset to 0 at the end of each epoch. + + The epoch is set to 0 at initialization and incremented at the start + of each epoch. Suspending and resuming the sampler is done by saving and loading the - state dict. The state dict contains the epoch and progress. This works - because the permutation of the dataset is deterministic given the seed - and epoch. + state dict. The state dict contains the epoch and progress. """ super().__init__(dataset, num_replicas, rank, shuffle, seed, drop_last) @@ -195,7 +195,7 @@ def __init__( buffer_per_group = defaultdict(list) samples_per_group = defaultdict(list) - self.num_batches = math.ceil(len(indices)/ batch_size) # why not? + self.num_batches = math.ceil(len(indices)/ batch_size) self.batches = [] # pre-computed so progress refers to batches, not samples. for idx in indices: diff --git a/cycling_utils/cycling_utils/saving.py b/cycling_utils/cycling_utils/saving.py index ddfda40d..17f91898 100644 --- a/cycling_utils/cycling_utils/saving.py +++ b/cycling_utils/cycling_utils/saving.py @@ -18,6 +18,47 @@ def atomic_torch_save(obj, f: str | Path, timer=None, **kwargs): return class MetricsTracker: + ''' + This is a general purpose MetricsTracker to assist with recording metrics from + a disributed cluster. + + The MetricsTracker is initialised without any prior knowledge of the metrics + to be tracked. + + >>> metrics = MetricsTracker() + + Metrics can be accumulated as required, for example after each batch is procesed + by the model, by passing a dictionary with metrics to be updated, then reduced + accross all nodes. Metric values are stored in a defaultdict. + + >>> preds = model(input) + >>> loss = loss_fn(preds, targs) + >>> metrics.update({"images_seen": len(images), "loss": loss.item()}) + >>> metrics.reduce() + + Metrics are assumed to be summable scalar values. After calling reduce(), the + metrics.local object contains the sum of corresponding metrics from all nodes + which can be used for intermediate reporting or logging. + + >>> writer = SummaryWriter() + >>> for metric,val in metrics.local.items(): + >>> writer.add_scalar(metric, val, step) + >>> writer.flush() + >>> writer.close() + + Once all processing of the current batch has been completed, the MetricsTracker + can be prepared for the next batch using reset_local(). + + >>> metrics.reset_loca() + + Metrics are also accumulated for consecutive batches in the metrics.agg object. + At the end of an epoch the MetricsTracker can be reset using end_epoch(). + + >>> metrics.end_epoch() + + The MetricsTracker saves a copy of the accumulated metrics (metrics.agg) for + each epoch which can be stored within a checkpoint. + ''' def __init__(self): self.local = defaultdict(float) self.agg = defaultdict(float) @@ -42,92 +83,5 @@ def end_epoch(self): self.epoch_reports.append(dict(self.agg)) self.local = defaultdict(float) self.agg = defaultdict(float) - - -# ## ENABLING ACTIVE PROGRESS TRACKING -# # If ths is a desireable proto-solution, how should this be integrated with the cluster_server repo, -# # and how then imported for use in training scripts? Authentication required? - -# from sqlalchemy.orm import Session, sessionmaker -# from sqlmodel import SQLModel, create_engine -# from strenum import StrEnum - -# class Experiment(SQLModel, table=True): -# __tablename__ = "experiments" - -# id: int = Field(primary_key=True, index=True) -# org_id: str = Field(unique=False) -# user_id: int = Field(foreign_key="users.id", unique=False) -# user: User = Relationship(back_populates="experiments") -# runtime: int | None = Field(unique=False, nullable=True) -# name: str = Field(unique=False) -# output_path_used: str | None = Field(unique=False, nullable=True) -# output_path: str = Field(unique=False) -# ips: dict[str, int] | None = Field(sa_column=Column(JSON, nullable=True)) -# status: str = Field(nullable=False) -# gpu_type: str = Field(unique=False) -# nnodes: int = Field(unique=False) -# venv_path: str = Field(unique=False) -# command: str = Field(unique=False) -# work_dir: str = Field(unique=False) -# framework: str | None = Field(unique=False, nullable=True) -# created_at: datetime = Field(default_factory=timestamp_factory, nullable=False) -# usage_wall_time: int = Field(default=0, nullable=False) -# last_ran_at: datetime | None = Field(default=None, nullable=True) -# # started_at: datetime | None = Field(nullable=True) -# # finished_at: datetime | None = Field(nullable=True) -# progress: int = Field(default=0, nullable = False) # perhaps - - -# SQLALCHEMY_DATABASE_URL = "postgresql://postgres:postgres@localhost:5432/cluster_server" - -# def get_db(database_url: str | None = None) -> Session: -# if database_url is None: -# database_url = SQLALCHEMY_DATABASE_URL -# """Returns a session to the database""" -# engine = create_engine(database_url, isolation_level="AUTOCOMMIT") -# SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) -# return SessionLocal() - -# # SessionLocal: Callable[[], Session] = get_db -# SessionLocal = get_db - -# class AtomicTorchSave: -# def __init__(self): -# self.progress = 0 # Stored as internal state, posted to database upon save -# self.experiment_id = os.environ["STRONG_EXPERIMENT_ID"] - -# def commit_progress(self, authID): -# db = SessionLocal(authID) -# db_experiment = db.query(Experiment).filter(Experiment.id == self.experiment_id).first() -# assert db_experiment is not None -# db_experiment.progress = self.progress -# db.commit() -# db.refresh(db_experiment) -# return db_experiment - -# def save(self, obj, f: str | Path, timer=None, **kwargs): -# f = str(f) -# temp_f = f + ".temp" -# torch.save(obj, temp_f, **kwargs) -# if timer is not None: -# timer.report(f'saving temp checkpoint') -# os.replace(temp_f, f) -# timer.report(f'replacing temp checkpoint with checkpoint') - -# if self.experiment_id is not None: -# try: -# self.commit_progress() -# except: -# print("Progress commit impossible without experiment_id.") -# else: -# print("Experiment id not set.") -# timer.report(f'committing progress to database') - -# if timer is not None: -# return timer -# else: -# return - diff --git a/cycling_utils/cycling_utils/timer.py b/cycling_utils/cycling_utils/timer.py index 47d68b84..98048e2f 100644 --- a/cycling_utils/cycling_utils/timer.py +++ b/cycling_utils/cycling_utils/timer.py @@ -1,13 +1,29 @@ -import time -import os +import os, time +from datetime import datetime class Timer: + ''' + This Timer can be integrated within a training routine to provide point-to-point + script timing and reporting. + + def main(): + timer = Timer() + time.sleep(2) + timer.report("sleeping for 2 seconds") + time.sleep(3) + timer.report("sleeping for 3 seconds") + + >>> main() + Start 0.000 ms 0.000 s total + Completed sleeping for 2 seconds 2,000.000 ms 2.000 s total + Completed sleeping for 3 seconds 3,000.000 ms 5.000 s total + ''' def __init__(self, report=None, start_time=None, running=0): self.start_time = start_time if start_time is not None else time.time() self.running = running if str(os.environ["RANK"]) == "0": report = report if report else "Start" - print("{:<80}{:>12} ms, {:>12} s total".format(report, f'{0.0:,.3f}', f'{0.0:,.2f}')) + print("[{:<80}] {:>12} ms, {:>12} s total".format(report, f'{0.0:,.3f}', f'{0.0:,.2f}')) def report(self, annot): if str(os.environ["RANK"]) == "0": now = time.time() @@ -16,19 +32,33 @@ def report(self, annot): print("Completed {:<70}{:>12} ms, {:>12} s total".format(annot, f'{1000*duration:,.3f}', f'{self.running:,.2f}')) self.start_time = now -# import os -# from dateime import datetime -# class Timer: -# def __init__(self, report=None, start_time=None, running=0): -# self.start_time = start_time if start_time is not None else datetime.now() -# self.running = running -# if str(os.environ["RANK"]) == "0": -# report = report if report else "Start" -# print("{:<30} {:<70}{:>12} ms, {:>12} s total".format(self.start_time, report, f'{0.0:,.3f}', f'{0.0:,.2f}')) -# def report(self, annot): -# if str(os.environ["RANK"]) == "0": -# now = datetime.now() -# duration = (now - self.start_time).total_seconds() -# self.running += duration -# print("{:<30} Completed {:<70}{:>12} ms, {:>12} s total".format(now, annot, f'{1000*duration:,.3f}', f'{self.running:,.2f}')) -# self.start_time = now \ No newline at end of file +class TimestampedTimer: + ''' + This TimestampedTimer can be integrated within a training routine to provide + point-to-point script timing and reporting. + + def main(): + timer = TimestampedTimer() + time.sleep(2) + timer.report("sleeping for 2 seconds") + time.sleep(3) + timer.report("sleeping for 3 seconds") + + >>> main() + [TIME] Start 0.000 ms 0.000 s total + [TIME] Completed sleeping for 2 seconds 2,000.000 ms 2.000 s total + [TIME] Completed sleeping for 3 seconds 3,000.000 ms 5.000 s total + ''' + def __init__(self, report=None, start_time=None, running=0): + if str(os.environ["RANK"]) == "0": + self.start_time = start_time if start_time is not None else time.time() + self.running = running + report = report if report else "Start" + print("[ {} ] Completed {:<70}{:>12} ms, {:>12} s total".format(time.strftime("%Y-%m-%d %H:%M:%S"), report, f'{0.0:,.3f}', f'{0.0:,.2f}')) + def report(self, annot): + if str(os.environ["RANK"]) == "0": + now = time.time() + duration = now - self.start_time + self.running += duration + print("[ {} ] Completed {:<70}{:>12} ms, {:>12} s total".format(time.strftime("%Y-%m-%d %H:%M:%S"), annot, f'{1000*duration:,.3f}', f'{self.running:,.2f}')) + self.start_time = now \ No newline at end of file diff --git a/hello_world/hello_world.isc b/hello_world/hello_world.isc deleted file mode 100644 index c956c7af..00000000 --- a/hello_world/hello_world.isc +++ /dev/null @@ -1,7 +0,0 @@ - -experiment_name="hello_world" -gpu_type="24GB VRAM GPU" -nnodes = 11 -venv_path = "~/.venv/bin/activate" -output_path = "~/output_hello_world" -command="hello_world.py --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/hello_world/hello_world.py b/hello_world/hello_world.py deleted file mode 100644 index f351d4a9..00000000 --- a/hello_world/hello_world.py +++ /dev/null @@ -1,6 +0,0 @@ -import socket -import os - -print("Hello world!") -print(socket.gethostname()) -print(os.environ) \ No newline at end of file diff --git a/llama2-qlora/requirements.txt b/llama2-qlora/requirements.txt deleted file mode 100644 index 9deb35bf..00000000 --- a/llama2-qlora/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -torch -accelerate @ git+https://github.com/huggingface/accelerate.git -bitsandbytes -datasets==2.13.1 -transformers @ git+https://github.com/huggingface/transformers.git -peft @ git+https://github.com/huggingface/peft.git -trl @ git+https://github.com/lvwerra/trl.git -scipy \ No newline at end of file diff --git a/monai/brats_mri_2d_v0/brats_mri_2d_diff.isc b/monai/brats_mri_2d_v0/brats_mri_2d_diff.isc deleted file mode 100644 index 9409c35c..00000000 --- a/monai/brats_mri_2d_v0/brats_mri_2d_diff.isc +++ /dev/null @@ -1,6 +0,0 @@ -experiment_name="brats_mri_2d_diff" -gpu_type="24GB VRAM GPU" -nnodes = 9 -venv_path = "~/.venv/bin/activate" -output_path = "~/output_brats_mri_2d_diff" -command="train_cycling_diff.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc, --gen-load-path ~/output_brats_mri_2d_gen/exp_1645/checkpoint.isc" \ No newline at end of file diff --git a/monai/brats_mri_2d_v0/brats_mri_2d_gen.isc b/monai/brats_mri_2d_v0/brats_mri_2d_gen.isc deleted file mode 100644 index ff29a6c2..00000000 --- a/monai/brats_mri_2d_v0/brats_mri_2d_gen.isc +++ /dev/null @@ -1,6 +0,0 @@ -experiment_name="brats_mri_2d_gen" -gpu_type="24GB VRAM GPU" -nnodes = 9 -venv_path = "~/.venv/bin/activate" -output_path = "~/output_brats_mri_2d_gen" -command="train_cycling_gen.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/monai/brats_mri_2d_v0/loops.py b/monai/brats_mri_2d_v0/loops.py deleted file mode 100644 index 106dc3d9..00000000 --- a/monai/brats_mri_2d_v0/loops.py +++ /dev/null @@ -1,339 +0,0 @@ -from tqdm import tqdm -import torch -from torch.cuda.amp import GradScaler, autocast -import torch.nn.functional as F -import utils -from cycling_utils import atomic_torch_save - -## -- AUTO-ENCODER - ## - -def train_generator_one_epoch( - args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, - scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, - train_images_seen, val_images_seen, epoch_loss, gen_epoch_loss, disc_epoch_loss, val_loss - ): - - # Maybe pull these out into args later - kl_weight = 1e-6 - generator_warm_up_n_epochs = 3 - perceptual_weight = 0.001 - adv_weight = 0.01 - - generator.train() - discriminator.train() - - train_step = train_sampler.progress // train_loader.batch_size - total_steps = int(len(train_sampler) / train_loader.batch_size) - print(f'\nTraining / resuming epoch {epoch} from training step {train_step}\n') - - for step, batch in enumerate(train_loader): - - images = batch["image"].to(device) - timer.report(f'train batch {train_step} to device') - - # TRAIN GENERATOR - - optimizer_g.zero_grad(set_to_none=True) - - with autocast(enabled=True): - - reconstruction, z_mu, z_sigma = generator(images) - timer.report(f'train batch {train_step} generator forward') - recons_loss = F.l1_loss(reconstruction.float(), images.float()) - timer.report(f'train batch {train_step} recons_loss') - p_loss = perceptual_loss(reconstruction.float(), images.float()) - timer.report(f'train batch {train_step} p_loss') - kl_loss = 0.5 * torch.sum(z_mu.pow(2) + z_sigma.pow(2) - torch.log(z_sigma.pow(2)) - 1, dim=[1, 2, 3]) - kl_loss = torch.sum(kl_loss) / kl_loss.shape[0] - timer.report(f'train batch {train_step} kl_loss') - loss_g = recons_loss + (kl_weight * kl_loss) + (perceptual_weight * p_loss) - timer.report(f'train batch {train_step} loss_g (1)') - - if epoch > generator_warm_up_n_epochs: # Train generator for n epochs on reconstruction, KL, and perceptual loss before introducing discriminator loss - - logits_fake = discriminator(reconstruction.contiguous().float())[-1] - timer.report(f'train batch {train_step} logits_fake from discriminator') - generator_loss = adv_loss(logits_fake, target_is_real=True, for_discriminator=False) - timer.report(f'train batch {train_step} generator_loss (adv_loss)') - loss_g += adv_weight * generator_loss - timer.report(f'train batch {train_step} loss_g (2)') - - scaler_g.scale(loss_g).backward() - scaler_g.step(optimizer_g) - scaler_g.update() - timer.report(f'train batch {train_step} generator backward') - - # TRAIN DISCRIMINATOR - - if epoch > generator_warm_up_n_epochs: # Train generator for n epochs before starting discriminator training - - optimizer_d.zero_grad(set_to_none=True) - - with autocast(enabled=True): - - logits_fake = discriminator(reconstruction.contiguous().detach())[-1] - timer.report(f'train batch {train_step} discriminator forward (fake)') - loss_d_fake = adv_loss(logits_fake, target_is_real=False, for_discriminator=True) - timer.report(f'train batch {train_step} loss_d_fake') - logits_real = discriminator(images.contiguous().detach())[-1] - timer.report(f'train batch {train_step} discriminator forward (real)') - loss_d_real = adv_loss(logits_real, target_is_real=True, for_discriminator=True) - timer.report(f'train batch {train_step} loss_d_real') - discriminator_loss = (loss_d_fake + loss_d_real) * 0.5 - timer.report(f'train batch {train_step} discriminator_loss') - loss_d = adv_weight * discriminator_loss - timer.report(f'train batch {train_step} loss_d') - - scaler_d.scale(loss_d).backward() - scaler_d.step(optimizer_d) - scaler_d.update() - timer.report(f'train batch {train_step} discriminator backward') - - epoch_loss += recons_loss.item() - if epoch > generator_warm_up_n_epochs: - gen_epoch_loss += generator_loss.item() - disc_epoch_loss += discriminator_loss.item() - timer.report(f'train batch {train_step} metrics update') - - train_images_seen += len(images) - recons_loss = epoch_loss / train_images_seen - gen_loss = gen_epoch_loss / train_images_seen - disc_loss = disc_epoch_loss / train_images_seen - print("Epoch [{}] Step [{}/{}] :: recons_loss: {:,.3f}, gen_loss: {:,.3f}, disc_loss: {:,.3f}".format(epoch, train_step+1, total_steps, recons_loss, gen_loss, disc_loss)) - - ## Checkpointing - print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") - train_sampler.advance(len(images)) - train_step = train_sampler.progress // train_loader.batch_size - if utils.is_main_process() and train_step % 1 == 0: # Checkpointing every batch - checkpoint = { - # Universals - "args": args, - "epoch": epoch, - - # State variables - "generator": generator.module.state_dict(), - "discriminator": discriminator.module.state_dict(), - "optimizer_g": optimizer_g.state_dict(), - "optimizer_d": optimizer_d.state_dict(), - "scaler_g": scaler_g.state_dict(), - "scaler_d": scaler_d.state_dict(), - "train_sampler": train_sampler.state_dict(), - "val_sampler": val_sampler.state_dict(), - - # Evaluation metrics - "train_images_seen": train_images_seen, - "val_images_seen": val_images_seen, - "epoch_loss": epoch_loss, - "gen_epoch_loss": gen_epoch_loss, - "disc_epoch_loss": disc_epoch_loss, - "val_loss": val_loss, - } - timer = atomic_torch_save(checkpoint, args.resume, timer) - - return generator, timer - - -def evaluate_generator( - args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, - scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, - train_images_seen, val_images_seen, epoch_loss, gen_epoch_loss, disc_epoch_loss, val_loss - ): - - generator.eval() - - val_step = val_sampler.progress // val_loader.batch_size - print(f'\nEvaluating / resuming epoch {epoch} from training step {val_step}\n') - - with torch.no_grad(): - for step, batch in enumerate(val_loader): - - images = batch["image"].to(device) - timer.report(f'eval batch {val_step} to device') - - with autocast(enabled=True): - - reconstruction, _, _ = generator(images) - timer.report(f'eval batch {val_step} forward') - recons_loss = F.l1_loss(images.float(), reconstruction.float()) - timer.report(f'eval batch {val_step} recons_loss') - - val_loss += recons_loss.item() - val_images_seen += len(images) - timer.report(f'eval batch {val_step} metrics update') - - ## Checkpointing - print(f"Saving checkpoint at epoch {epoch} val batch {val_step}") - val_sampler.advance(len(images)) - if utils.is_main_process() and val_step % 1 == 0: # Checkpointing every batch - print(f"Saving checkpoint at epoch {epoch} train batch {val_step}") - checkpoint = { - # Universals - "args": args, - "epoch": epoch, - - # State variables - "generator": generator.module.state_dict(), - "discriminator": discriminator.module.state_dict(), - "optimizer_g": optimizer_g.state_dict(), - "optimizer_d": optimizer_d.state_dict(), - "scaler_g": scaler_g.state_dict(), - "scaler_d": scaler_d.state_dict(), - "train_sampler": train_sampler.state_dict(), - "val_sampler": val_sampler.state_dict(), - - # Evaluation metrics - "train_images_seen": train_images_seen, - "val_images_seen": val_images_seen, - "epoch_loss": epoch_loss, - "gen_epoch_loss": gen_epoch_loss, - "disc_epoch_loss": disc_epoch_loss, - "val_loss": val_loss, - } - timer = atomic_torch_save(checkpoint, args.resume, timer) - - val_loss /= val_images_seen - print(f"Epoch {epoch} val loss: {val_loss:.4f}") - - return timer - - - -## -- DIFFUSION MODEL - ## - -def train_diffusion_one_epoch( - args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, - train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer - ): - - unet.train() - generator.eval() - - train_step = train_sampler.progress // train_loader.batch_size - total_steps = int(len(train_sampler) / train_loader.batch_size) - print(f'\nTraining / resuming epoch {epoch} from training step {train_step}\n') - - for step, batch in enumerate(train_loader): - - images = batch["image"].to(device) - timer.report(f'train batch {train_step} to device') - - optimizer_u.zero_grad(set_to_none=True) - - with autocast(enabled=True): - - z_mu, z_sigma = generator.encode(images) - timer.report(f'train batch {train_step} generator encoded') - z = generator.sampling(z_mu, z_sigma) - timer.report(f'train batch {train_step} generator sampling') - noise = torch.randn_like(z).to(device) - timer.report(f'train batch {train_step} noise') - timesteps = torch.randint(0, inferer.scheduler.num_train_timesteps, (z.shape[0],), device=z.device).long() - timer.report(f'train batch {train_step} timesteps') - noise_pred = inferer(inputs=images, diffusion_model=unet, noise=noise, timesteps=timesteps, autoencoder_model=generator) - timer.report(f'train batch {train_step} noise_pred') - loss = F.mse_loss(noise_pred.float(), noise.float()) - timer.report(f'train batch {train_step} loss') - - scaler_u.scale(loss).backward() - scaler_u.step(optimizer_u) - scaler_u.update() - timer.report(f'train batch {train_step} unet backward') - - epoch_loss += loss.item() - train_images_seen += len(images) - recons_loss = epoch_loss / train_images_seen - print("Epoch [{}] Step [{}/{}] :: recons_loss: {:,.3f}".format(epoch, train_step+1, total_steps, recons_loss)) - - ## Checkpointing - print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") - train_sampler.advance(len(images)) - train_step = train_sampler.progress // train_loader.batch_size - if utils.is_main_process() and train_step % 1 == 0: # Checkpointing every batch - checkpoint = { - # Universals - "args": args, - "epoch": epoch, - - # State variables - "unet": unet.module.state_dict(), - "optimizer_u": optimizer_u.state_dict(), - "scaler_u": scaler_u.state_dict(), - "train_sampler": train_sampler.state_dict(), - "val_sampler": val_sampler.state_dict(), - - # Evaluation metrics - "train_images_seen": train_images_seen, - "val_images_seen": val_images_seen, - "epoch_loss": epoch_loss, - "val_loss": val_loss, - } - timer = atomic_torch_save(checkpoint, args.resume, timer) - - return unet, timer - - -def evaluate_diffusion( - args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, - train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer - ): - - unet.eval() - - val_step = val_sampler.progress // val_loader.batch_size - print(f'\nEvaluating / resuming epoch {epoch} from training step {val_step}\n') - - with torch.no_grad(): - for step, batch in enumerate(val_loader): - - images = batch["image"].to(device) - timer.report(f'eval batch {val_step} to device') - - with autocast(enabled=True): - - z_mu, z_sigma = generator.encode(images) - timer.report(f'eval batch {val_step} generator encoded') - z = generator.sampling(z_mu, z_sigma) - timer.report(f'eval batch {val_step} generator sampling') - noise = torch.randn_like(z).to(device) - timer.report(f'eval batch {val_step} noise') - timesteps = torch.randint(0, inferer.scheduler.num_train_timesteps, (z.shape[0],), device=z.device).long() - timer.report(f'eval batch {val_step} timesteps') - noise_pred = inferer(inputs=images,diffusion_model=unet,noise=noise,timesteps=timesteps,autoencoder_model=generator) - timer.report(f'eval batch {val_step} noise_pred') - loss = F.mse_loss(noise_pred.float(), noise.float()) - timer.report(f'eval batch {val_step} loss') - - val_loss += loss.item() - val_images_seen += len(images) - timer.report(f'eval batch {val_step} metrics update') - - ## Checkpointing - print(f"Saving checkpoint at epoch {epoch} val batch {val_step}") - val_sampler.advance(len(images)) - if utils.is_main_process() and val_step % 1 == 0: # Checkpointing every batch - print(f"Saving checkpoint at epoch {epoch} train batch {val_step}") - checkpoint = { - # Universals - "args": args, - "epoch": epoch, - - # State variables - "unet": unet.module.state_dict(), - "optimizer_u": optimizer_u.state_dict(), - "scaler_u": scaler_u.state_dict(), - "train_sampler": train_sampler.state_dict(), - "val_sampler": val_sampler.state_dict(), - - # Evaluation metrics - "train_images_seen": train_images_seen, - "val_images_seen": val_images_seen, - "epoch_loss": epoch_loss, - "val_loss": val_loss, - } - timer = atomic_torch_save(checkpoint, args.resume, timer) - - val_loss /= val_images_seen - print(f"Epoch {epoch} diff val loss: {val_loss:.4f}") - - return timer diff --git a/monai/brats_mri_2d_v0/prep.py b/monai/brats_mri_2d_v0/prep.py deleted file mode 100644 index a3a43134..00000000 --- a/monai/brats_mri_2d_v0/prep.py +++ /dev/null @@ -1,19 +0,0 @@ -# Download the toy dataset from MONAI -print("Downloadning BraTS2016/17") -from monai.apps import DecathlonDataset - -# _ = DecathlonDataset(root_dir="/mnt/Datasets/Open-Datasets/MONAI", task="Task01_BrainTumour", section="training", download=True) -_ = DecathlonDataset(root_dir="/mnt/.node1/Open-Datsets/MONAI", task="Task01_BrainTumour", section="training", download=True) - -perceptual_loss = PerceptualLoss( - spatial_dims=2, network_type="resnet50", pretrained=True, #ImageNet pretrained weights used -) - -# # Download the bigger dataset from Synapse -# print("Downloadning BraTS2023") -# import synapseclient -# syn = synapseclient.Synapse() -# syn.login('adam_peaston','AXXXXXXXXX2') -# syn51514132 = syn.get(entity='syn51514132', downloadFile=True, downloadLocation="/mnt/Datasets/strongcompute_adam/MONAI", ifcollision="overwrite.local") -# filepath = syn51514132.path -# print(f"BraTS2023-GLI downloaded to {filepath}") \ No newline at end of file diff --git a/monai/brats_mri_2d_v0/train_cycling_diff.py b/monai/brats_mri_2d_v0/train_cycling_diff.py deleted file mode 100644 index c1665320..00000000 --- a/monai/brats_mri_2d_v0/train_cycling_diff.py +++ /dev/null @@ -1,285 +0,0 @@ -from cycling_utils import Timer - -timer = Timer() -timer.report('importing Timer') - -import os - -import matplotlib.pyplot as plt -import numpy as np -import torch -import torch.nn.functional as F -from monai import transforms -from monai.apps import DecathlonDataset -from monai.config import print_config -from monai.data import DataLoader, Dataset -from monai.utils import first, set_determinism -from torch.cuda.amp import GradScaler, autocast -from pathlib import Path -from tqdm import tqdm - -from generative.inferers import LatentDiffusionInferer -from generative.losses.adversarial_loss import PatchAdversarialLoss -from generative.losses.perceptual import PerceptualLoss -from generative.networks.nets import AutoencoderKL, DiffusionModelUNet, PatchDiscriminator -from generative.networks.schedulers import DDPMScheduler - -from cycling_utils import InterruptableDistributedSampler, Timer -# from loops import train_generator_one_epoch, evaluate_generator -from loops import train_diffusion_one_epoch, evaluate_diffusion -import utils - -def get_args_parser(add_help=True): - import argparse - - parser = argparse.ArgumentParser(description="Latent Diffusion Model Training", add_help=add_help) - - parser.add_argument("--resume", type=str, help="path of checkpoint", required=True) # for checkpointing - parser.add_argument("--gen-load-path", type=str, help="path of checkpoint", dest="gen_load_path") # for checkpointing - parser.add_argument("--prev-resume", default=None, help="path of previous job checkpoint for strong fail resume", dest="prev_resume") # for checkpointing - # parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") - parser.add_argument("--data-path", default="/mnt/Datasets/Open-Datasets/MONAI", type=str, help="dataset path", dest="data_path") - parser.add_argument("--device", default="cuda", type=str, help="device (Use cuda or cpu Default: cuda)") - parser.add_argument("--start-epoch", default=0, type=int, metavar="N", help="start epoch") - # parser.add_argument("-b", "--batch-size", default=8, type=int, help="images per gpu, the total batch size is $NGPU x batch_size", dest="batch_size") - # parser.add_argument("--epochs", default=30, type=int, metavar="N", help="number of total epochs to run") - # parser.add_argument("--print-freq", default=1, type=int, help="print frequency") - parser.add_argument("--dist-url", default="env://", type=str, help="url used to set up distributed training") - parser.add_argument("-j", "--workers", default=16, type=int, metavar="N", help="number of data loading workers (default: 16)") - - return parser - -timer.report('importing everything else') - -def main(args, timer): - - # ## Distributed training prelims - # if args.output_dir: - # utils.mkdir(args.output_dir) - - utils.init_distributed_mode(args) # Sets args.distributed among other things - assert args.distributed # don't support cycling when not distributed for simplicity - - device = torch.device(args.device) - - timer.report('preliminaries') - - # Maybe this will work? - set_determinism(42) - - channel = 0 # 0 = Flair - assert channel in [0, 1, 2, 3], "Choose a valid channel" - ## NEED TO COME BACK AND ALIGN WITH BRATS CONFIG - train_transforms = transforms.Compose([ - transforms.LoadImaged(keys=["image", "label"], image_only=False), # image_only current default will change soon, so including explicitly - transforms.EnsureChannelFirstd(keys=["image", "label"]), - transforms.Lambdad(keys=["image"], func=lambda x: x[channel, None, :, :, :]), - transforms.EnsureTyped(keys=["image", "label"]), - transforms.Orientationd(keys=["image", "label"], axcodes="RAS"), - transforms.Spacingd(keys=["image", "label"], pixdim=(3.0, 3.0, 2.0), mode=("bilinear", "nearest")), - transforms.CenterSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 44)), - transforms.ScaleIntensityRangePercentilesd(keys="image", lower=0, upper=99.5, b_min=0, b_max=1), - transforms.RandSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 1), random_size=False, random_centre=True), # Eeach of the 44 slices will be randomly sampled. - transforms.Lambdad(keys=["image", "label"], func=lambda x: x.squeeze(-1)), - transforms.CopyItemsd(keys=["label"], times=1, names=["slice_label"]), - transforms.Lambdad(keys=["slice_label"], func=lambda x: 1.0 if x.sum() > 0 else 0.0), - ]) - - train_ds = DecathlonDataset( - root_dir=args.data_path, task="Task01_BrainTumour", section="training", cache_rate=0.0, - num_workers=4, download=False, seed=0, transform=train_transforms, - ) - val_ds = DecathlonDataset( - root_dir=args.data_path, task="Task01_BrainTumour", section="validation", cache_rate=0.0, - num_workers=4, download=False, seed=0, transform=train_transforms, - ) - - timer.report('build datasets') - - train_sampler = InterruptableDistributedSampler(train_ds) - val_sampler = InterruptableDistributedSampler(val_ds) - - timer.report('build samplers') - - train_loader = DataLoader(train_ds, batch_size=2, sampler=train_sampler, num_workers=args.workers) - val_loader = DataLoader(val_ds, batch_size=1, sampler=val_sampler, num_workers=args.workers) - check_data = first(train_loader) # Used later - - timer.report('build dataloaders') - - # Auto-encoder definition - generator = AutoencoderKL( - spatial_dims=2, in_channels=1, out_channels=1, num_channels=(128, 128, 256), latent_channels=3, - num_res_blocks=2, attention_levels=(False, False, False),with_encoder_nonlocal_attn=False, - with_decoder_nonlocal_attn=False, - ) - # saved_generator_checkpoint = torch.load("/output_brats_mri_2d_gen/exp_1645/checkpoint.isc", map_location="cpu") - saved_generator_checkpoint = torch.load(args.gen_load_path, map_location="cpu") - generator.load_state_dict(saved_generator_checkpoint["generator"]) - generator = generator.to(device) - - timer.report('generator to device') - - # # Discriminator definition - # discriminator = PatchDiscriminator( - # spatial_dims=2, num_layers_d=3, num_channels=64, - # in_channels=1, out_channels=1 - # ) - # discriminator = discriminator.to(device) - - # timer.report('discriminator to device') - - # Diffusion model (unet) - unet = DiffusionModelUNet( - spatial_dims=2, in_channels=3, out_channels=3, num_res_blocks=2, - num_channels=(128, 256, 512),attention_levels=(False, True, True), - num_head_channels=(0, 256, 512), - ) - unet = unet.to(device) - - timer.report('unet to device') - - # # Autoencoder loss functions - # adv_loss = PatchAdversarialLoss(criterion="least_squares") - # perceptual_loss = PerceptualLoss(spatial_dims=2, network_type="alex") - # perceptual_loss.to(device) - - # timer.report('loss functions') - - # Prepare for distributed training - # generator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) - # discriminator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) - unet = torch.nn.SyncBatchNorm.convert_sync_batchnorm(unet) - - # generator_without_ddp = generator - # discriminator_without_ddp = discriminator - unet_without_ddp = unet - if args.distributed: - # generator = torch.nn.parallel.DistributedDataParallel(generator, device_ids=[args.gpu], find_unused_parameters=True) - # discriminator = torch.nn.parallel.DistributedDataParallel(discriminator, device_ids=[args.gpu], find_unused_parameters=True) - unet = torch.nn.parallel.DistributedDataParallel(unet, device_ids=[args.gpu], find_unused_parameters=True) - # generator_without_ddp = generator.module - # discriminator_without_ddp = discriminator.module - unet_without_ddp = unet.module - - timer.report('unet prepped for distribution') - - # Optimizers - # optimizer_g = torch.optim.Adam(generator_without_ddp.parameters(), lr=1e-4) - # optimizer_d = torch.optim.Adam(discriminator_without_ddp.parameters(), lr=5e-4) - optimizer_u = torch.optim.Adam(unet_without_ddp.parameters(), lr=1e-4) - - timer.report('optimizers') - - # For mixed precision training - # scaler_g = GradScaler() - # scaler_d = GradScaler() - scaler_u = GradScaler() - - timer.report('grad scalers') - - # Init tracking metrics - train_images_seen = 0 - val_images_seen = 0 - epoch_loss = 0 - val_loss = 0 - - # RETRIEVE GENERATOR CHECKPOINT FROM PREVIOUS JOB - - # RETRIEVE CHECKPOINT - Path(args.resume).parent.mkdir(parents=True, exist_ok=True) - checkpoint = None - if args.resume and os.path.isfile(args.resume): # If we're resuming... - checkpoint = torch.load(args.resume, map_location="cpu") - elif args.prev_resume and os.path.isfile(args.prev_resume): - checkpoint = torch.load(args.prev_resume, map_location="cpu") - if checkpoint is not None: - args.start_epoch = checkpoint["epoch"] - unet_without_ddp.load_state_dict(checkpoint["unet"]) - optimizer_u.load_state_dict(checkpoint["optimizer_u"]) - scaler_u.load_state_dict(checkpoint["scaler_u"]) - train_sampler.load_state_dict(checkpoint["train_sampler"]) - val_sampler.load_state_dict(checkpoint["val_sampler"]) - train_images_seen = checkpoint["train_images_seen"] - val_images_seen = checkpoint["val_images_seen"] - # Metrics - train_images_seen = checkpoint["train_images_seen"] - val_images_seen = checkpoint["val_images_seen"] - epoch_loss = checkpoint["epoch_loss"] - val_loss = checkpoint["val_loss"] - - timer.report('checkpoint retrieval') - - # ## -- TRAINING THE AUTO-ENCODER - ## - - # n_gen_epochs = 100 - # gen_val_interval = 1 - - # for epoch in range(args.start_epoch, n_gen_epochs): - - # print('\n') - # print(f"EPOCH :: {epoch}") - # print('\n') - - # with train_sampler.in_epoch(epoch): - # timer = Timer("Start training") - # generator, timer = train_generator_one_epoch( - # args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, - # scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, - # train_images_seen, val_images_seen, epoch_loss, gen_epoch_loss, disc_epoch_loss, val_loss - # ) - # timer.report(f'training generator for epoch {epoch}') - - # if epoch % gen_val_interval == 0: # Eval every epoch - # with val_sampler.in_epoch(epoch): - # timer = Timer("Start evaluation") - # timer = evaluate_generator( - # args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, - # scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, - # train_images_seen, val_images_seen, epoch_loss, gen_epoch_loss, disc_epoch_loss, val_loss - # ) - # timer.report(f'evaluating generator for epoch {epoch}') - - - ## -- TRAINING THE DIFFUSION MODEL - ## - - n_diff_epochs = 200 - diff_val_interval = 1 - - # Prepare LatentDiffusionInferer - scheduler = DDPMScheduler(num_train_timesteps=1000, schedule="linear_beta", beta_start=0.0015, beta_end=0.0195) - with torch.no_grad(): - with autocast(enabled=True): - z = generator.encode_stage_2_inputs(check_data["image"].to(device)) - scale_factor = 1 / torch.std(z) - inferer = LatentDiffusionInferer(scheduler, scale_factor=scale_factor) - - timer.report('building inferer') - - for epoch in range(args.start_epoch, n_diff_epochs): - - print('\n') - print(f"EPOCH :: {epoch}") - print('\n') - - with train_sampler.in_epoch(epoch): - timer = Timer("Start training") - unet, timer = train_diffusion_one_epoch( - args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, - train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer - ) - timer.report(f'training unet for epoch {epoch}') - - if epoch % diff_val_interval == 0: - with val_sampler.in_epoch(epoch): - timer = Timer("Start evaluation") - timer = evaluate_diffusion( - args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, - train_sampler, val_sampler, train_images_seen, val_images_seen, epoch_loss, val_loss, device, timer - ) - timer.report(f'evaluating unet for epoch {epoch}') - - -if __name__ == "__main__": - args = get_args_parser().parse_args() - main(args, timer) diff --git a/monai/brats_mri_2d_v0/train_cycling_gen.py b/monai/brats_mri_2d_v0/train_cycling_gen.py deleted file mode 100644 index 72e93c29..00000000 --- a/monai/brats_mri_2d_v0/train_cycling_gen.py +++ /dev/null @@ -1,277 +0,0 @@ -from cycling_utils import Timer - -timer = Timer() -timer.report('importing Timer') - -import os - -# import matplotlib.pyplot as plt -# import numpy as np -import torch -# import torch.nn.functional as F -from monai import transforms -from monai.apps import DecathlonDataset -# from monai.config import print_config -from monai.data import DataLoader# , Dataset -from monai.utils import first, set_determinism -from torch.cuda.amp import GradScaler# , autocast -from pathlib import Path -# from tqdm import tqdm - -# from generative.inferers import LatentDiffusionInferer -from generative.losses.adversarial_loss import PatchAdversarialLoss -from generative.losses.perceptual import PerceptualLoss -from generative.networks.nets import AutoencoderKL, PatchDiscriminator # , DiffusionModelUNet -# from generative.networks.schedulers import DDPMScheduler - -from cycling_utils import InterruptableDistributedSampler, Timer -from loops import train_generator_one_epoch, evaluate_generator -# from loops import train_diffusion_one_epoch, evaluate_diffusion -import utils - -def get_args_parser(add_help=True): - import argparse - - parser = argparse.ArgumentParser(description="Latent Diffusion Model Training", add_help=add_help) - - parser.add_argument("--resume", type=str, help="path of checkpoint", required=True) # for checkpointing - parser.add_argument("--prev-resume", default=None, help="path of previous job checkpoint for strong fail resume", dest="prev_resume") # for checkpointing - # parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") - parser.add_argument("--data-path", default="/mnt/Datasets/Open-Datasets/MONAI", type=str, help="dataset path", dest="data_path") - parser.add_argument("--device", default="cuda", type=str, help="device (Use cuda or cpu Default: cuda)") - parser.add_argument("--start-epoch", default=0, type=int, metavar="N", help="start epoch") - # parser.add_argument("-b", "--batch-size", default=8, type=int, help="images per gpu, the total batch size is $NGPU x batch_size", dest="batch_size") - # parser.add_argument("--epochs", default=30, type=int, metavar="N", help="number of total epochs to run") - # parser.add_argument("--print-freq", default=1, type=int, help="print frequency") - parser.add_argument("--dist-url", default="env://", type=str, help="url used to set up distributed training") - parser.add_argument("-j", "--workers", default=16, type=int, metavar="N", help="number of data loading workers (default: 16)") - - return parser - -timer.report('importing everything else') - -def main(args, timer): - - utils.init_distributed_mode(args) # Sets args.distributed among other things - assert args.distributed # don't support cycling when not distributed for simplicity - - device = torch.device(args.device) - - timer.report('preliminaries') - - # Maybe this will work? - set_determinism(42) - - channel = 0 # 0 = Flair - assert channel in [0, 1, 2, 3], "Choose a valid channel" - ## NEED TO COME BACK AND ALIGN WITH BRATS CONFIG - train_transforms = transforms.Compose([ - transforms.LoadImaged(keys=["image", "label"], image_only=False), # image_only current default will change soon, so including explicitly - transforms.EnsureChannelFirstd(keys=["image", "label"]), - transforms.Lambdad(keys=["image"], func=lambda x: x[channel, None, :, :, :]), - transforms.EnsureTyped(keys=["image", "label"]), - transforms.Orientationd(keys=["image", "label"], axcodes="RAS"), - transforms.Spacingd(keys=["image", "label"], pixdim=(3.0, 3.0, 2.0), mode=("bilinear", "nearest")), - transforms.CenterSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 44)), - transforms.ScaleIntensityRangePercentilesd(keys="image", lower=0, upper=99.5, b_min=0, b_max=1), - transforms.RandSpatialCropd(keys=["image", "label"], roi_size=(64, 64, 1), random_size=False), # Each of the 44 slices will be randomly sampled. - transforms.Lambdad(keys=["image", "label"], func=lambda x: x.squeeze(-1)), - transforms.CopyItemsd(keys=["label"], times=1, names=["slice_label"]), - transforms.Lambdad(keys=["slice_label"], func=lambda x: 1.0 if x.sum() > 0 else 0.0), - ]) - - train_ds = DecathlonDataset( - root_dir=args.data_path, task="Task01_BrainTumour", section="training", cache_rate=0.0, - num_workers=1, download=False, seed=0, transform=train_transforms, - ) - val_ds = DecathlonDataset( - root_dir=args.data_path, task="Task01_BrainTumour", section="validation", cache_rate=0.0, - num_workers=1, download=False, seed=0, transform=train_transforms, - ) - - timer.report('build datasets') - - train_sampler = InterruptableDistributedSampler(train_ds) - val_sampler = InterruptableDistributedSampler(val_ds) - - timer.report('build samplers') - - train_loader = DataLoader(train_ds, batch_size=2, sampler=train_sampler, num_workers=1) - val_loader = DataLoader(val_ds, batch_size=1, sampler=val_sampler, num_workers=1) - # check_data = first(train_loader) # Used later - - timer.report('build dataloaders') - - # Auto-encoder definition - generator = AutoencoderKL( - spatial_dims=2, in_channels=1, out_channels=1, num_channels=(128, 128, 256), - latent_channels=3, num_res_blocks=2, attention_levels=(False, False, False), - with_encoder_nonlocal_attn=False, with_decoder_nonlocal_attn=False, - ) - generator = generator.to(device) - - timer.report('generator to device') - - # Discriminator definition - discriminator = PatchDiscriminator( - spatial_dims=2, num_layers_d=3, num_channels=64, - in_channels=1, out_channels=1 - ) - discriminator = discriminator.to(device) - - timer.report('discriminator to device') - - # # Diffusion model (unet) - # unet = DiffusionModelUNet( - # spatial_dims=2, in_channels=3, out_channels=3, num_res_blocks=2, - # num_channels=(128, 256, 512),attention_levels=(False, True, True), - # num_head_channels=(0, 256, 512), - # ) - # unet = unet.to(device) - - # timer.report('unet to device') - - # Autoencoder loss functions - adv_loss = PatchAdversarialLoss(criterion="least_squares") - perceptual_loss = PerceptualLoss(spatial_dims=2, network_type="alex") - perceptual_loss.to(device) - - timer.report('loss functions') - - # Prepare for distributed training - generator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) - discriminator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) - # unet = torch.nn.SyncBatchNorm.convert_sync_batchnorm(unet) - - generator_without_ddp = generator - discriminator_without_ddp = discriminator - # unet_without_ddp = unet - if args.distributed: - generator = torch.nn.parallel.DistributedDataParallel(generator, device_ids=[args.gpu], find_unused_parameters=True) # find_unused_parameters necessary for monai training - discriminator = torch.nn.parallel.DistributedDataParallel(discriminator, device_ids=[args.gpu], find_unused_parameters=True) # find_unused_parameters necessary for monai training - # unet = torch.nn.parallel.DistributedDataParallel(unet, device_ids=[args.gpu], find_unused_parameters=True) - generator_without_ddp = generator.module - discriminator_without_ddp = discriminator.module - # unet_without_ddp = unet.module - - timer.report('models prepped for distribution') - - # Optimizers - optimizer_g = torch.optim.Adam(generator_without_ddp.parameters(), lr=1e-4) - optimizer_d = torch.optim.Adam(discriminator_without_ddp.parameters(), lr=5e-4) - # optimizer_u = torch.optim.Adam(unet_without_ddp.parameters(), lr=1e-4) - - timer.report('optimizers') - - # For mixed precision training - scaler_g = GradScaler() - scaler_d = GradScaler() - # scaler_u = GradScaler() - - timer.report('grad scalers') - - # Init tracking metrics - train_images_seen = 0 - val_images_seen = 0 - epoch_loss = 0 - gen_epoch_loss = 0 - disc_epoch_loss = 0 - val_loss = 0 - - # RETRIEVE CHECKPOINT - Path(args.resume).parent.mkdir(parents=True, exist_ok=True) - checkpoint = None - if args.resume and os.path.isfile(args.resume): # If we're resuming... - checkpoint = torch.load(args.resume, map_location="cpu") - elif args.prev_resume and os.path.isfile(args.prev_resume): - checkpoint = torch.load(args.prev_resume, map_location="cpu") - if checkpoint is not None: - args.start_epoch = checkpoint["epoch"] - generator_without_ddp.load_state_dict(checkpoint["generator"]) - discriminator_without_ddp.load_state_dict(checkpoint["discriminator"]) - optimizer_g.load_state_dict(checkpoint["optimizer_g"]) - optimizer_d.load_state_dict(checkpoint["optimizer_d"]) - scaler_g.load_state_dict(checkpoint["scaler_g"]) - scaler_d.load_state_dict(checkpoint["scaler_d"]) - train_sampler.load_state_dict(checkpoint["train_sampler"]) - val_sampler.load_state_dict(checkpoint["val_sampler"]) - # Metrics - train_images_seen = checkpoint["train_images_seen"] - val_images_seen = checkpoint["val_images_seen"] - epoch_loss = checkpoint["epoch_loss"] - gen_epoch_loss = checkpoint["gen_epoch_loss"] - disc_epoch_loss = checkpoint["disc_epoch_loss"] - val_loss = checkpoint["val_loss"] - - timer.report('checkpoint retrieval') - - ## -- TRAINING THE AUTO-ENCODER - ## - - n_gen_epochs = 100 - gen_val_interval = 1 - - for epoch in range(args.start_epoch, n_gen_epochs): - - print('\n') - print(f"EPOCH :: {epoch}") - print('\n') - - with train_sampler.in_epoch(epoch): - timer = Timer("Start training") - generator, timer = train_generator_one_epoch( - args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, - scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, - train_images_seen, val_images_seen, epoch_loss, gen_epoch_loss, disc_epoch_loss, val_loss - ) - timer.report(f'training generator for epoch {epoch}') - - if epoch % gen_val_interval == 0: # Eval every epoch - with val_sampler.in_epoch(epoch): - timer = Timer("Start evaluation") - timer = evaluate_generator( - args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, - scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, - train_images_seen, val_images_seen, epoch_loss, gen_epoch_loss, disc_epoch_loss, val_loss - ) - timer.report(f'evaluating generator for epoch {epoch}') - - - # ## -- TRAINING THE DIFFUSION MODEL - ## - - # n_diff_epochs = 200 - # diff_val_interval = 1 - - # # Prepare LatentDiffusionInferer - # scheduler = DDPMScheduler(num_train_timesteps=1000, schedule="linear_beta", beta_start=0.0015, beta_end=0.0195) - # with torch.no_grad(): - # with autocast(enabled=True): - # z = generator.encode_stage_2_inputs(check_data["image"].to(device)) - # scale_factor = 1 / torch.std(z) - # inferer = LatentDiffusionInferer(scheduler, scale_factor=scale_factor) - - # timer.report('building inferer') - - # for epoch in range(n_diff_epochs): - - # print('\n') - # print(f"EPOCH :: {epoch}") - # print('\n') - - # with train_sampler.in_epoch(epoch): - # timer = Timer() - # unet, timer, _ = train_diffusion_one_epoch( - # epoch, unet, generator, optimizer_u, - # inferer, scaler_u, train_loader, device, timer - # ) - # timer.report(f'training unet for epoch {epoch}') - - # if epoch % diff_val_interval == 0: - # with val_sampler.in_epoch(epoch): - # timer = Timer() - # _ = evaluate_diffusion(epoch, unet, generator, inferer, val_loader, device) - # timer.report(f'evaluating unet for epoch {epoch}') - - -if __name__ == "__main__": - args = get_args_parser().parse_args() - main(args, timer) diff --git a/monai/brats_mri_2d_v1/utils.py b/monai/brats_mri_2d_v1/utils.py deleted file mode 100644 index 1b268b82..00000000 --- a/monai/brats_mri_2d_v1/utils.py +++ /dev/null @@ -1,72 +0,0 @@ -import torch, os, errno -import torch.distributed as dist - -def mkdir(path): - try: - os.makedirs(path) - except OSError as e: - if e.errno != errno.EEXIST: - raise - -def setup_for_distributed(is_master): - """ - This function disables printing when not in master process - """ - import builtins as __builtin__ - - builtin_print = __builtin__.print - - def print(*args, **kwargs): - force = kwargs.pop("force", False) - if is_master or force: - builtin_print(*args, **kwargs) - - __builtin__.print = print - -def init_distributed_mode(args): - if "RANK" in os.environ and "WORLD_SIZE" in os.environ: - args.rank = int(os.environ["RANK"]) - args.world_size = int(os.environ["WORLD_SIZE"]) - args.gpu = int(os.environ["LOCAL_RANK"]) - elif "SLURM_PROCID" in os.environ: - args.rank = int(os.environ["SLURM_PROCID"]) - args.gpu = args.rank % torch.cuda.device_count() - else: - print("Not using distributed mode") - args.distributed = False - return - - args.distributed = True - - torch.cuda.set_device(args.gpu) - args.dist_backend = "nccl" - print(f"| distributed init (rank {args.rank}): {args.dist_url}", flush=True) - torch.distributed.init_process_group( - backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank - ) - torch.distributed.barrier() - setup_for_distributed(args.rank == 0) - - -def is_dist_avail_and_initialized(): - if not dist.is_available(): - return False - if not dist.is_initialized(): - return False - return True - - -def get_world_size(): - if not is_dist_avail_and_initialized(): - return 1 - return dist.get_world_size() - - -def get_rank(): - if not is_dist_avail_and_initialized(): - return 0 - return dist.get_rank() - - -def is_main_process(): - return get_rank() == 0 \ No newline at end of file diff --git a/monai/monai.bundle/LICENSE b/monai/monai.bundle/LICENSE deleted file mode 100644 index 261eeb9e..00000000 --- a/monai/monai.bundle/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/monai/monai.bundle/brats_mri_2d.isc b/monai/monai.bundle/brats_mri_2d.isc deleted file mode 100644 index 631a6738..00000000 --- a/monai/monai.bundle/brats_mri_2d.isc +++ /dev/null @@ -1,10 +0,0 @@ -experiment_name="brats_mri_2d" -gpu_type="24GB VRAM GPU" -nnodes = 11 -venv_path = "~/.venv/bin/activate" -output_path = "~/output_brats_mri_2d" - - -torchrun --standalone --nnodes=1 --nproc_per_node=8 -m monai.bundle run --config_file "['configs/train_autoencoder.json','configs/multi_gpu_train_autoencoder.json']" --lr 4e-4 - -command="train_cycling.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/Datasets/Open-Datasets/coco --resume $OUTPUT_PATH/checkpoint.isc" \ No newline at end of file diff --git a/monai/monai.bundle/configs/inference.json b/monai/monai.bundle/configs/inference.json deleted file mode 100644 index 9e9e74b2..00000000 --- a/monai/monai.bundle/configs/inference.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "imports": [ - "$import torch", - "$from datetime import datetime", - "$from pathlib import Path", - "$from PIL import Image", - "$from scripts.utils import visualize_2d_image" - ], - "bundle_root": ".", - "model_dir": "$@bundle_root + '/models'", - "output_dir": "$@bundle_root + '/output'", - "create_output_dir": "$Path(@output_dir).mkdir(exist_ok=True)", - "device": "$torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')", - "output_postfix": "$datetime.now().strftime('sample_%Y%m%d_%H%M%S')", - "channel": 0, - "spatial_dims": 2, - "image_channels": 1, - "latent_channels": 1, - "latent_shape": [ - "@latent_channels", - 64, - 64 - ], - "autoencoder_def": { - "_target_": "generative.networks.nets.AutoencoderKL", - "spatial_dims": "@spatial_dims", - "in_channels": "@image_channels", - "out_channels": "@image_channels", - "latent_channels": "@latent_channels", - "num_channels": [ - 64, - 128, - 256 - ], - "num_res_blocks": 2, - "norm_num_groups": 32, - "norm_eps": 1e-06, - "attention_levels": [ - false, - false, - false - ], - "with_encoder_nonlocal_attn": true, - "with_decoder_nonlocal_attn": true - }, - "network_def": { - "_target_": "generative.networks.nets.DiffusionModelUNet", - "spatial_dims": "@spatial_dims", - "in_channels": "@latent_channels", - "out_channels": "@latent_channels", - "num_channels": [ - 32, - 64, - 128, - 256 - ], - "attention_levels": [ - false, - true, - true, - true - ], - "num_head_channels": [ - 0, - 32, - 32, - 32 - ], - "num_res_blocks": 2 - }, - "load_autoencoder_path": "$@bundle_root + '/models/model_autoencoder.pt'", - "load_autoencoder": "$@autoencoder_def.load_state_dict(torch.load(@load_autoencoder_path))", - "autoencoder": "$@autoencoder_def.to(@device)", - "load_diffusion_path": "$@model_dir + '/model.pt'", - "load_diffusion": "$@network_def.load_state_dict(torch.load(@load_diffusion_path))", - "diffusion": "$@network_def.to(@device)", - "noise_scheduler": { - "_target_": "generative.networks.schedulers.DDIMScheduler", - "_requires_": [ - "@load_diffusion", - "@load_autoencoder" - ], - "num_train_timesteps": 1000, - "beta_start": 0.0015, - "beta_end": 0.0195, - "schedule": "scaled_linear_beta", - "clip_sample": false - }, - "noise": "$torch.randn([1]+@latent_shape).to(@device)", - "set_timesteps": "$@noise_scheduler.set_timesteps(num_inference_steps=50)", - "inferer": { - "_target_": "scripts.ldm_sampler.LDMSampler", - "_requires_": "@set_timesteps" - }, - "sample": "$@inferer.sampling_fn(@noise, @autoencoder, @diffusion, @noise_scheduler)", - "generated_image": "$@sample", - "generated_image_np": "$@generated_image[0,0].cpu().numpy().transpose(1, 0)[::-1, ::-1]", - "img_pil": "$Image.fromarray(visualize_2d_image(@generated_image_np), 'RGB')", - "run": [ - "$@create_output_dir", - "$@img_pil.save(@output_dir+'/synimg_'+@output_postfix+'.png')" - ] -} diff --git a/monai/monai.bundle/configs/inference_autoencoder.json b/monai/monai.bundle/configs/inference_autoencoder.json deleted file mode 100644 index b31a3ea4..00000000 --- a/monai/monai.bundle/configs/inference_autoencoder.json +++ /dev/null @@ -1,156 +0,0 @@ -{ - "imports": [ - "$import torch", - "$from datetime import datetime", - "$from pathlib import Path", - "$from PIL import Image", - "$from scripts.utils import visualize_2d_image" - ], - "bundle_root": ".", - "model_dir": "$@bundle_root + '/models'", - "dataset_dir": "/workspace/data/medical", - "output_dir": "$@bundle_root + '/output'", - "create_output_dir": "$Path(@output_dir).mkdir(exist_ok=True)", - "device": "$torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')", - "output_postfix": "$datetime.now().strftime('%Y%m%d_%H%M%S')", - "channel": 0, - "spatial_dims": 2, - "image_channels": 1, - "latent_channels": 1, - "infer_patch_size": [ - 240, - 240 - ], - "infer_batch_size_img": 1, - "infer_batch_size_slice": 1, - "autoencoder_def": { - "_target_": "generative.networks.nets.AutoencoderKL", - "spatial_dims": "@spatial_dims", - "in_channels": "@image_channels", - "out_channels": "@image_channels", - "latent_channels": "@latent_channels", - "num_channels": [ - 64, - 128, - 256 - ], - "num_res_blocks": 2, - "norm_num_groups": 32, - "norm_eps": 1e-06, - "attention_levels": [ - false, - false, - false - ], - "with_encoder_nonlocal_attn": true, - "with_decoder_nonlocal_attn": true - }, - "load_autoencoder_path": "$@bundle_root + '/models/model_autoencoder.pt'", - "load_autoencoder": "$@autoencoder_def.load_state_dict(torch.load(@load_autoencoder_path))", - "autoencoder": "$@autoencoder_def.to(@device)", - "preprocessing_transforms": [ - { - "_target_": "LoadImaged", - "keys": "image" - }, - { - "_target_": "EnsureChannelFirstd", - "keys": "image" - }, - { - "_target_": "Lambdad", - "keys": "image", - "func": "$lambda x: x[@channel, :, :, :]" - }, - { - "_target_": "AddChanneld", - "keys": "image" - }, - { - "_target_": "EnsureTyped", - "keys": "image" - }, - { - "_target_": "Orientationd", - "keys": "image", - "axcodes": "RAS" - }, - { - "_target_": "CenterSpatialCropd", - "keys": "image", - "roi_size": "$[@infer_patch_size[0], @infer_patch_size[1], 20]" - }, - { - "_target_": "ScaleIntensityRangePercentilesd", - "keys": "image", - "lower": 0, - "upper": 100, - "b_min": 0, - "b_max": 1 - } - ], - "crop_transforms": [ - { - "_target_": "DivisiblePadd", - "keys": "image", - "k": [ - 4, - 4, - 1 - ] - }, - { - "_target_": "RandSpatialCropSamplesd", - "keys": "image", - "random_size": false, - "roi_size": "$[@infer_patch_size[0], @infer_patch_size[1], 1]", - "num_samples": "@infer_batch_size_slice" - }, - { - "_target_": "SqueezeDimd", - "keys": "image", - "dim": 3 - } - ], - "final_transforms": [ - { - "_target_": "ScaleIntensityRangePercentilesd", - "keys": "image", - "lower": 0, - "upper": 100, - "b_min": 0, - "b_max": 1 - } - ], - "preprocessing": { - "_target_": "Compose", - "transforms": "$@preprocessing_transforms + @crop_transforms + @final_transforms" - }, - "dataset": { - "_target_": "monai.apps.DecathlonDataset", - "root_dir": "@dataset_dir", - "task": "Task01_BrainTumour", - "section": "validation", - "cache_rate": 0.0, - "num_workers": 8, - "download": false, - "transform": "@preprocessing" - }, - "dataloader": { - "_target_": "DataLoader", - "dataset": "@dataset", - "batch_size": 1, - "shuffle": true, - "num_workers": 0 - }, - "recon_img_pil": "$Image.fromarray(visualize_2d_image(@recon_img), 'RGB')", - "orig_img_pil": "$Image.fromarray(visualize_2d_image(@input_img[0,0,...]), 'RGB')", - "input_img": "$monai.utils.first(@dataloader)['image'].to(@device)", - "recon_img": "$@autoencoder(@input_img)[0][0,0,...]", - "run": [ - "$@create_output_dir", - "$@load_autoencoder", - "$@orig_img_pil.save(@output_dir+'/orig_img_'+@output_postfix+'.png')", - "$@recon_img_pil.save(@output_dir+'/recon_img_'+@output_postfix+'.png')" - ] -} diff --git a/monai/monai.bundle/configs/logging.conf b/monai/monai.bundle/configs/logging.conf deleted file mode 100644 index 91c1a21c..00000000 --- a/monai/monai.bundle/configs/logging.conf +++ /dev/null @@ -1,21 +0,0 @@ -[loggers] -keys=root - -[handlers] -keys=consoleHandler - -[formatters] -keys=fullFormatter - -[logger_root] -level=INFO -handlers=consoleHandler - -[handler_consoleHandler] -class=StreamHandler -level=INFO -formatter=fullFormatter -args=(sys.stdout,) - -[formatter_fullFormatter] -format=%(asctime)s - %(name)s - %(levelname)s - %(message)s diff --git a/monai/monai.bundle/configs/metadata.json b/monai/monai.bundle/configs/metadata.json deleted file mode 100644 index 19be69af..00000000 --- a/monai/monai.bundle/configs/metadata.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "schema": "https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/meta_schema_generator_ldm_20230507.json", - "version": "1.0.6", - "changelog": { - "1.0.6": "update with new lr scheduler api in inference", - "1.0.5": "fix the wrong GPU index issue of multi-node", - "1.0.4": "update with new lr scheduler api", - "1.0.3": "update required packages", - "1.0.2": "remove unused saver in inference", - "1.0.1": "fix inference folder error", - "1.0.0": "Initial release" - }, - "monai_version": "1.2.0", - "pytorch_version": "1.13.1", - "numpy_version": "1.22.2", - "optional_packages_version": { - "nibabel": "5.1.0", - "lpips": "0.1.4", - "monai-generative": "0.2.2" - }, - "name": "BraTS MRI axial slices latent diffusion generation", - "task": "BraTS MRI axial slices synthesis", - "description": "A generative model for creating 2D brain MRI axial slices from Gaussian noise based on BraTS dataset", - "authors": "MONAI team", - "copyright": "Copyright (c) MONAI Consortium", - "data_source": "http://medicaldecathlon.com/", - "data_type": "nibabel", - "image_classes": "Flair brain MRI axial slices with 1x1 mm voxel size", - "eval_metrics": {}, - "intended_use": "This is a research tool/prototype and not to be used clinically", - "references": [], - "autoencoder_data_format": { - "inputs": { - "image": { - "type": "image", - "format": "image", - "num_channels": 1, - "spatial_shape": [ - 240, - 240 - ], - "dtype": "float32", - "value_range": [ - 0, - 1 - ], - "is_patch_data": true - } - }, - "outputs": { - "pred": { - "type": "image", - "format": "image", - "num_channels": 1, - "spatial_shape": [ - 240, - 240 - ], - "dtype": "float32", - "value_range": [ - 0, - 1 - ], - "is_patch_data": true, - "channel_def": { - "0": "image" - } - } - } - }, - "generator_data_format": { - "inputs": { - "latent": { - "type": "noise", - "format": "image", - "num_channels": 1, - "spatial_shape": [ - 64, - 64 - ], - "dtype": "float32", - "value_range": [ - 0, - 1 - ], - "is_patch_data": true - } - }, - "outputs": { - "pred": { - "type": "feature", - "format": "image", - "num_channels": 1, - "spatial_shape": [ - 64, - 64 - ], - "dtype": "float32", - "value_range": [ - 0, - 1 - ], - "is_patch_data": true, - "channel_def": { - "0": "image" - } - } - } - } -} diff --git a/monai/monai.bundle/configs/multi_gpu_train_autoencoder.json b/monai/monai.bundle/configs/multi_gpu_train_autoencoder.json deleted file mode 100644 index 3fe340be..00000000 --- a/monai/monai.bundle/configs/multi_gpu_train_autoencoder.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "device": "$torch.device('cuda:' + os.environ['LOCAL_RANK'])", - "gnetwork": { - "_target_": "torch.nn.parallel.DistributedDataParallel", - "module": "$@autoencoder_def.to(@device)", - "device_ids": [ - "@device" - ], - "find_unused_parameters": true - }, - "dnetwork": { - "_target_": "torch.nn.parallel.DistributedDataParallel", - "module": "$@discriminator_def.to(@device)", - "device_ids": [ - "@device" - ], - "find_unused_parameters": true - }, - "train#sampler": { - "_target_": "DistributedSampler", - "dataset": "@train#dataset", - "even_divisible": true, - "shuffle": true - }, - "train#dataloader#sampler": "@train#sampler", - "train#dataloader#shuffle": false, - "train#trainer#train_handlers": "$@train#handlers[: -2 if dist.get_rank() > 0 else None]", - "initialize": [ - "$import torch.distributed as dist", - "$import os", - "$dist.is_initialized() or dist.init_process_group(backend='nccl')", - "$torch.cuda.set_device(@device)", - "$monai.utils.set_determinism(seed=123)", - "$import logging", - "$@train#trainer.logger.setLevel(logging.WARNING if dist.get_rank() > 0 else logging.INFO)" - ], - "run": [ - "$@train#trainer.run()" - ], - "finalize": [ - "$dist.is_initialized() and dist.destroy_process_group()" - ] -} diff --git a/monai/monai.bundle/configs/multi_gpu_train_diffusion.json b/monai/monai.bundle/configs/multi_gpu_train_diffusion.json deleted file mode 100644 index fbf130d2..00000000 --- a/monai/monai.bundle/configs/multi_gpu_train_diffusion.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "diffusion": { - "_target_": "torch.nn.parallel.DistributedDataParallel", - "module": "$@network_def.to(@device)", - "device_ids": [ - "@device" - ], - "find_unused_parameters": true - }, - "run": [ - "@load_autoencoder", - "$@autoencoder.eval()", - "$print('scale factor:',@scale_factor)", - "$@train#trainer.run()" - ] -} diff --git a/monai/monai.bundle/configs/train_autoencoder.json b/monai/monai.bundle/configs/train_autoencoder.json deleted file mode 100644 index e73c6bb4..00000000 --- a/monai/monai.bundle/configs/train_autoencoder.json +++ /dev/null @@ -1,151 +0,0 @@ -{ - "imports": ["$import functools", "$import glob", "$import scripts"], - "bundle_root": ".", - "device": "$torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')", - "ckpt_dir": "$@bundle_root + '/models'", - "tf_dir": "$@bundle_root + '/eval'", - "dataset_dir": "/workspace/data/medical", - "pretrained": false, - "perceptual_loss_model_weights_path": null, - "train_batch_size_img": 1, - "train_batch_size_slice": 26, - "lr": 5e-05, - "train_patch_size": [240, 240], - "channel": 0, - "spatial_dims": 2, - "image_channels": 1, - "latent_channels": 1, - - "discriminator_def": { - "_target_": "generative.networks.nets.PatchDiscriminator", - "spatial_dims": "@spatial_dims", - "num_layers_d": 3, - "num_channels": 32, - "in_channels": 1, - "out_channels": 1, - "norm": "INSTANCE" - }, - - "autoencoder_def": { - "_target_": "generative.networks.nets.AutoencoderKL", - "spatial_dims": "@spatial_dims", - "in_channels": "@image_channels", - "out_channels": "@image_channels", - "latent_channels": "@latent_channels", - "num_channels": [64, 128, 256], - "num_res_blocks": 2, - "norm_num_groups": 32, - "norm_eps": 1e-06, - "attention_levels": [false, false, false], - "with_encoder_nonlocal_attn": true, - "with_decoder_nonlocal_attn": true - }, - - "perceptual_loss_def": { - "_target_": "generative.losses.PerceptualLoss", - "spatial_dims": "@spatial_dims", - "network_type": "resnet50", - "pretrained": "@pretrained", - "pretrained_path": "@perceptual_loss_model_weights_path", - "pretrained_state_dict_key": "state_dict" - }, - - "dnetwork": "$@discriminator_def.to(@device)", - "gnetwork": "$@autoencoder_def.to(@device)", - "loss_perceptual": "$@perceptual_loss_def.to(@device)", - "doptimizer": {"_target_": "torch.optim.Adam", "params": "$@dnetwork.parameters()", "lr": "@lr"}, - "goptimizer": {"_target_": "torch.optim.Adam", "params": "$@gnetwork.parameters()", "lr": "@lr"}, - - "preprocessing_transforms": [ - {"_target_": "LoadImaged", "keys": "image"}, - {"_target_": "EnsureChannelFirstd", "keys": "image"}, - {"_target_": "Lambdad", "keys": "image", "func": "$lambda x: x[@channel, :, :, :]"}, - {"_target_": "AddChanneld", "keys": "image"}, - {"_target_": "EnsureTyped", "keys": "image"}, - {"_target_": "Orientationd", "keys": "image", "axcodes": "RAS"}, - {"_target_": "CenterSpatialCropd", "keys": "image", "roi_size": "$[@train_patch_size[0], @train_patch_size[1], 100]"}, - {"_target_": "ScaleIntensityRangePercentilesd", "keys": "image", "lower": 0, "upper": 100, "b_min": 0, "b_max": 1} - ], - - "train": { - "crop_transforms": [ - {"_target_": "DivisiblePadd", "keys": "image", "k": [4, 4, 1]}, - - { - "_target_": "RandSpatialCropSamplesd", "keys": "image", "random_size": false, - "roi_size": "$[@train_patch_size[0], @train_patch_size[1], 1]", "num_samples": "@train_batch_size_slice" - }, - - {"_target_": "SqueezeDimd", "keys": "image", "dim": 3}, - {"_target_": "RandFlipd", "keys": ["image"], "prob": 0.5, "spatial_axis": 0}, - {"_target_": "RandFlipd", "keys": ["image"], "prob": 0.5, "spatial_axis": 1} - ], - - "preprocessing": {"_target_": "Compose", "transforms": "$@preprocessing_transforms + @train#crop_transforms"}, - - "dataset": { - "_target_": "monai.apps.DecathlonDataset", - "root_dir": "@dataset_dir", - "task": "Task01_BrainTumour", - "section": "training", - "cache_rate": 1.0, - "num_workers": 8, - "download": false, - "transform": "@train#preprocessing" - }, - "dataloader": { - "_target_": "DataLoader", - "dataset": "@train#dataset", - "batch_size": "@train_batch_size_img", - "shuffle": true, - "num_workers": 0 - }, - "handlers": [ - { - "_target_": "CheckpointSaver", - "save_dir": "@ckpt_dir", - "save_dict": { - "model": "@gnetwork" - }, - "save_interval": 0, - "save_final": true, - "epoch_level": true, - "final_filename": "model_autoencoder.pt" - }, - { - "_target_": "StatsHandler", - "tag_name": "train_loss", - "output_transform": "$lambda x: monai.handlers.from_engine(['g_loss'], first=True)(x)[0]" - }, - { - "_target_": "TensorBoardStatsHandler", - "log_dir": "@tf_dir", - "tag_name": "train_loss", - "output_transform": "$lambda x: monai.handlers.from_engine(['g_loss'], first=True)(x)[0]" - } - ], - "trainer": { - "_target_": "scripts.ldm_trainer.VaeGanTrainer", - "device": "@device", - "max_epochs": 1500, - "train_data_loader": "@train#dataloader", - "g_network": "@gnetwork", - "g_optimizer": "@goptimizer", - "g_loss_function": "$functools.partial(scripts.losses.generator_loss, disc_net=@dnetwork, loss_perceptual=@loss_perceptual)", - "d_network": "@dnetwork", - "d_optimizer": "@doptimizer", - "d_loss_function": "$functools.partial(scripts.losses.discriminator_loss, disc_net=@dnetwork)", - "d_train_steps": 1, - "g_update_latents": true, - "latent_shape": "@latent_channels", - "key_train_metric": "$None", - "train_handlers": "@train#handlers" - } - }, - "initialize": [ - "$monai.utils.set_determinism(seed=0)" - ], - "run": [ - "$@train#trainer.run()" - ] -} diff --git a/monai/monai.bundle/configs/train_diffusion.json b/monai/monai.bundle/configs/train_diffusion.json deleted file mode 100644 index 2aeede23..00000000 --- a/monai/monai.bundle/configs/train_diffusion.json +++ /dev/null @@ -1,174 +0,0 @@ -{ - "ckpt_dir": "$@bundle_root + '/models'", - "train_batch_size_img": 2, - "train_batch_size_slice": 50, - "lr": 5e-05, - "train_patch_size": [ - 256, - 256 - ], - "latent_shape": [ - "@latent_channels", - 64, - 64 - ], - "load_autoencoder_path": "$@bundle_root + '/models/model_autoencoder.pt'", - "load_autoencoder": "$@autoencoder_def.load_state_dict(torch.load(@load_autoencoder_path))", - "autoencoder": "$@autoencoder_def.to(@device)", - "network_def": { - "_target_": "generative.networks.nets.DiffusionModelUNet", - "spatial_dims": "@spatial_dims", - "in_channels": "@latent_channels", - "out_channels": "@latent_channels", - "num_channels": [ - 32, - 64, - 128, - 256 - ], - "attention_levels": [ - false, - true, - true, - true - ], - "num_head_channels": [ - 0, - 32, - 32, - 32 - ], - "num_res_blocks": 2 - }, - "diffusion": "$@network_def.to(@device)", - "optimizer": { - "_target_": "torch.optim.Adam", - "params": "$@diffusion.parameters()", - "lr": "@lr" - }, - "lr_scheduler": { - "_target_": "torch.optim.lr_scheduler.MultiStepLR", - "optimizer": "@optimizer", - "milestones": [ - 1000 - ], - "gamma": 0.1 - }, - "scale_factor": "$scripts.utils.compute_scale_factor(@autoencoder,@train#dataloader,@device)", - "noise_scheduler": { - "_target_": "generative.networks.schedulers.DDPMScheduler", - "_requires_": [ - "@load_autoencoder" - ], - "schedule": "scaled_linear_beta", - "num_train_timesteps": 1000, - "beta_start": 0.0015, - "beta_end": 0.0195 - }, - "loss": { - "_target_": "torch.nn.MSELoss" - }, - "train": { - "inferer": { - "_target_": "generative.inferers.LatentDiffusionInferer", - "scheduler": "@noise_scheduler", - "scale_factor": "@scale_factor" - }, - "crop_transforms": [ - { - "_target_": "DivisiblePadd", - "keys": "image", - "k": [ - 32, - 32, - 1 - ] - }, - { - "_target_": "RandSpatialCropSamplesd", - "keys": "image", - "random_size": false, - "roi_size": "$[@train_patch_size[0], @train_patch_size[1], 1]", - "num_samples": "@train_batch_size_slice" - }, - { - "_target_": "SqueezeDimd", - "keys": "image", - "dim": 3 - } - ], - "preprocessing": { - "_target_": "Compose", - "transforms": "$@preprocessing_transforms + @train#crop_transforms" - }, - "dataset": { - "_target_": "monai.apps.DecathlonDataset", - "root_dir": "@dataset_dir", - "task": "Task01_BrainTumour", - "section": "training", - "cache_rate": 1.0, - "num_workers": 8, - "download": false, - "transform": "@train#preprocessing" - }, - "dataloader": { - "_target_": "DataLoader", - "dataset": "@train#dataset", - "batch_size": "@train_batch_size_img", - "shuffle": true, - "num_workers": 0 - }, - "handlers": [ - { - "_target_": "LrScheduleHandler", - "lr_scheduler": "@lr_scheduler", - "print_lr": true - }, - { - "_target_": "CheckpointSaver", - "save_dir": "@ckpt_dir", - "save_dict": { - "model": "@diffusion" - }, - "save_interval": 0, - "save_final": true, - "epoch_level": true, - "final_filename": "model.pt" - }, - { - "_target_": "StatsHandler", - "tag_name": "train_diffusion_loss", - "output_transform": "$lambda x: monai.handlers.from_engine(['loss'], first=True)(x)" - }, - { - "_target_": "TensorBoardStatsHandler", - "log_dir": "@tf_dir", - "tag_name": "train_diffusion_loss", - "output_transform": "$lambda x: monai.handlers.from_engine(['loss'], first=True)(x)" - } - ], - "trainer": { - "_target_": "scripts.ldm_trainer.LDMTrainer", - "device": "@device", - "max_epochs": 1000, - "train_data_loader": "@train#dataloader", - "network": "@diffusion", - "autoencoder_model": "@autoencoder", - "optimizer": "@optimizer", - "loss_function": "@loss", - "latent_shape": "@latent_shape", - "inferer": "@train#inferer", - "key_train_metric": "$None", - "train_handlers": "@train#handlers" - } - }, - "initialize": [ - "$monai.utils.set_determinism(seed=0)" - ], - "run": [ - "@load_autoencoder", - "$@autoencoder.eval()", - "$print('scale factor:',@scale_factor)", - "$@train#trainer.run()" - ] -} diff --git a/monai/monai.bundle/docs/README.md b/monai/monai.bundle/docs/README.md deleted file mode 100644 index b8129fd2..00000000 --- a/monai/monai.bundle/docs/README.md +++ /dev/null @@ -1,166 +0,0 @@ -# GETTING SET UP -pip install monai -pip install 'monai[all]' - -# Model Overview -A pre-trained model for 2D Latent Diffusion Generative Model on axial slices of BraTS MRI. - -This model is trained on BraTS 2016 and 2017 data from [Medical Decathlon](http://medicaldecathlon.com/), using the Latent diffusion model [1]. - -![model workflow](https://developer.download.nvidia.com/assets/Clara/Images/monai_brain_image_gen_ldm3d_network.png) - -This model is a generator for creating images like the Flair MRIs based on BraTS 2016 and 2017 data. It was trained as a 2d latent diffusion model and accepts Gaussian random noise as inputs to produce an image output. The `train_autoencoder.json` file describes the training process of the variational autoencoder with GAN loss. The `train_diffusion.json` file describes the training process of the 2D latent diffusion model. - -In this bundle, the autoencoder uses perceptual loss, which is based on ResNet50 with pre-trained weights (the network is frozen and will not be trained in the bundle). In default, the `pretrained` parameter is specified as `False` in `train_autoencoder.json`. To ensure correct training, changing the default settings is necessary. There are two ways to utilize pretrained weights: -1. if set `pretrained` to `True`, ImageNet pretrained weights from [torchvision](https://pytorch.org/vision/stable/_modules/torchvision/models/resnet.html#ResNet50_Weights) will be used. However, the weights are for non-commercial use only. -2. if set `pretrained` to `True` and specifies the `perceptual_loss_model_weights_path` parameter, users are able to load weights from a local path. This is the way this bundle used to train, and the pre-trained weights are from some internal data. - -Please note that each user is responsible for checking the data source of the pre-trained models, the applicable licenses, and determining if suitable for the intended use. - -#### Example synthetic image -An example result from inference is shown below: -![Example synthetic image](https://developer.download.nvidia.com/assets/Clara/Images/monai_brain_image_gen_ldm2d_example_generation_v2.png) - -**This is a demonstration network meant to just show the training process for this sort of network with MONAI. To achieve better performance, users need to use larger dataset like [BraTS 2021](https://www.synapse.org/#!Synapse:syn25829067/wiki/610865).** - -## Data -The training data is BraTS 2016 and 2017 from the Medical Segmentation Decathalon. Users can find more details on the dataset (`Task01_BrainTumour`) at http://medicaldecathlon.com/. - -- Target: Image Generation -- Task: Synthesis -- Modality: MRI -- Size: 388 3D MRI volumes (1 channel used) -- Training data size: 38800 2D MRI axial slices (1 channel used) - -## Training Configuration -If you have a GPU with less than 32G of memory, you may need to decrease the batch size when training. To do so, modify the `"train_batch_size_img"` and `"train_batch_size_slice"` parameters in the `configs/train_autoencoder.json` and `configs/train_diffusion.json` configuration files. -- `"train_batch_size_img"` is number of 3D volumes loaded in each batch. -- `"train_batch_size_slice"` is the number of 2D axial slices extracted from each image. The actual batch size is the product of them. - -### Training Configuration of Autoencoder -The autoencoder was trained using the following configuration: - -- GPU: at least 32GB GPU memory -- Actual Model Input: 240 x 240 -- AMP: False -- Optimizer: Adam -- Learning Rate: 5e-5 -- Loss: L1 loss, perceptual loss, KL divergence loss, adversarial loss, GAN BCE loss - -#### Input -1 channel 2D MRI Flair axial patches - -#### Output -- 1 channel 2D MRI reconstructed patches -- 1 channel mean of latent features -- 1 channel standard deviation of latent features - -### Training Configuration of Diffusion Model -The latent diffusion model was trained using the following configuration: - -- GPU: at least 32GB GPU memory -- Actual Model Input: 64 x 64 -- AMP: False -- Optimizer: Adam -- Learning Rate: 5e-5 -- Loss: MSE loss - -#### Training Input -- 1 channel noisy latent features -- a long int that indicates the time step - -#### Training Output -1 channel predicted added noise - -#### Inference Input -1 channel noise - -#### Inference Output -1 channel denoised latent features - -### Memory Consumption Warning - -If you face memory issues with data loading, you can lower the caching rate `cache_rate` in the configurations within range [0, 1] to minimize the System RAM requirements. - -## Performance - -#### Training Loss -![A graph showing the autoencoder training curve](https://developer.download.nvidia.com/assets/Clara/Images/monai_brain_image_gen_ldm2d_train_autoencoder_loss_v3.png) - -![A graph showing the latent diffusion training curve](https://developer.download.nvidia.com/assets/Clara/Images/monai_brain_image_gen_ldm2d_train_diffusion_loss_v3.png) - - -## MONAI Bundle Commands -In addition to the Pythonic APIs, a few command line interfaces (CLI) are provided to interact with the bundle. The CLI supports flexible use cases, such as overriding configs at runtime and predefining arguments in a file. - -For more details usage instructions, visit the [MONAI Bundle Configuration Page](https://docs.monai.io/en/latest/config_syntax.html). - -### Execute Autoencoder Training - -#### Execute Autoencoder Training on single GPU -``` -python -m monai.bundle run --config_file configs/train_autoencoder.json -``` - -Please note that if the default dataset path is not modified with the actual path (it should be the path that contains Task01_BrainTumour) in the bundle config files, you can also override it by using `--dataset_dir`: - -``` -python -m monai.bundle run --config_file configs/train_autoencoder.json --dataset_dir -``` - -#### Override the `train` config to execute multi-GPU training for Autoencoder -To train with multiple GPUs, use the following command, which requires scaling up the learning rate according to the number of GPUs. - -``` -torchrun --standalone --nnodes=1 --nproc_per_node=8 -m monai.bundle run --config_file "['configs/train_autoencoder.json','configs/multi_gpu_train_autoencoder.json']" --lr 4e-4 -``` - -#### Check the Autoencoder Training result -The following code generates a reconstructed image from a random input image. -We can visualize it to see if the autoencoder is trained correctly. -``` -python -m monai.bundle run --config_file configs/inference_autoencoder.json -``` - -An example of reconstructed image from inference is shown below. If the autoencoder is trained correctly, the reconstructed image should look similar to original image. - -![Example reconstructed image](https://developer.download.nvidia.com/assets/Clara/Images/monai_brain_image_gen_ldm2d_recon_example.png) - -### Execute Latent Diffusion Model Training - -#### Execute Latent Diffusion Model Training on single GPU -After training the autoencoder, run the following command to train the latent diffusion model. This command will print out the scale factor of the latent feature space. If your autoencoder is well trained, this value should be close to 1.0. - -``` -python -m monai.bundle run --config_file "['configs/train_autoencoder.json','configs/train_diffusion.json']" -``` - -#### Override the `train` config to execute multi-GPU training for Latent Diffusion Model -To train with multiple GPUs, use the following command, which requires scaling up the learning rate according to the number of GPUs. - -``` -torchrun --standalone --nnodes=1 --nproc_per_node=8 -m monai.bundle run --config_file "['configs/train_autoencoder.json','configs/train_diffusion.json','configs/multi_gpu_train_autoencoder.json','configs/multi_gpu_train_diffusion.json']" --lr 4e-4 -``` -### Execute inference -The following code generates a synthetic image from a random sampled noise. -``` -python -m monai.bundle run --config_file configs/inference.json -``` - -# References -[1] Rombach, Robin, et al. "High-resolution image synthesis with latent diffusion models." Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition. 2022. https://openaccess.thecvf.com/content/CVPR2022/papers/Rombach_High-Resolution_Image_Synthesis_With_Latent_Diffusion_Models_CVPR_2022_paper.pdf - -# License -Copyright (c) MONAI Consortium - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/monai/monai.bundle/docs/data_license.txt b/monai/monai.bundle/docs/data_license.txt deleted file mode 100644 index d3d7e227..00000000 --- a/monai/monai.bundle/docs/data_license.txt +++ /dev/null @@ -1,49 +0,0 @@ -Third Party Licenses ------------------------------------------------------------------------ - -/*********************************************************************/ -i. Multimodal Brain Tumor Segmentation Challenge 2018 - https://www.med.upenn.edu/sbia/brats2018/data.html -/*********************************************************************/ - -Data Usage Agreement / Citations - -You are free to use and/or refer to the BraTS datasets in your own -research, provided that you always cite the following two manuscripts: - -[1] Menze BH, Jakab A, Bauer S, Kalpathy-Cramer J, Farahani K, Kirby -[J, Burren Y, Porz N, Slotboom J, Wiest R, Lanczi L, Gerstner E, Weber -[MA, Arbel T, Avants BB, Ayache N, Buendia P, Collins DL, Cordier N, -[Corso JJ, Criminisi A, Das T, Delingette H, Demiralp Γ, Durst CR, -[Dojat M, Doyle S, Festa J, Forbes F, Geremia E, Glocker B, Golland P, -[Guo X, Hamamci A, Iftekharuddin KM, Jena R, John NM, Konukoglu E, -[Lashkari D, Mariz JA, Meier R, Pereira S, Precup D, Price SJ, Raviv -[TR, Reza SM, Ryan M, Sarikaya D, Schwartz L, Shin HC, Shotton J, -[Silva CA, Sousa N, Subbanna NK, Szekely G, Taylor TJ, Thomas OM, -[Tustison NJ, Unal G, Vasseur F, Wintermark M, Ye DH, Zhao L, Zhao B, -[Zikic D, Prastawa M, Reyes M, Van Leemput K. "The Multimodal Brain -[Tumor Image Segmentation Benchmark (BRATS)", IEEE Transactions on -[Medical Imaging 34(10), 1993-2024 (2015) DOI: -[10.1109/TMI.2014.2377694 - -[2] Bakas S, Akbari H, Sotiras A, Bilello M, Rozycki M, Kirby JS, -[Freymann JB, Farahani K, Davatzikos C. "Advancing The Cancer Genome -[Atlas glioma MRI collections with expert segmentation labels and -[radiomic features", Nature Scientific Data, 4:170117 (2017) DOI: -[10.1038/sdata.2017.117 - -In addition, if there are no restrictions imposed from the -journal/conference you submit your paper about citing "Data -Citations", please be specific and also cite the following: - -[3] Bakas S, Akbari H, Sotiras A, Bilello M, Rozycki M, Kirby J, -[Freymann J, Farahani K, Davatzikos C. "Segmentation Labels and -[Radiomic Features for the Pre-operative Scans of the TCGA-GBM -[collection", The Cancer Imaging Archive, 2017. DOI: -[10.7937/K9/TCIA.2017.KLXWJJ1Q - -[4] Bakas S, Akbari H, Sotiras A, Bilello M, Rozycki M, Kirby J, -[Freymann J, Farahani K, Davatzikos C. "Segmentation Labels and -[Radiomic Features for the Pre-operative Scans of the TCGA-LGG -[collection", The Cancer Imaging Archive, 2017. DOI: -[10.7937/K9/TCIA.2017.GJQ7R0EF diff --git a/monai/monai.bundle/scripts/__init__.py b/monai/monai.bundle/scripts/__init__.py deleted file mode 100644 index 2041a809..00000000 --- a/monai/monai.bundle/scripts/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) MONAI Consortium -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from . import ldm_sampler, ldm_trainer, losses, utils diff --git a/monai/monai.bundle/scripts/ldm_sampler.py b/monai/monai.bundle/scripts/ldm_sampler.py deleted file mode 100644 index 392d3333..00000000 --- a/monai/monai.bundle/scripts/ldm_sampler.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) MONAI Consortium -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -import torch -import torch.nn as nn -from monai.utils import optional_import -from torch.cuda.amp import autocast - -tqdm, has_tqdm = optional_import("tqdm", name="tqdm") - - -class LDMSampler: - def __init__(self) -> None: - super().__init__() - - @torch.no_grad() - def sampling_fn( - self, - input_noise: torch.Tensor, - autoencoder_model: nn.Module, - diffusion_model: nn.Module, - scheduler: nn.Module, - conditioning: torch.Tensor | None = None, - ) -> torch.Tensor: - if has_tqdm: - progress_bar = tqdm(scheduler.timesteps) - else: - progress_bar = iter(scheduler.timesteps) - - image = input_noise - if conditioning is not None: - cond_concat = conditioning.squeeze(1).unsqueeze(-1).unsqueeze(-1).unsqueeze(-1) - cond_concat = cond_concat.expand(list(cond_concat.shape[0:2]) + list(input_noise.shape[2:])) - - for t in progress_bar: - with torch.no_grad(): - if conditioning is not None: - input_t = torch.cat((image, cond_concat), dim=1) - else: - input_t = image - model_output = diffusion_model( - input_t, timesteps=torch.Tensor((t,)).to(input_noise.device).long(), context=conditioning - ) - image, _ = scheduler.step(model_output, t, image) - - with torch.no_grad(): - with autocast(): - sample = autoencoder_model.decode_stage_2_outputs(image) - - return sample diff --git a/monai/monai.bundle/scripts/ldm_trainer.py b/monai/monai.bundle/scripts/ldm_trainer.py deleted file mode 100644 index a298461e..00000000 --- a/monai/monai.bundle/scripts/ldm_trainer.py +++ /dev/null @@ -1,380 +0,0 @@ -# Copyright (c) MONAI Consortium -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any, Callable, Iterable, Sequence - -import torch -from monai.config import IgniteInfo -from monai.engines.utils import IterationEvents, default_metric_cmp_fn, default_prepare_batch -from monai.inferers import Inferer, SimpleInferer -from monai.transforms import Transform -from monai.utils import min_version, optional_import -from monai.utils.enums import CommonKeys, GanKeys -from torch.optim.optimizer import Optimizer -from torch.utils.data import DataLoader - -if TYPE_CHECKING: - from ignite.engine import Engine, EventEnum - from ignite.metrics import Metric -else: - Engine, _ = optional_import("ignite.engine", IgniteInfo.OPT_IMPORT_VERSION, min_version, "Engine") - Metric, _ = optional_import("ignite.metrics", IgniteInfo.OPT_IMPORT_VERSION, min_version, "Metric") - EventEnum, _ = optional_import("ignite.engine", IgniteInfo.OPT_IMPORT_VERSION, min_version, "EventEnum") -from monai.engines.trainer import SupervisedTrainer, Trainer - - -class VaeGanTrainer(Trainer): - """ - Generative adversarial network training based on Goodfellow et al. 2014 https://arxiv.org/abs/1406.266, - inherits from ``Trainer`` and ``Workflow``. - Training Loop: for each batch of data size `m` - 1. Generate `m` fakes from random latent codes. - 2. Update discriminator with these fakes and current batch reals, repeated d_train_steps times. - 3. If g_update_latents, generate `m` fakes from new random latent codes. - 4. Update generator with these fakes using discriminator feedback. - Args: - device: an object representing the device on which to run. - max_epochs: the total epoch number for engine to run. - train_data_loader: Core ignite engines uses `DataLoader` for training loop batchdata. - g_network: generator (G) network architecture. - g_optimizer: G optimizer function. - g_loss_function: G loss function for optimizer. - d_network: discriminator (D) network architecture. - d_optimizer: D optimizer function. - d_loss_function: D loss function for optimizer. - epoch_length: number of iterations for one epoch, default to `len(train_data_loader)`. - g_inferer: inference method to execute G model forward. Defaults to ``SimpleInferer()``. - d_inferer: inference method to execute D model forward. Defaults to ``SimpleInferer()``. - d_train_steps: number of times to update D with real data minibatch. Defaults to ``1``. - latent_shape: size of G input latent code. Defaults to ``64``. - non_blocking: if True and this copy is between CPU and GPU, the copy may occur asynchronously - with respect to the host. For other cases, this argument has no effect. - d_prepare_batch: callback function to prepare batchdata for D inferer. - Defaults to return ``GanKeys.REALS`` in batchdata dict. for more details please refer to: - https://pytorch.org/ignite/generated/ignite.engine.create_supervised_trainer.html. - g_prepare_batch: callback function to create batch of latent input for G inferer. - Defaults to return random latents. for more details please refer to: - https://pytorch.org/ignite/generated/ignite.engine.create_supervised_trainer.html. - g_update_latents: Calculate G loss with new latent codes. Defaults to ``True``. - iteration_update: the callable function for every iteration, expect to accept `engine` - and `engine.state.batch` as inputs, return data will be stored in `engine.state.output`. - if not provided, use `self._iteration()` instead. for more details please refer to: - https://pytorch.org/ignite/generated/ignite.engine.engine.Engine.html. - postprocessing: execute additional transformation for the model output data. - Typically, several Tensor based transforms composed by `Compose`. - key_train_metric: compute metric when every iteration completed, and save average value to - engine.state.metrics when epoch completed. key_train_metric is the main metric to compare and save the - checkpoint into files. - additional_metrics: more Ignite metrics that also attach to Ignite Engine. - metric_cmp_fn: function to compare current key metric with previous best key metric value, - it must accept 2 args (current_metric, previous_best) and return a bool result: if `True`, will update - `best_metric` and `best_metric_epoch` with current metric and epoch, default to `greater than`. - train_handlers: every handler is a set of Ignite Event-Handlers, must have `attach` function, like: - CheckpointHandler, StatsHandler, etc. - decollate: whether to decollate the batch-first data to a list of data after model computation, - recommend `decollate=True` when `postprocessing` uses components from `monai.transforms`. - default to `True`. - optim_set_to_none: when calling `optimizer.zero_grad()`, instead of setting to zero, set the grads to None. - more details: https://pytorch.org/docs/stable/generated/torch.optim.Optimizer.zero_grad.html. - to_kwargs: dict of other args for `prepare_batch` API when converting the input data, except for - `device`, `non_blocking`. - amp_kwargs: dict of the args for `torch.cuda.amp.autocast()` API, for more details: - https://pytorch.org/docs/stable/amp.html#torch.cuda.amp.autocast. - """ - - def __init__( - self, - device: str | torch.device, - max_epochs: int, - train_data_loader: DataLoader, - g_network: torch.nn.Module, - g_optimizer: Optimizer, - g_loss_function: Callable, - d_network: torch.nn.Module, - d_optimizer: Optimizer, - d_loss_function: Callable, - epoch_length: int | None = None, - g_inferer: Inferer | None = None, - d_inferer: Inferer | None = None, - d_train_steps: int = 1, - latent_shape: int = 64, - non_blocking: bool = False, - d_prepare_batch: Callable = default_prepare_batch, - g_prepare_batch: Callable = default_prepare_batch, - g_update_latents: bool = True, - iteration_update: Callable[[Engine, Any], Any] | None = None, - postprocessing: Transform | None = None, - key_train_metric: dict[str, Metric] | None = None, - additional_metrics: dict[str, Metric] | None = None, - metric_cmp_fn: Callable = default_metric_cmp_fn, - train_handlers: Sequence | None = None, - decollate: bool = True, - optim_set_to_none: bool = False, - to_kwargs: dict | None = None, - amp_kwargs: dict | None = None, - ): - if not isinstance(train_data_loader, DataLoader): - raise ValueError("train_data_loader must be PyTorch DataLoader.") - - # set up Ignite engine and environments - super().__init__( - device=device, - max_epochs=max_epochs, - data_loader=train_data_loader, - epoch_length=epoch_length, - non_blocking=non_blocking, - prepare_batch=d_prepare_batch, - iteration_update=iteration_update, - key_metric=key_train_metric, - additional_metrics=additional_metrics, - metric_cmp_fn=metric_cmp_fn, - handlers=train_handlers, - postprocessing=postprocessing, - decollate=decollate, - to_kwargs=to_kwargs, - amp_kwargs=amp_kwargs, - ) - self.g_network = g_network - self.g_optimizer = g_optimizer - self.g_loss_function = g_loss_function - self.g_inferer = SimpleInferer() if g_inferer is None else g_inferer - self.d_network = d_network - self.d_optimizer = d_optimizer - self.d_loss_function = d_loss_function - self.d_inferer = SimpleInferer() if d_inferer is None else d_inferer - self.d_train_steps = d_train_steps - self.latent_shape = latent_shape - self.g_prepare_batch = g_prepare_batch - self.g_update_latents = g_update_latents - self.optim_set_to_none = optim_set_to_none - - def _iteration( - self, engine: VaeGanTrainer, batchdata: dict | Sequence - ) -> dict[str, torch.Tensor | int | float | bool]: - """ - Callback function for Adversarial Training processing logic of 1 iteration in Ignite Engine. - Args: - engine: `VaeGanTrainer` to execute operation for an iteration. - batchdata: input data for this iteration, usually can be dictionary or tuple of Tensor data. - Raises: - ValueError: must provide batch data for current iteration. - """ - if batchdata is None: - raise ValueError("must provide batch data for current iteration.") - - d_input = engine.prepare_batch(batchdata, engine.state.device, engine.non_blocking, **engine.to_kwargs)[0] - g_input = d_input - g_output, z_mu, z_sigma = engine.g_inferer(g_input, engine.g_network) - - # Train Generator - engine.g_optimizer.zero_grad(set_to_none=engine.optim_set_to_none) - g_loss = engine.g_loss_function(g_output, g_input, z_mu, z_sigma) - g_loss.backward() - engine.g_optimizer.step() - - # Train Discriminator - d_total_loss = torch.zeros(1) - for _ in range(engine.d_train_steps): - engine.d_optimizer.zero_grad(set_to_none=engine.optim_set_to_none) - dloss = engine.d_loss_function(g_output, d_input) - dloss.backward() - engine.d_optimizer.step() - d_total_loss += dloss.item() - - return { - GanKeys.REALS: d_input, - GanKeys.FAKES: g_output, - GanKeys.LATENTS: g_input, - GanKeys.GLOSS: g_loss.item(), - GanKeys.DLOSS: d_total_loss.item(), - } - - -class LDMTrainer(SupervisedTrainer): - """ - Standard supervised training method with image and label, inherits from ``Trainer`` and ``Workflow``. - Args: - device: an object representing the device on which to run. - max_epochs: the total epoch number for trainer to run. - train_data_loader: Ignite engine use data_loader to run, must be Iterable or torch.DataLoader. - network: network to train in the trainer, should be regular PyTorch `torch.nn.Module`. - optimizer: the optimizer associated to the network, should be regular PyTorch optimizer from `torch.optim` - or its subclass. - loss_function: the loss function associated to the optimizer, should be regular PyTorch loss, - which inherit from `torch.nn.modules.loss`. - epoch_length: number of iterations for one epoch, default to `len(train_data_loader)`. - non_blocking: if True and this copy is between CPU and GPU, the copy may occur asynchronously - with respect to the host. For other cases, this argument has no effect. - prepare_batch: function to parse expected data (usually `image`, `label` and other network args) - from `engine.state.batch` for every iteration, for more details please refer to: - https://pytorch.org/ignite/generated/ignite.engine.create_supervised_trainer.html. - iteration_update: the callable function for every iteration, expect to accept `engine` - and `engine.state.batch` as inputs, return data will be stored in `engine.state.output`. - if not provided, use `self._iteration()` instead. for more details please refer to: - https://pytorch.org/ignite/generated/ignite.engine.engine.Engine.html. - inferer: inference method that execute model forward on input data, like: SlidingWindow, etc. - postprocessing: execute additional transformation for the model output data. - Typically, several Tensor based transforms composed by `Compose`. - key_train_metric: compute metric when every iteration completed, and save average value to - engine.state.metrics when epoch completed. key_train_metric is the main metric to compare and save the - checkpoint into files. - additional_metrics: more Ignite metrics that also attach to Ignite Engine. - metric_cmp_fn: function to compare current key metric with previous best key metric value, - it must accept 2 args (current_metric, previous_best) and return a bool result: if `True`, will update - `best_metric` and `best_metric_epoch` with current metric and epoch, default to `greater than`. - train_handlers: every handler is a set of Ignite Event-Handlers, must have `attach` function, like: - CheckpointHandler, StatsHandler, etc. - amp: whether to enable auto-mixed-precision training, default is False. - event_names: additional custom ignite events that will register to the engine. - new events can be a list of str or `ignite.engine.events.EventEnum`. - event_to_attr: a dictionary to map an event to a state attribute, then add to `engine.state`. - for more details, check: https://pytorch.org/ignite/generated/ignite.engine.engine.Engine.html - #ignite.engine.engine.Engine.register_events. - decollate: whether to decollate the batch-first data to a list of data after model computation, - recommend `decollate=True` when `postprocessing` uses components from `monai.transforms`. - default to `True`. - optim_set_to_none: when calling `optimizer.zero_grad()`, instead of setting to zero, set the grads to None. - more details: https://pytorch.org/docs/stable/generated/torch.optim.Optimizer.zero_grad.html. - to_kwargs: dict of other args for `prepare_batch` API when converting the input data, except for - `device`, `non_blocking`. - amp_kwargs: dict of the args for `torch.cuda.amp.autocast()` API, for more details: - https://pytorch.org/docs/stable/amp.html#torch.cuda.amp.autocast. - """ - - def __init__( - self, - device: str | torch.device, - max_epochs: int, - train_data_loader: Iterable | DataLoader, - network: torch.nn.Module, - autoencoder_model: torch.nn.Module, - optimizer: Optimizer, - loss_function: Callable, - latent_shape: Sequence, - inferer: Inferer, - epoch_length: int | None = None, - non_blocking: bool = False, - prepare_batch: Callable = default_prepare_batch, - iteration_update: Callable[[Engine, Any], Any] | None = None, - postprocessing: Transform | None = None, - key_train_metric: dict[str, Metric] | None = None, - additional_metrics: dict[str, Metric] | None = None, - metric_cmp_fn: Callable = default_metric_cmp_fn, - train_handlers: Sequence | None = None, - amp: bool = False, - event_names: list[str | EventEnum | type[EventEnum]] | None = None, - event_to_attr: dict | None = None, - decollate: bool = True, - optim_set_to_none: bool = False, - to_kwargs: dict | None = None, - amp_kwargs: dict | None = None, - ) -> None: - super().__init__( - device=device, - max_epochs=max_epochs, - train_data_loader=train_data_loader, - network=network, - optimizer=optimizer, - loss_function=loss_function, - inferer=inferer, - optim_set_to_none=optim_set_to_none, - epoch_length=epoch_length, - non_blocking=non_blocking, - prepare_batch=prepare_batch, - iteration_update=iteration_update, - postprocessing=postprocessing, - key_train_metric=key_train_metric, - additional_metrics=additional_metrics, - metric_cmp_fn=metric_cmp_fn, - train_handlers=train_handlers, - amp=amp, - event_names=event_names, - event_to_attr=event_to_attr, - decollate=decollate, - to_kwargs=to_kwargs, - amp_kwargs=amp_kwargs, - ) - - self.latent_shape = latent_shape - self.autoencoder_model = autoencoder_model - - def _iteration(self, engine: LDMTrainer, batchdata: dict[str, torch.Tensor]) -> dict: - """ - Callback function for the Supervised Training processing logic of 1 iteration in Ignite Engine. - Return below items in a dictionary: - - IMAGE: image Tensor data for model input, already moved to device. - - LABEL: label Tensor data corresponding to the image, already moved to device. - - PRED: prediction result of model. - - LOSS: loss value computed by loss function. - Args: - engine: `SupervisedTrainer` to execute operation for an iteration. - batchdata: input data for this iteration, usually can be dictionary or tuple of Tensor data. - Raises: - ValueError: When ``batchdata`` is None. - """ - if batchdata is None: - raise ValueError("Must provide batch data for current iteration.") - batch = engine.prepare_batch(batchdata, engine.state.device, engine.non_blocking, **engine.to_kwargs) - if len(batch) == 2: - images, labels = batch - args: tuple = () - kwargs: dict = {} - else: - images, labels, args, kwargs = batch - # put iteration outputs into engine.state - engine.state.output = {CommonKeys.IMAGE: images} - - # generate noise - noise_shape = [images.shape[0]] + list(self.latent_shape) - noise = torch.randn(noise_shape, dtype=images.dtype).to(images.device) - engine.state.output = {"noise": noise} - - # Create timesteps - timesteps = torch.randint( - 0, engine.inferer.scheduler.num_train_timesteps, (images.shape[0],), device=images.device - ).long() - - def _compute_pred_loss(): - # predicted noise - engine.state.output[CommonKeys.PRED] = engine.inferer( - inputs=images, - autoencoder_model=self.autoencoder_model, - diffusion_model=engine.network, - noise=noise, - timesteps=timesteps, - ) - engine.fire_event(IterationEvents.FORWARD_COMPLETED) - # compute loss - engine.state.output[CommonKeys.LOSS] = engine.loss_function( - engine.state.output[CommonKeys.PRED], noise - ).mean() - engine.fire_event(IterationEvents.LOSS_COMPLETED) - - engine.network.train() - engine.optimizer.zero_grad(set_to_none=engine.optim_set_to_none) - - if engine.amp and engine.scaler is not None: - with torch.cuda.amp.autocast(**engine.amp_kwargs): - _compute_pred_loss() - engine.scaler.scale(engine.state.output[CommonKeys.LOSS]).backward() - engine.fire_event(IterationEvents.BACKWARD_COMPLETED) - engine.scaler.step(engine.optimizer) - engine.scaler.update() - else: - _compute_pred_loss() - engine.state.output[CommonKeys.LOSS].backward() - engine.fire_event(IterationEvents.BACKWARD_COMPLETED) - engine.optimizer.step() - engine.fire_event(IterationEvents.MODEL_COMPLETED) - - return engine.state.output diff --git a/monai/monai.bundle/scripts/losses.py b/monai/monai.bundle/scripts/losses.py deleted file mode 100644 index 9c824bd4..00000000 --- a/monai/monai.bundle/scripts/losses.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (c) MONAI Consortium -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and - -import torch -from generative.losses import PatchAdversarialLoss - -intensity_loss = torch.nn.L1Loss() -adv_loss = PatchAdversarialLoss(criterion="least_squares") - -adv_weight = 0.5 -perceptual_weight = 1.0 -# kl_weight: important hyper-parameter. -# If too large, decoder cannot recon good results from latent space. -# If too small, latent space will not be regularized enough for the diffusion model -kl_weight = 1e-6 - - -def compute_kl_loss(z_mu, z_sigma): - kl_loss = 0.5 * torch.sum( - z_mu.pow(2) + z_sigma.pow(2) - torch.log(z_sigma.pow(2)) - 1, dim=list(range(1, len(z_sigma.shape))) - ) - return torch.sum(kl_loss) / kl_loss.shape[0] - - -def generator_loss(gen_images, real_images, z_mu, z_sigma, disc_net, loss_perceptual): - recons_loss = intensity_loss(gen_images, real_images) - kl_loss = compute_kl_loss(z_mu, z_sigma) - p_loss = loss_perceptual(gen_images.float(), real_images.float()) - loss_g = recons_loss + kl_weight * kl_loss + perceptual_weight * p_loss - - logits_fake = disc_net(gen_images)[-1] - generator_loss = adv_loss(logits_fake, target_is_real=True, for_discriminator=False) - loss_g = loss_g + adv_weight * generator_loss - - return loss_g - - -def discriminator_loss(gen_images, real_images, disc_net): - logits_fake = disc_net(gen_images.contiguous().detach())[-1] - loss_d_fake = adv_loss(logits_fake, target_is_real=False, for_discriminator=True) - logits_real = disc_net(real_images.contiguous().detach())[-1] - loss_d_real = adv_loss(logits_real, target_is_real=True, for_discriminator=True) - discriminator_loss = (loss_d_fake + loss_d_real) * 0.5 - loss_d = adv_weight * discriminator_loss - return loss_d diff --git a/monai/monai.bundle/scripts/utils.py b/monai/monai.bundle/scripts/utils.py deleted file mode 100644 index 5a5a5aa8..00000000 --- a/monai/monai.bundle/scripts/utils.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) MONAI Consortium -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and - -import numpy as np -import torch -from monai.utils import first -from monai.utils.type_conversion import convert_to_numpy - - -def compute_scale_factor(autoencoder, train_loader, device): - with torch.no_grad(): - check_data = first(train_loader) - z = autoencoder.encode_stage_2_inputs(check_data["image"].to(device)) - scale_factor = 1 / torch.std(z) - return scale_factor.item() - - -def normalize_image_to_uint8(image): - """ - Normalize image to uint8 - Args: - image: numpy array - """ - draw_img = image - if np.amin(draw_img) < 0: - draw_img[draw_img < 0] = 0 - if np.amax(draw_img) > 0.1: - draw_img /= np.amax(draw_img) - draw_img = (255 * draw_img).astype(np.uint8) - return draw_img - - -def visualize_2d_image(image): - """ - Prepare a 2D image for visualization. - Args: - image: image numpy array, sized (H, W) - """ - image = convert_to_numpy(image) - # draw image - draw_img = normalize_image_to_uint8(image) - draw_img = np.stack([draw_img, draw_img, draw_img], axis=-1) - return draw_img diff --git a/monai/brats_mri_2d_v1/brats_mri_2d_diff.isc b/monai_brats_mri_2d/brats_mri_2d_diff.isc similarity index 82% rename from monai/brats_mri_2d_v1/brats_mri_2d_diff.isc rename to monai_brats_mri_2d/brats_mri_2d_diff.isc index 9b90b185..3220f07b 100644 --- a/monai/brats_mri_2d_v1/brats_mri_2d_diff.isc +++ b/monai_brats_mri_2d/brats_mri_2d_diff.isc @@ -3,4 +3,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_brats_mri_2d_diff" -command="train_cycling_diff.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc, --gen-load-path ~/output_brats_mri_2d_gen/exp_1645/checkpoint.isc --tboard-path ~/output_brats_mri_2d_diff/tb" \ No newline at end of file +command="train_cycling_diff.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc, --gen-load-path ~/output_brats_mri_2d_gen/exp_1791/checkpoint.isc --tboard-path ~/output_brats_mri_2d_diff/tb" \ No newline at end of file diff --git a/monai/brats_mri_2d_v1/brats_mri_2d_gen.isc b/monai_brats_mri_2d/brats_mri_2d_gen.isc similarity index 100% rename from monai/brats_mri_2d_v1/brats_mri_2d_gen.isc rename to monai_brats_mri_2d/brats_mri_2d_gen.isc diff --git a/monai/brats_mri_2d_v1/loops.py b/monai_brats_mri_2d/loops.py similarity index 95% rename from monai/brats_mri_2d_v1/loops.py rename to monai_brats_mri_2d/loops.py index 99a2c4d6..d60d5cb7 100644 --- a/monai/brats_mri_2d_v1/loops.py +++ b/monai_brats_mri_2d/loops.py @@ -232,7 +232,7 @@ def train_diffusion_one_epoch( generator.eval() train_step = train_sampler.progress // train_loader.batch_size - total_steps = int(len(train_sampler) / train_loader.batch_size) + total_steps = len(train_sampler) // train_loader.batch_size print(f'\nTraining / resuming epoch {epoch} from training step {train_step}\n') for step, batch in enumerate(train_loader): @@ -264,11 +264,11 @@ def train_diffusion_one_epoch( timer.report(f'train batch {train_step} unet backward') # Reduce metrics accross nodes - metrics["train"].update({"train_images_seen":len(images), "epoch_loss":loss.item()}) + metrics["train"].update({"images_seen":len(images), "loss":loss.item()}) metrics["train"].reduce() - recons_loss = metrics["train"].local["epoch_loss"] / metrics["train"].local["train_images_seen"] - print("Epoch [{}] Step [{}/{}] :: recons_loss: {:,.3f}".format(epoch, train_step, total_steps, recons_loss)) + recons_loss = metrics["train"].local["loss"] / metrics["train"].local["images_seen"] + print("Epoch [{}] Step [{}/{}] :: loss: {:,.3f}".format(epoch, train_step, total_steps, recons_loss)) metrics["train"].reset_local() @@ -285,7 +285,7 @@ def train_diffusion_one_epoch( if utils.is_main_process() and train_step % 1 == 0: # Checkpointing every batch writer = SummaryWriter(log_dir=args.tboard_path) - writer.add_scalar("Train/recons_loss", recons_loss, train_step + epoch * total_steps) + writer.add_scalar("Train/loss", recons_loss, train_step + epoch * total_steps) writer.flush() writer.close() @@ -305,7 +305,7 @@ def train_diffusion_one_epoch( } timer = atomic_torch_save(checkpoint, args.resume, timer) - train_loss = metrics["train"].epoch_reports[-1]["epoch_loss"] / metrics["train"].epoch_reports[-1]["train_images_seen"] + train_loss = metrics["train"].epoch_reports[-1]["loss"] / metrics["train"].epoch_reports[-1]["images_seen"] print("Epoch [{}] :: epoch_loss: {:,.3f}".format(epoch, train_loss)) return unet, timer, metrics @@ -342,7 +342,7 @@ def evaluate_diffusion( loss = F.mse_loss(noise_pred.float(), noise.float()) timer.report(f'eval batch {val_step} loss') - metrics["val"].update({"val_images_seen": len(images), "val_loss": loss.item()}) + metrics["val"].update({"images_seen": len(images), "loss": loss.item()}) metrics["val"].reduce() metrics["val"].reset_local() @@ -351,6 +351,7 @@ def evaluate_diffusion( ## Checkpointing print(f"Saving checkpoint at epoch {epoch} val batch {val_step}") val_sampler.advance(len(images)) + val_step = val_sampler.progress // val_loader.batch_size if val_step == total_steps: metrics["val"].end_epoch() @@ -374,7 +375,7 @@ def evaluate_diffusion( timer = atomic_torch_save(checkpoint, args.resume, timer) # val_loss = metrics["val"].agg[metrics["val"].map["val_loss"]] / metrics["val"].agg[metrics["val"].map["val_images_seen"]] - val_loss = metrics["val"].epoch_reports[-1]["val_loss"] / metrics["val"].epoch_reports[-1]["val_images_seen"] + val_loss = metrics["val"].epoch_reports[-1]["loss"] / metrics["val"].epoch_reports[-1]["images_seen"] if utils.is_main_process(): writer = SummaryWriter(log_dir=args.tboard_path) writer.add_scalar("Val/loss", val_loss, epoch) diff --git a/monai/brats_mri_2d_v1/prep.py b/monai_brats_mri_2d/prep.py similarity index 100% rename from monai/brats_mri_2d_v1/prep.py rename to monai_brats_mri_2d/prep.py diff --git a/monai/brats_mri_2d_v1/train_cycling_diff.py b/monai_brats_mri_2d/train_cycling_diff.py similarity index 93% rename from monai/brats_mri_2d_v1/train_cycling_diff.py rename to monai_brats_mri_2d/train_cycling_diff.py index 7d57f684..7721abb7 100644 --- a/monai/brats_mri_2d_v1/train_cycling_diff.py +++ b/monai_brats_mri_2d/train_cycling_diff.py @@ -1,6 +1,6 @@ -from cycling_utils import Timer +from cycling_utils import TimestampedTimer -timer = Timer() +timer = TimestampedTimer() timer.report('importing Timer') import os @@ -100,8 +100,8 @@ def main(args, timer): timer.report('build samplers') - # Original trainer had batch size = 2 * 50. Using 10 nodes x batch size 10 => eff batch size = 100 - train_loader = DataLoader(train_ds, batch_size=10, sampler=train_sampler, num_workers=1) + # Original trainer had batch size = 2 * 50. Using 11 nodes x 6 GPUs x batch size 2 => eff batch size = 132 + train_loader = DataLoader(train_ds, batch_size=2, sampler=train_sampler, num_workers=1) val_loader = DataLoader(val_ds, batch_size=1, sampler=val_sampler, num_workers=1) # check_data = first(train_loader) # Used later @@ -109,9 +109,10 @@ def main(args, timer): # Auto-encoder definition generator = AutoencoderKL( - spatial_dims=2, in_channels=1, out_channels=1, num_channels=(128, 128, 256), latent_channels=3, - num_res_blocks=2, attention_levels=(False, False, False),with_encoder_nonlocal_attn=False, - with_decoder_nonlocal_attn=False, + spatial_dims=2, in_channels=1, out_channels=1, num_channels=(64, 128, 256), + latent_channels=1, num_res_blocks=2, norm_num_groups=32, norm_eps=1e-06, + attention_levels=(False, False, False), with_encoder_nonlocal_attn=True, + with_decoder_nonlocal_attn=True, ) # saved_generator_checkpoint = torch.load("/output_brats_mri_2d_gen/exp_1645/checkpoint.isc", map_location="cpu") saved_generator_checkpoint = torch.load(args.gen_load_path, map_location="cpu") @@ -200,7 +201,7 @@ def main(args, timer): print('\n') with train_sampler.in_epoch(epoch): - timer = Timer("Start training") + timer = TimestampedTimer("Start training") unet, timer, metrics = train_diffusion_one_epoch( args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, train_sampler, val_sampler, lr_scheduler, device, timer, metrics @@ -209,7 +210,7 @@ def main(args, timer): if epoch % diff_val_interval == 0: with val_sampler.in_epoch(epoch): - timer = Timer("Start evaluation") + timer = TimestampedTimer("Start evaluation") timer, metrics = evaluate_diffusion( args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, train_sampler, val_sampler, lr_scheduler, device, timer, metrics diff --git a/monai/brats_mri_2d_v1/train_cycling_gen.py b/monai_brats_mri_2d/train_cycling_gen.py similarity index 100% rename from monai/brats_mri_2d_v1/train_cycling_gen.py rename to monai_brats_mri_2d/train_cycling_gen.py diff --git a/monai/brats_mri_2d_v0/utils.py b/monai_brats_mri_2d/utils.py similarity index 100% rename from monai/brats_mri_2d_v0/utils.py rename to monai_brats_mri_2d/utils.py diff --git a/tv-detection/engine.py b/tv-detection/engine.py index 625b8b1e..54b76712 100644 --- a/tv-detection/engine.py +++ b/tv-detection/engine.py @@ -4,6 +4,7 @@ import torch import torchvision.models.detection.mask_rcnn +import torch.distributed as dist import utils from coco_eval import CocoEvaluator from coco_utils import get_coco_api_from_dataset @@ -28,14 +29,25 @@ def train_one_epoch( images = list(image.to(device) for image in images) targets = [{k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in t.items()} for t in targets] timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: moving batch data to device') - print(f"First 2 image shapes: {images[0].shape}, {images[1].shape}") + # print(f"First 2 image shapes: {images[0].shape}, {images[1].shape}") optimizer.zero_grad() with torch.cuda.amp.autocast(enabled=scaler is not None): loss_dict = model(images, targets) + + # CHECK IF NUMERIC ERROR HAS OCCURRED AND IF SO, SKIP THIS BATCH + check_0 = 1 if torch.tensor([torch.isnan(v) for v in loss_dict.values()]).any() else 0 + check_1 = 1 if not all([math.isfinite(v) for v in loss_dict.values()]) else 0 + check_tensor = torch.tensor([check_0, check_1], requires_grad=False, device=device) + dist.all_reduce(check_tensor, op=dist.ReduceOp.SUM) + if check_tensor.sum() > 0: + print(f"CONTINUE CONDITION: {[e for e in check_tensor]}") + train_sampler.advance() # Advance sampler to try next batch + continue + losses = sum(loss for loss in loss_dict.values()) - timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: forward pass') + timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: forward pass') if scaler is not None: scaler.scale(losses).backward() diff --git a/tv-detection/retinanet_resnet50_fpn.isc b/tv-detection/retinanet_resnet50_fpn.isc deleted file mode 100644 index c1165d81..00000000 --- a/tv-detection/retinanet_resnet50_fpn.isc +++ /dev/null @@ -1,7 +0,0 @@ -experiment_name="retinanet_resnet50_fpn" -gpu_type="24GB VRAM GPU" -nnodes = 11 -venv_path = "~/.venv/bin/activate" -output_path = "~/output_retinanet_resnet50_fpn" -command="train_cycling.py --dataset coco --model retinanet_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --lr 0.01 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_retinanet_resnet50_fpn/tb" - diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index 181cc80e..fa698f56 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -17,9 +17,9 @@ Because the number of images is smaller in the person keypoint subset of COCO, the number of epochs should be adapted so that we have the same number of iterations. """ -from cycling_utils import Timer +from cycling_utils import TimestampedTimer -timer = Timer() +timer = TimestampedTimer() timer.report('importing Timer') import os @@ -281,7 +281,7 @@ def main(args, timer): print('\n') with train_sampler.in_epoch(epoch): - timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch + timer = TimestampedTimer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch model, timer, metrics = train_one_epoch( model, optimizer, data_loader_train, train_sampler, test_sampler, lr_scheduler, warmup_lr_scheduler, args, device, coco_evaluator, epoch, scaler, timer, metrics @@ -289,7 +289,7 @@ def main(args, timer): # NEST THE TEST SAMPLER IN TRAIN SAMPLER CONTEXT TO AVOID EPOCH RESTART? with test_sampler.in_epoch(epoch): - timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch + timer = TimestampedTimer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch coco_evaluator, timer, metrics = evaluate( model, data_loader_test, epoch, test_sampler, args, coco_evaluator, optimizer, lr_scheduler, warmup_lr_scheduler, train_sampler, device, scaler, timer, metrics @@ -344,7 +344,7 @@ def get_args_parser(add_help=True): parser.add_argument("--amp", action="store_true", help="Use torch.cuda.amp for mixed precision training") # Use CopyPaste augmentation training parameter - parser.add_argument("--use-copypaste",action="store_true",help="Use CopyPaste data augmentation. Works only with data-augmentation='lsj'.",) + parser.add_argument("--use-copypaste", action="store_true",help="Use CopyPaste data augmentation. Works only with data-augmentation='lsj'.",) parser.add_argument("--backend", default="PIL", type=str.lower, help="PIL or tensor - case insensitive") parser.add_argument("--use-v2", action="store_true", help="Use V2 transforms") diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index 558c483d..c1ff52f9 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -1,6 +1,6 @@ -from cycling_utils import Timer +from cycling_utils import TimestampedTimer -timer = Timer() +timer = TimestampedTimer() timer.report('importing Timer') import os @@ -176,7 +176,7 @@ def evaluate( "test_sampler": test_sampler.state_dict(), "confmat": confmat.mat, # For storing eval metric "confmat_temp": confmat.temp_mat, # For storing eval metric - "train_metrics": train_metrics, + "metrics": metrics, } if args.amp: checkpoint["scaler"] = scaler.state_dict() @@ -372,7 +372,7 @@ def main(args, timer): print('\n') with train_sampler.in_epoch(epoch): - timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch + timer = TimestampedTimer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch model, timer, metrics = train_one_epoch( args, model, criterion, optimizer, data_loader_train, train_sampler, test_sampler, confmat, lr_scheduler, @@ -382,7 +382,7 @@ def main(args, timer): # NEST TEST SAMPLER IN TRAIN SAMPLER CONTEXT TO AVOID EPOCH RESTART? with test_sampler.in_epoch(epoch): - timer = Timer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch + timer = TimestampedTimer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch confmat, timer, metrics = evaluate( args, model, data_loader_test, num_classes, confmat, optimizer, lr_scheduler, train_sampler, test_sampler, From 62e5360e5e80035930477b62f1faef427bc3cc0a Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Thu, 21 Sep 2023 16:01:27 +1000 Subject: [PATCH 36/44] just in case --- monai_brats_mri_2d/train_cycling_diff.py | 22 +- monai_brats_mri_2d/train_cycling_gen.py | 8 +- tv-detection/SS/COCO_edits/COCO_interface.py | 521 ------------------ tv-detection/SS/COCO_edits/coco_utils_SC.py | 234 -------- .../SS/COCO_edits/group_by_aspect_ratio_SC.py | 198 ------- tv-detection/SS/maskrcnn_resnet50_fpn.isc | 6 - tv-detection/SS/retinanet_resnet50_fpn.isc | 7 - 7 files changed, 11 insertions(+), 985 deletions(-) delete mode 100644 tv-detection/SS/COCO_edits/COCO_interface.py delete mode 100644 tv-detection/SS/COCO_edits/coco_utils_SC.py delete mode 100644 tv-detection/SS/COCO_edits/group_by_aspect_ratio_SC.py delete mode 100644 tv-detection/SS/maskrcnn_resnet50_fpn.isc delete mode 100644 tv-detection/SS/retinanet_resnet50_fpn.isc diff --git a/monai_brats_mri_2d/train_cycling_diff.py b/monai_brats_mri_2d/train_cycling_diff.py index 7721abb7..13768e1a 100644 --- a/monai_brats_mri_2d/train_cycling_diff.py +++ b/monai_brats_mri_2d/train_cycling_diff.py @@ -153,7 +153,13 @@ def main(args, timer): # Init metric tracker metrics = {'train': MetricsTracker(), 'val': MetricsTracker()} - # RETRIEVE GENERATOR CHECKPOINT FROM PREVIOUS JOB + # Prepare LatentDiffusionInferer + + scale_factor = compute_scale_factor(generator, train_loader, device) + scheduler = DDPMScheduler(num_train_timesteps=1000, schedule="scaled_linear_beta", beta_start=0.0015, beta_end=0.0195) + inferer = LatentDiffusionInferer(scheduler, scale_factor=scale_factor) + + timer.report('building inferer') # RETRIEVE CHECKPOINT Path(args.resume).parent.mkdir(parents=True, exist_ok=True) @@ -180,20 +186,6 @@ def main(args, timer): n_diff_epochs = 200 diff_val_interval = 1 - # Prepare LatentDiffusionInferer - - # with torch.no_grad(): - # with autocast(enabled=True): - # z = generator.encode_stage_2_inputs(check_data["image"].to(device)) - # scale_factor = 1 / torch.std(z) - - scale_factor = compute_scale_factor(generator, train_loader, device) - - scheduler = DDPMScheduler(num_train_timesteps=1000, schedule="scaled_linear_beta", beta_start=0.0015, beta_end=0.0195) - inferer = LatentDiffusionInferer(scheduler, scale_factor=scale_factor) - - timer.report('building inferer') - for epoch in range(args.start_epoch, n_diff_epochs): print('\n') diff --git a/monai_brats_mri_2d/train_cycling_gen.py b/monai_brats_mri_2d/train_cycling_gen.py index 2a55c1f6..7d3683bb 100644 --- a/monai_brats_mri_2d/train_cycling_gen.py +++ b/monai_brats_mri_2d/train_cycling_gen.py @@ -1,6 +1,6 @@ -from cycling_utils import Timer +from cycling_utils import TimestampedTimer -timer = Timer() +timer = TimestampedTimer() timer.report('importing Timer') import os @@ -184,7 +184,7 @@ def main(args, timer): print('\n') with train_sampler.in_epoch(epoch): - timer = Timer("Start training") + timer = TimestampedTimer("Start training") generator, timer, metrics = train_generator_one_epoch( args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, metrics @@ -193,7 +193,7 @@ def main(args, timer): if epoch % gen_val_interval == 0: # Eval every epoch with val_sampler.in_epoch(epoch): - timer = Timer("Start evaluation") + timer = TimestampedTimer("Start evaluation") timer, metrics = evaluate_generator( args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, metrics diff --git a/tv-detection/SS/COCO_edits/COCO_interface.py b/tv-detection/SS/COCO_edits/COCO_interface.py deleted file mode 100644 index c1b48eb0..00000000 --- a/tv-detection/SS/COCO_edits/COCO_interface.py +++ /dev/null @@ -1,521 +0,0 @@ - -import os.path -from typing import Any, Callable, List, Optional, Tuple - -from PIL import Image - -from torchvision.datasets import VisionDataset - - -class torch_CocoDetection(VisionDataset): - """`MS Coco Detection `_ Dataset. - - It requires the `COCO API to be installed `_. - - Args: - root (string): Root directory where images are downloaded to. - annFile (string): Path to json annotation file. - transform (callable, optional): A function/transform that takes in an PIL image - and returns a transformed version. E.g, ``transforms.PILToTensor`` - target_transform (callable, optional): A function/transform that takes in the - target and transforms it. - transforms (callable, optional): A function/transform that takes input sample and its target as entry - and returns a transformed version. - """ - - def __init__( - self, - root: str, - annFile: str, - transform: Optional[Callable] = None, - target_transform: Optional[Callable] = None, - transforms: Optional[Callable] = None, - ) -> None: - super().__init__(root, transforms, transform, target_transform) - # from pycocotools.coco import COCO - - self.coco = COCO(annFile) - self.ids = list(sorted(self.coco.imgs.keys())) - - def _load_image(self, id: int) -> Image.Image: - path = self.coco.loadImgs(id)[0]["file_name"] - return Image.open(os.path.join(self.root, path)).convert("RGB") - - def _load_target(self, id: int) -> List[Any]: - return self.coco.loadAnns(self.coco.getAnnIds(id)) - - def __getitem__(self, index: int) -> Tuple[Any, Any]: - id = self.ids[index] - image = self._load_image(id) - target = self._load_target(id) - - if self.transforms is not None: - image, target = self.transforms(image, target) - - return image, target - - - def __len__(self) -> int: - return len(self.ids) - - - - -__author__ = 'tylin' -__version__ = '2.0' -# Interface for accessing the Microsoft COCO dataset. - -# Microsoft COCO is a large image dataset designed for object detection, -# segmentation, and caption generation. pycocotools is a Python API that -# assists in loading, parsing and visualizing the annotations in COCO. -# Please visit http://mscoco.org/ for more information on COCO, including -# for the data, paper, and tutorials. The exact format of the annotations -# is also described on the COCO website. For example usage of the pycocotools -# please see pycocotools_demo.ipynb. In addition to this API, please download both -# the COCO images and annotations in order to run the demo. - -# An alternative to using the API is to load the annotations directly -# into Python dictionary -# Using the API provides additional utility functions. Note that this API -# supports both *instance* and *caption* annotations. In the case of -# captions not all functions are defined (e.g. categories are undefined). - -# The following API functions are defined: -# COCO - COCO api class that loads COCO annotation file and prepare data structures. -# decodeMask - Decode binary mask M encoded via run-length encoding. -# encodeMask - Encode binary mask M using run-length encoding. -# getAnnIds - Get ann ids that satisfy given filter conditions. -# getCatIds - Get cat ids that satisfy given filter conditions. -# getImgIds - Get img ids that satisfy given filter conditions. -# loadAnns - Load anns with the specified ids. -# loadCats - Load cats with the specified ids. -# loadImgs - Load imgs with the specified ids. -# annToMask - Convert segmentation in an annotation to binary mask. -# showAnns - Display the specified annotations. -# loadRes - Load algorithm results and create API for accessing them. -# download - Download COCO images from mscoco.org server. -# Throughout the API "ann"=annotation, "cat"=category, and "img"=image. -# Help on each functions can be accessed by: "help COCO>function". - -# See also COCO>decodeMask, -# COCO>encodeMask, COCO>getAnnIds, COCO>getCatIds, -# COCO>getImgIds, COCO>loadAnns, COCO>loadCats, -# COCO>loadImgs, COCO>annToMask, COCO>showAnns - -# Microsoft COCO Toolbox. version 2.0 -# Data, paper, and tutorials available at: http://mscoco.org/ -# Code written by Piotr Dollar and Tsung-Yi Lin, 2014. -# Licensed under the Simplified BSD License [see bsd.txt] - -import json -import time -import matplotlib.pyplot as plt -from matplotlib.collections import PatchCollection -from matplotlib.patches import Polygon -import numpy as np -import copy -import itertools -from pycocotools import mask as maskUtils -import os -from collections import defaultdict -from itertools import groupby -import sys -PYTHON_VERSION = sys.version_info[0] -if PYTHON_VERSION == 2: - from urllib import urlretrieve -elif PYTHON_VERSION == 3: - from urllib.request import urlretrieve - - -def _isArrayLike(obj): - return hasattr(obj, '__iter__') and hasattr(obj, '__len__') - - -class COCO: - def __init__(self, annotation_file=None): - """ - Constructor of Microsoft COCO helper class for reading and visualizing annotations. - :param annotation_file (str): location of annotation file - :param image_folder (str): location to the folder that hosts images. - :return: - """ - # load dataset - self.dataset,self.anns,self.cats,self.imgs = dict(),dict(),dict(),dict() - self.imgToAnns, self.catToImgs = defaultdict(list), defaultdict(list) - if not annotation_file == None: - print('loading annotations into memory...') - tic = time.time() - dataset = json.load(open(annotation_file, 'r')) - assert type(dataset)==dict, 'annotation file format {} not supported'.format(type(dataset)) - print('Done (t={:0.2f}s)'.format(time.time()- tic)) - self.dataset = dataset - self.createIndex() - - def createIndex(self): - # create index - print('creating index...') - # anns, cats, imgs = {}, {}, {} - # imgToAnns,catToImgs = defaultdict(list),defaultdict(list) - - # self.dataset = { - # 'images': [{'id': ...}], - # 'annotations': [{'image_id': ..., 'id': ..., }, {...}, ...], - # 'categories': [...], - # } - - if 'annotations' in self.dataset: - # for ann in self.dataset['annotations']: - # imgToAnns[ann['image_id']].append(ann) - # anns[ann['id']] = ann - - anns_by_iid = groupby(self.dataset['annotations'], lambda ann: ann['image_id']) - self.imgToAnns = {iid:anns_group for iid,anns_group in anns_by_iid} - self.anns = {ann['id']:ann for ann in self.dataset['annotations']} - - if 'images' in self.dataset: - # for img in self.dataset['images']: - # imgs[img['id']] = img - self.imgs = {img['id']:img for img in self.dataset['images']} - - if 'categories' in self.dataset: - # for cat in self.dataset['categories']: - # cats[cat['id']] = cat - self.cats = {cat['id']:cat for cat in self.dataset['categories']} - - if 'annotations' in self.dataset and 'categories' in self.dataset: - # for ann in self.dataset['annotations']: - # catToImgs[ann['category_id']].append(ann['image_id']) - anns_by_cid = groupby(self.dataset['annotations'], lambda ann: ann['category_id']) - self.catToImgs = {cid:anns_group for cid,anns_group in anns_by_cid} - - - print('index created!') - - # create class members - # self.anns = anns - # self.imgToAnns = imgToAnns - # self.catToImgs = catToImgs - # self.imgs = imgs - # self.cats = cats - - def info(self): - """ - Print information about the annotation file. - :return: - """ - for key, value in self.dataset['info'].items(): - print('{}: {}'.format(key, value)) - - def getAnnIds(self, imgIds=[], catIds=[], areaRng=[], iscrowd=None): - """ - Get ann ids that satisfy given filter conditions. default skips that filter - :param imgIds (int array) : get anns for given imgs - catIds (int array) : get anns for given cats - areaRng (float array) : get anns for given area range (e.g. [0 inf]) - iscrowd (boolean) : get anns for given crowd label (False or True) - :return: ids (int array) : integer array of ann ids - """ - imgIds = imgIds if _isArrayLike(imgIds) else [imgIds] - catIds = catIds if _isArrayLike(catIds) else [catIds] - - if len(imgIds) == len(catIds) == len(areaRng) == 0: - anns = self.dataset['annotations'] - else: - if not len(imgIds) == 0: - lists = [self.imgToAnns[imgId] for imgId in imgIds if imgId in self.imgToAnns] - anns = list(itertools.chain.from_iterable(lists)) - else: - anns = self.dataset['annotations'] - anns = anns if len(catIds) == 0 else [ann for ann in anns if ann['category_id'] in catIds] - anns = anns if len(areaRng) == 0 else [ann for ann in anns if ann['area'] > areaRng[0] and ann['area'] < areaRng[1]] - if not iscrowd == None: - ids = [ann['id'] for ann in anns if ann['iscrowd'] == iscrowd] - else: - ids = [ann['id'] for ann in anns] - return ids - - def getCatIds(self, catNms=[], supNms=[], catIds=[]): - """ - filtering parameters. default skips that filter. - :param catNms (str array) : get cats for given cat names - :param supNms (str array) : get cats for given supercategory names - :param catIds (int array) : get cats for given cat ids - :return: ids (int array) : integer array of cat ids - """ - catNms = catNms if _isArrayLike(catNms) else [catNms] - supNms = supNms if _isArrayLike(supNms) else [supNms] - catIds = catIds if _isArrayLike(catIds) else [catIds] - - if len(catNms) == len(supNms) == len(catIds) == 0: - cats = self.dataset['categories'] - else: - cats = self.dataset['categories'] - cats = cats if len(catNms) == 0 else [cat for cat in cats if cat['name'] in catNms] - cats = cats if len(supNms) == 0 else [cat for cat in cats if cat['supercategory'] in supNms] - cats = cats if len(catIds) == 0 else [cat for cat in cats if cat['id'] in catIds] - ids = [cat['id'] for cat in cats] - return ids - - def getImgIds(self, imgIds=[], catIds=[]): - ''' - Get img ids that satisfy given filter conditions. - :param imgIds (int array) : get imgs for given ids - :param catIds (int array) : get imgs with all given cats - :return: ids (int array) : integer array of img ids - ''' - imgIds = imgIds if _isArrayLike(imgIds) else [imgIds] - catIds = catIds if _isArrayLike(catIds) else [catIds] - - if len(imgIds) == len(catIds) == 0: - ids = self.imgs.keys() - else: - ids = set(imgIds) - for i, catId in enumerate(catIds): - if i == 0 and len(ids) == 0: - ids = set(self.catToImgs[catId]) - else: - ids &= set(self.catToImgs[catId]) - return list(ids) - - def loadAnns(self, ids=[]): - """ - Load anns with the specified ids. - :param ids (int array) : integer ids specifying anns - :return: anns (object array) : loaded ann objects - """ - if _isArrayLike(ids): - return [self.anns[id] for id in ids] - elif type(ids) == int: - return [self.anns[ids]] - - def loadCats(self, ids=[]): - """ - Load cats with the specified ids. - :param ids (int array) : integer ids specifying cats - :return: cats (object array) : loaded cat objects - """ - if _isArrayLike(ids): - return [self.cats[id] for id in ids] - elif type(ids) == int: - return [self.cats[ids]] - - def loadImgs(self, ids=[]): - """ - Load anns with the specified ids. - :param ids (int array) : integer ids specifying img - :return: imgs (object array) : loaded img objects - """ - if _isArrayLike(ids): - return [self.imgs[id] for id in ids] - elif type(ids) == int: - return [self.imgs[ids]] - - def showAnns(self, anns, draw_bbox=False): - """ - Display the specified annotations. - :param anns (array of object): annotations to display - :return: None - """ - if len(anns) == 0: - return 0 - if 'segmentation' in anns[0] or 'keypoints' in anns[0]: - datasetType = 'instances' - elif 'caption' in anns[0]: - datasetType = 'captions' - else: - raise Exception('datasetType not supported') - if datasetType == 'instances': - ax = plt.gca() - ax.set_autoscale_on(False) - polygons = [] - color = [] - for ann in anns: - c = (np.random.random((1, 3))*0.6+0.4).tolist()[0] - if 'segmentation' in ann: - if type(ann['segmentation']) == list: - # polygon - for seg in ann['segmentation']: - poly = np.array(seg).reshape((int(len(seg)/2), 2)) - polygons.append(Polygon(poly)) - color.append(c) - else: - # mask - t = self.imgs[ann['image_id']] - if type(ann['segmentation']['counts']) == list: - rle = maskUtils.frPyObjects([ann['segmentation']], t['height'], t['width']) - else: - rle = [ann['segmentation']] - m = maskUtils.decode(rle) - img = np.ones( (m.shape[0], m.shape[1], 3) ) - if ann['iscrowd'] == 1: - color_mask = np.array([2.0,166.0,101.0])/255 - if ann['iscrowd'] == 0: - color_mask = np.random.random((1, 3)).tolist()[0] - for i in range(3): - img[:,:,i] = color_mask[i] - ax.imshow(np.dstack( (img, m*0.5) )) - if 'keypoints' in ann and type(ann['keypoints']) == list: - # turn skeleton into zero-based index - sks = np.array(self.loadCats(ann['category_id'])[0]['skeleton'])-1 - kp = np.array(ann['keypoints']) - x = kp[0::3] - y = kp[1::3] - v = kp[2::3] - for sk in sks: - if np.all(v[sk]>0): - plt.plot(x[sk],y[sk], linewidth=3, color=c) - plt.plot(x[v>0], y[v>0],'o',markersize=8, markerfacecolor=c, markeredgecolor='k',markeredgewidth=2) - plt.plot(x[v>1], y[v>1],'o',markersize=8, markerfacecolor=c, markeredgecolor=c, markeredgewidth=2) - - if draw_bbox: - [bbox_x, bbox_y, bbox_w, bbox_h] = ann['bbox'] - poly = [[bbox_x, bbox_y], [bbox_x, bbox_y+bbox_h], [bbox_x+bbox_w, bbox_y+bbox_h], [bbox_x+bbox_w, bbox_y]] - np_poly = np.array(poly).reshape((4,2)) - polygons.append(Polygon(np_poly)) - color.append(c) - - p = PatchCollection(polygons, facecolor=color, linewidths=0, alpha=0.4) - ax.add_collection(p) - p = PatchCollection(polygons, facecolor='none', edgecolors=color, linewidths=2) - ax.add_collection(p) - elif datasetType == 'captions': - for ann in anns: - print(ann['caption']) - - def loadRes(self, resFile): - """ - Load result file and return a result api object. - :param resFile (str) : file name of result file - :return: res (obj) : result api object - """ - res = COCO() - res.dataset['images'] = [img for img in self.dataset['images']] - - print('Loading and preparing results...') - tic = time.time() - if type(resFile) == str or (PYTHON_VERSION == 2 and type(resFile) == unicode): - anns = json.load(open(resFile)) - elif type(resFile) == np.ndarray: - anns = self.loadNumpyAnnotations(resFile) - else: - anns = resFile - assert type(anns) == list, 'results in not an array of objects' - annsImgIds = [ann['image_id'] for ann in anns] - assert set(annsImgIds) == (set(annsImgIds) & set(self.getImgIds())), \ - 'Results do not correspond to current coco set' - if 'caption' in anns[0]: - imgIds = set([img['id'] for img in res.dataset['images']]) & set([ann['image_id'] for ann in anns]) - res.dataset['images'] = [img for img in res.dataset['images'] if img['id'] in imgIds] - for id, ann in enumerate(anns): - ann['id'] = id+1 - elif 'bbox' in anns[0] and not anns[0]['bbox'] == []: - res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) - for id, ann in enumerate(anns): - bb = ann['bbox'] - x1, x2, y1, y2 = [bb[0], bb[0]+bb[2], bb[1], bb[1]+bb[3]] - if not 'segmentation' in ann: - ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]] - ann['area'] = bb[2]*bb[3] - ann['id'] = id+1 - ann['iscrowd'] = 0 - elif 'segmentation' in anns[0]: - res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) - for id, ann in enumerate(anns): - # now only support compressed RLE format as segmentation results - ann['area'] = maskUtils.area(ann['segmentation']) - if not 'bbox' in ann: - ann['bbox'] = maskUtils.toBbox(ann['segmentation']) - ann['id'] = id+1 - ann['iscrowd'] = 0 - elif 'keypoints' in anns[0]: - res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) - for id, ann in enumerate(anns): - s = ann['keypoints'] - x = s[0::3] - y = s[1::3] - x0,x1,y0,y1 = np.min(x), np.max(x), np.min(y), np.max(y) - ann['area'] = (x1-x0)*(y1-y0) - ann['id'] = id + 1 - ann['bbox'] = [x0,y0,x1-x0,y1-y0] - print('DONE (t={:0.2f}s)'.format(time.time()- tic)) - - res.dataset['annotations'] = anns - res.createIndex() - return res - - def download(self, tarDir = None, imgIds = [] ): - ''' - Download COCO images from mscoco.org server. - :param tarDir (str): COCO results directory name - imgIds (list): images to be downloaded - :return: - ''' - if tarDir is None: - print('Please specify target directory') - return -1 - if len(imgIds) == 0: - imgs = self.imgs.values() - else: - imgs = self.loadImgs(imgIds) - N = len(imgs) - if not os.path.exists(tarDir): - os.makedirs(tarDir) - for i, img in enumerate(imgs): - tic = time.time() - fname = os.path.join(tarDir, img['file_name']) - if not os.path.exists(fname): - urlretrieve(img['coco_url'], fname) - print('downloaded {}/{} images (t={:0.1f}s)'.format(i, N, time.time()- tic)) - - def loadNumpyAnnotations(self, data): - """ - Convert result data from a numpy array [Nx7] where each row contains {imageID,x1,y1,w,h,score,class} - :param data (numpy.ndarray) - :return: annotations (python nested list) - """ - print('Converting ndarray to lists...') - assert(type(data) == np.ndarray) - print(data.shape) - assert(data.shape[1] == 7) - N = data.shape[0] - ann = [] - for i in range(N): - if i % 1000000 == 0: - print('{}/{}'.format(i,N)) - ann += [{ - 'image_id' : int(data[i, 0]), - 'bbox' : [ data[i, 1], data[i, 2], data[i, 3], data[i, 4] ], - 'score' : data[i, 5], - 'category_id': int(data[i, 6]), - }] - return ann - - def annToRLE(self, ann): - """ - Convert annotation which can be polygons, uncompressed RLE to RLE. - :return: binary mask (numpy 2D array) - """ - t = self.imgs[ann['image_id']] - h, w = t['height'], t['width'] - segm = ann['segmentation'] - if type(segm) == list: - # polygon -- a single object might consist of multiple parts - # we merge all parts into one mask rle code - rles = maskUtils.frPyObjects(segm, h, w) - rle = maskUtils.merge(rles) - elif type(segm['counts']) == list: - # uncompressed RLE - rle = maskUtils.frPyObjects(segm, h, w) - else: - # rle - rle = ann['segmentation'] - return rle - - def annToMask(self, ann): - """ - Convert annotation which can be polygons, uncompressed RLE, or RLE to binary mask. - :return: binary mask (numpy 2D array) - """ - rle = self.annToRLE(ann) - m = maskUtils.decode(rle) - return m \ No newline at end of file diff --git a/tv-detection/SS/COCO_edits/coco_utils_SC.py b/tv-detection/SS/COCO_edits/coco_utils_SC.py deleted file mode 100644 index a807e3a8..00000000 --- a/tv-detection/SS/COCO_edits/coco_utils_SC.py +++ /dev/null @@ -1,234 +0,0 @@ -import os - -import torch -import torch.utils.data -import torchvision -import transforms as T -from pycocotools import mask as coco_mask -from COCO_interface import torch_CocoDetection, COCO - - -def convert_coco_poly_to_mask(segmentations, height, width): - masks = [] - for polygons in segmentations: - rles = coco_mask.frPyObjects(polygons, height, width) - mask = coco_mask.decode(rles) - if len(mask.shape) < 3: - mask = mask[..., None] - mask = torch.as_tensor(mask, dtype=torch.uint8) - mask = mask.any(dim=2) - masks.append(mask) - if masks: - masks = torch.stack(masks, dim=0) - else: - masks = torch.zeros((0, height, width), dtype=torch.uint8) - return masks - - -class ConvertCocoPolysToMask: - def __call__(self, image, target): - w, h = image.size - - image_id = target["image_id"] - - anno = target["annotations"] - - anno = [obj for obj in anno if obj["iscrowd"] == 0] - - boxes = [obj["bbox"] for obj in anno] - # guard against no boxes via resizing - boxes = torch.as_tensor(boxes, dtype=torch.float32).reshape(-1, 4) - boxes[:, 2:] += boxes[:, :2] - boxes[:, 0::2].clamp_(min=0, max=w) - boxes[:, 1::2].clamp_(min=0, max=h) - - classes = [obj["category_id"] for obj in anno] - classes = torch.tensor(classes, dtype=torch.int64) - - segmentations = [obj["segmentation"] for obj in anno] - masks = convert_coco_poly_to_mask(segmentations, h, w) - - keypoints = None - if anno and "keypoints" in anno[0]: - keypoints = [obj["keypoints"] for obj in anno] - keypoints = torch.as_tensor(keypoints, dtype=torch.float32) - num_keypoints = keypoints.shape[0] - if num_keypoints: - keypoints = keypoints.view(num_keypoints, -1, 3) - - keep = (boxes[:, 3] > boxes[:, 1]) & (boxes[:, 2] > boxes[:, 0]) - boxes = boxes[keep] - classes = classes[keep] - masks = masks[keep] - if keypoints is not None: - keypoints = keypoints[keep] - - target = {} - target["boxes"] = boxes - target["labels"] = classes - target["masks"] = masks - target["image_id"] = image_id - if keypoints is not None: - target["keypoints"] = keypoints - - # for conversion to coco api - area = torch.tensor([obj["area"] for obj in anno]) - iscrowd = torch.tensor([obj["iscrowd"] for obj in anno]) - target["area"] = area - target["iscrowd"] = iscrowd - - return image, target - - -def _coco_remove_images_without_annotations(dataset, cat_list=None): - def _has_only_empty_bbox(anno): - return all(any(o <= 1 for o in obj["bbox"][2:]) for obj in anno) - - def _count_visible_keypoints(anno): - return sum(sum(1 for v in ann["keypoints"][2::3] if v > 0) for ann in anno) - - min_keypoints_per_image = 10 - - def _has_valid_annotation(anno): - # if it's empty, there is no annotation - if len(anno) == 0: - return False - # if all boxes have close to zero area, there is no annotation - if _has_only_empty_bbox(anno): - return False - # keypoints task have a slight different criteria for considering - # if an annotation is valid - if "keypoints" not in anno[0]: - return True - # for keypoint detection tasks, only consider valid images those - # containing at least min_keypoints_per_image - if _count_visible_keypoints(anno) >= min_keypoints_per_image: - return True - return False - - ids = [] - for ds_idx, img_id in enumerate(dataset.ids): - ann_ids = dataset.coco.getAnnIds(imgIds=img_id, iscrowd=None) - anno = dataset.coco.loadAnns(ann_ids) - if cat_list: - anno = [obj for obj in anno if obj["category_id"] in cat_list] - if _has_valid_annotation(anno): - ids.append(ds_idx) - - dataset = torch.utils.data.Subset(dataset, ids) - return dataset - - -def convert_to_coco_api(ds): - coco_ds = COCO() - # annotation IDs need to start at 1, not 0, see torchvision issue #1530 - ann_id = 1 - dataset = {"images": [], "categories": [], "annotations": []} - categories = set() - for img_idx in range(len(ds)): - # find better way to get target - # targets = ds.get_annotations(img_idx) - img, targets = ds[img_idx] - image_id = targets["image_id"] - img_dict = {} - img_dict["id"] = image_id - img_dict["height"] = img.shape[-2] - img_dict["width"] = img.shape[-1] - dataset["images"].append(img_dict) - bboxes = targets["boxes"].clone() - bboxes[:, 2:] -= bboxes[:, :2] - bboxes = bboxes.tolist() - labels = targets["labels"].tolist() - areas = targets["area"].tolist() - iscrowd = targets["iscrowd"].tolist() - if "masks" in targets: - masks = targets["masks"] - # make masks Fortran contiguous for coco_mask - masks = masks.permute(0, 2, 1).contiguous().permute(0, 2, 1) - if "keypoints" in targets: - keypoints = targets["keypoints"] - keypoints = keypoints.reshape(keypoints.shape[0], -1).tolist() - num_objs = len(bboxes) - for i in range(num_objs): - ann = {} - ann["image_id"] = image_id - ann["bbox"] = bboxes[i] - ann["category_id"] = labels[i] - categories.add(labels[i]) - ann["area"] = areas[i] - ann["iscrowd"] = iscrowd[i] - ann["id"] = ann_id - if "masks" in targets: - ann["segmentation"] = coco_mask.encode(masks[i].numpy()) - if "keypoints" in targets: - ann["keypoints"] = keypoints[i] - ann["num_keypoints"] = sum(k != 0 for k in keypoints[i][2::3]) - dataset["annotations"].append(ann) - ann_id += 1 - dataset["categories"] = [{"id": i} for i in sorted(categories)] - coco_ds.dataset = dataset - coco_ds.createIndex() - return coco_ds - - -def get_coco_api_from_dataset(dataset): - # FIXME: This is... awful? - for _ in range(10): - if isinstance(dataset, torch_CocoDetection): - break - if isinstance(dataset, torch.utils.data.Subset): - dataset = dataset.dataset - if isinstance(dataset, torch_CocoDetection): - return dataset.coco - return convert_to_coco_api(dataset) - - -class CocoDetection(torch_CocoDetection): - def __init__(self, img_folder, ann_file, transforms): - super().__init__(img_folder, ann_file) - self._transforms = transforms - - def __getitem__(self, idx): - img, target = super().__getitem__(idx) - image_id = self.ids[idx] - target = dict(image_id=image_id, annotations=target) - if self._transforms is not None: - img, target = self._transforms(img, target) - return img, target - - -def get_coco(root, image_set, transforms, mode="instances", use_v2=False, with_masks=False): - anno_file_template = "{}_{}2017.json" - PATHS = { - "train": ("train2017", os.path.join("annotations", anno_file_template.format(mode, "train"))), - "val": ("val2017", os.path.join("annotations", anno_file_template.format(mode, "val"))), - # "train": ("val2017", os.path.join("annotations", anno_file_template.format(mode, "val"))) - } - - img_folder, ann_file = PATHS[image_set] - img_folder = os.path.join(root, img_folder) - ann_file = os.path.join(root, ann_file) - - if use_v2: - from torchvision.datasets import wrap_dataset_for_transforms_v2 - - dataset = torch_CocoDetection(img_folder, ann_file, transforms=transforms) - target_keys = ["boxes", "labels", "image_id"] - if with_masks: - target_keys += ["masks"] - dataset = wrap_dataset_for_transforms_v2(dataset, target_keys=target_keys) - else: - # TODO: handle with_masks for V1? - t = [ConvertCocoPolysToMask()] - if transforms is not None: - t.append(transforms) - transforms = T.Compose(t) - - dataset = CocoDetection(img_folder, ann_file, transforms=transforms) - - if image_set == "train": - dataset = _coco_remove_images_without_annotations(dataset) - - # dataset = torch.utils.data.Subset(dataset, [i for i in range(500)]) - - return dataset diff --git a/tv-detection/SS/COCO_edits/group_by_aspect_ratio_SC.py b/tv-detection/SS/COCO_edits/group_by_aspect_ratio_SC.py deleted file mode 100644 index 2fe5bfd0..00000000 --- a/tv-detection/SS/COCO_edits/group_by_aspect_ratio_SC.py +++ /dev/null @@ -1,198 +0,0 @@ -import bisect -import copy -import math -from collections import defaultdict -from itertools import chain, repeat - -import numpy as np -import torch -import torch.utils.data -import torchvision -from PIL import Image -from torch.utils.data.sampler import BatchSampler, Sampler -from torch.utils.model_zoo import tqdm - -from COCO_interface import torch_CocoDetection - - -def _repeat_to_at_least(iterable, n): - repeat_times = math.ceil(n / len(iterable)) - repeated = chain.from_iterable(repeat(iterable, repeat_times)) - return list(repeated) - - -class GroupedBatchSampler(BatchSampler): - """ - Wraps another sampler to yield a mini-batch of indices. - It enforces that the batch only contain elements from the same group. - It also tries to provide mini-batches which follows an ordering which is - as close as possible to the ordering from the original sampler. - Args: - sampler (Sampler): Base sampler. - group_ids (list[int]): If the sampler produces indices in range [0, N), - `group_ids` must be a list of `N` ints which contains the group id of each sample. - The group ids must be a continuous set of integers starting from - 0, i.e. they must be in the range [0, num_groups). - batch_size (int): Size of mini-batch. - """ - - def __init__(self, sampler, group_ids, batch_size): - if not isinstance(sampler, Sampler): - raise ValueError(f"sampler should be an instance of torch.utils.data.Sampler, but got sampler={sampler}") - self.sampler = sampler - self.group_ids = group_ids - self.batch_size = batch_size - - def __iter__(self): - buffer_per_group = defaultdict(list) - samples_per_group = defaultdict(list) - - num_batches = 0 - for idx in self.sampler: - group_id = self.group_ids[idx] - buffer_per_group[group_id].append(idx) - samples_per_group[group_id].append(idx) - if len(buffer_per_group[group_id]) == self.batch_size: - yield buffer_per_group[group_id] - num_batches += 1 - del buffer_per_group[group_id] - assert len(buffer_per_group[group_id]) < self.batch_size - - # now we have run out of elements that satisfy - # the group criteria, let's return the remaining - # elements so that the size of the sampler is - # deterministic - expected_num_batches = len(self) - num_remaining = expected_num_batches - num_batches - if num_remaining > 0: - # for the remaining batches, take first the buffers with the largest number - # of elements - for group_id, _ in sorted(buffer_per_group.items(), key=lambda x: len(x[1]), reverse=True): - remaining = self.batch_size - len(buffer_per_group[group_id]) - samples_from_group_id = _repeat_to_at_least(samples_per_group[group_id], remaining) - buffer_per_group[group_id].extend(samples_from_group_id[:remaining]) - assert len(buffer_per_group[group_id]) == self.batch_size - yield buffer_per_group[group_id] - num_remaining -= 1 - if num_remaining == 0: - break - assert num_remaining == 0 - - def __len__(self): - return len(self.sampler) // self.batch_size - - -def _compute_aspect_ratios_slow(dataset, indices=None): - print( - "Your dataset doesn't support the fast path for " - "computing the aspect ratios, so will iterate over " - "the full dataset and load every image instead. " - "This might take some time..." - ) - if indices is None: - indices = range(len(dataset)) - - class SubsetSampler(Sampler): - def __init__(self, indices): - self.indices = indices - - def __iter__(self): - return iter(self.indices) - - def __len__(self): - return len(self.indices) - - sampler = SubsetSampler(indices) - data_loader = torch.utils.data.DataLoader( - dataset, - batch_size=1, - sampler=sampler, - num_workers=14, # you might want to increase it for faster processing - collate_fn=lambda x: x[0], - ) - aspect_ratios = [] - with tqdm(total=len(dataset)) as pbar: - for _i, (img, _) in enumerate(data_loader): - pbar.update(1) - height, width = img.shape[-2:] - aspect_ratio = float(width) / float(height) - aspect_ratios.append(aspect_ratio) - return aspect_ratios - - -def _compute_aspect_ratios_custom_dataset(dataset, indices=None): - if indices is None: - indices = range(len(dataset)) - aspect_ratios = [] - for i in indices: - height, width = dataset.get_height_and_width(i) - aspect_ratio = float(width) / float(height) - aspect_ratios.append(aspect_ratio) - return aspect_ratios - - -def _compute_aspect_ratios_coco_dataset(dataset, indices=None): - if indices is None: - indices = range(len(dataset)) - aspect_ratios = [] - for i in indices: - img_info = dataset.coco.imgs[dataset.ids[i]] - aspect_ratio = float(img_info["width"]) / float(img_info["height"]) - aspect_ratios.append(aspect_ratio) - return aspect_ratios - - -def _compute_aspect_ratios_voc_dataset(dataset, indices=None): - if indices is None: - indices = range(len(dataset)) - aspect_ratios = [] - for i in indices: - # this doesn't load the data into memory, because PIL loads it lazily - width, height = Image.open(dataset.images[i]).size - aspect_ratio = float(width) / float(height) - aspect_ratios.append(aspect_ratio) - return aspect_ratios - - -def _compute_aspect_ratios_subset_dataset(dataset, indices=None): - if indices is None: - indices = range(len(dataset)) - - ds_indices = [dataset.indices[i] for i in indices] - return compute_aspect_ratios(dataset.dataset, ds_indices) - - -def compute_aspect_ratios(dataset, indices=None): - if hasattr(dataset, "get_height_and_width"): - return _compute_aspect_ratios_custom_dataset(dataset, indices) - - if isinstance(dataset, torch_CocoDetection): - return _compute_aspect_ratios_coco_dataset(dataset, indices) - - if isinstance(dataset, torchvision.datasets.VOCDetection): - return _compute_aspect_ratios_voc_dataset(dataset, indices) - - if isinstance(dataset, torch.utils.data.Subset): - return _compute_aspect_ratios_subset_dataset(dataset, indices) - - # slow path - return _compute_aspect_ratios_slow(dataset, indices) - - -def _quantize(x, bins): - bins = copy.deepcopy(bins) - bins = sorted(bins) - quantized = list(map(lambda y: bisect.bisect_right(bins, y), x)) - return quantized - - -def create_aspect_ratio_groups(dataset, k=0): - aspect_ratios = compute_aspect_ratios(dataset) # list of aspect ratios for each image in the dataset - bins = (2 ** np.linspace(-1, 1, 2 * k + 1)).tolist() if k > 0 else [1.0] - groups = _quantize(aspect_ratios, bins) # list of bin indexes to which each image belongs - # count number of elements per group - counts = np.unique(groups, return_counts=True)[1] - fbins = [0] + bins + [np.inf] - print(f"Using {fbins} as bins for aspect ratio quantization") - print(f"Count of instances per bin: {counts}") - return groups diff --git a/tv-detection/SS/maskrcnn_resnet50_fpn.isc b/tv-detection/SS/maskrcnn_resnet50_fpn.isc deleted file mode 100644 index 4cc8c061..00000000 --- a/tv-detection/SS/maskrcnn_resnet50_fpn.isc +++ /dev/null @@ -1,6 +0,0 @@ -experiment_name="maskrcnn_resnet50_fpn" -gpu_type="24GB VRAM GPU" -nnodes = 11 -venv_path = "~/.venv/bin/activate" -output_path = "~/output_maskrcnn_resnet50_fpn" -command="train_cycling.py --dataset coco --model maskrcnn_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_maskrcnn_resnet50_fpn/tb" \ No newline at end of file diff --git a/tv-detection/SS/retinanet_resnet50_fpn.isc b/tv-detection/SS/retinanet_resnet50_fpn.isc deleted file mode 100644 index f97baa3a..00000000 --- a/tv-detection/SS/retinanet_resnet50_fpn.isc +++ /dev/null @@ -1,7 +0,0 @@ -experiment_name="retinanet_resnet50_fpn" -gpu_type="24GB VRAM GPU" -nnodes = 11 -venv_path = "~/.venv/bin/activate" -output_path = "~/output_retinanet_resnet50_fpn" -command="train_cycling.py --dataset coco --model retinanet_resnet50_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --lr 0.01 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_retinanet_resnet50_fpn/tb" - From 43e7bd1066b258d68495c0a2b9dbc719c9b33eba Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Fri, 22 Sep 2023 09:32:58 +1000 Subject: [PATCH 37/44] fixed mess thank you Calvin! --- monai_brats_mri_2d/brats_mri_2d_diff.isc | 2 +- monai_brats_mri_2d/brats_mri_2d_gen.isc | 2 +- .../exp_1855 (autoencoder)/rank_0.txt | 12925 ++++++++++++++++ .../exp_1855 (autoencoder)/rank_1.txt | 1310 ++ .../exp_1855 (autoencoder)/rank_10.txt | 1310 ++ .../exp_1855 (autoencoder)/rank_2.txt | 1310 ++ .../exp_1855 (autoencoder)/rank_3.txt | 1310 ++ .../exp_1855 (autoencoder)/rank_4.txt | 1310 ++ .../exp_1855 (autoencoder)/rank_5.txt | 1310 ++ .../exp_1855 (autoencoder)/rank_6.txt | 1310 ++ .../exp_1855 (autoencoder)/rank_7.txt | 1310 ++ .../exp_1855 (autoencoder)/rank_8.txt | 1310 ++ .../exp_1855 (autoencoder)/rank_9.txt | 1310 ++ ...nts.out.tfevents.1695275223.sc14.2163737.0 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695275225.sc14.2163737.1 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695275228.sc14.2163737.2 | Bin 0 -> 132 bytes ...nts.out.tfevents.1695275232.sc14.2163737.3 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695275235.sc14.2163737.4 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695275238.sc14.2163737.5 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695275242.sc14.2163737.6 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695275246.sc14.2163737.7 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695275249.sc14.2163737.8 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695275253.sc14.2163737.9 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695275257.sc14.2163737.10 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695275259.sc14.2163737.11 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695275264.sc14.2163737.12 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695275267.sc14.2163737.13 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695275270.sc14.2163737.14 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695275274.sc14.2163737.15 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695275278.sc14.2163737.16 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695275280.sc14.2163737.17 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695275285.sc14.2163737.18 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695275288.sc14.2163737.19 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695275975.sc12.3473404.0 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695275980.sc12.3473404.1 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695275983.sc12.3473404.2 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695275986.sc12.3473404.3 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695275991.sc12.3473404.4 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695275995.sc12.3473404.5 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695275997.sc12.3473404.6 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695276002.sc12.3473404.7 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695276006.sc12.3473404.8 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695276009.sc12.3473404.9 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695276013.sc12.3473404.10 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695276016.sc12.3473404.11 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695276019.sc12.3473404.12 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695276024.sc12.3473404.13 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695276027.sc12.3473404.14 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695276030.sc12.3473404.15 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695276035.sc12.3473404.16 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695276038.sc12.3473404.17 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695276041.sc12.3473404.18 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695276756.sc13.1777137.0 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695276758.sc13.1777137.1 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695276761.sc13.1777137.2 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695276765.sc13.1777137.3 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695276769.sc13.1777137.4 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695276772.sc13.1777137.5 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695276776.sc13.1777137.6 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695276780.sc13.1777137.7 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695276783.sc13.1777137.8 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695276787.sc13.1777137.9 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695276791.sc13.1777137.10 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695276793.sc13.1777137.11 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695276798.sc13.1777137.12 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695276801.sc13.1777137.13 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695276804.sc13.1777137.14 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695276809.sc13.1777137.15 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695276812.sc13.1777137.16 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695276815.sc13.1777137.17 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695276819.sc13.1777137.18 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695276822.sc13.1777137.19 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695277599.sc12.4183255.0 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695277604.sc12.4183255.1 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695277608.sc12.4183255.2 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695277610.sc12.4183255.3 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695277615.sc12.4183255.4 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695277619.sc12.4183255.5 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695277622.sc12.4183255.6 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695277626.sc12.4183255.7 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695277630.sc12.4183255.8 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695277632.sc12.4183255.9 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695277637.sc12.4183255.10 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695277641.sc12.4183255.11 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695277644.sc12.4183255.12 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695277648.sc12.4183255.13 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695277652.sc12.4183255.14 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695277655.sc12.4183255.15 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695277659.sc12.4183255.16 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695277663.sc12.4183255.17 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695277666.sc12.4183255.18 | Bin 0 -> 134 bytes ...ents.out.tfevents.1695278445.sc20.758883.0 | Bin 0 -> 193 bytes ...ents.out.tfevents.1695278447.sc20.758883.1 | Bin 0 -> 193 bytes ...ents.out.tfevents.1695278450.sc20.758883.2 | Bin 0 -> 134 bytes ...ents.out.tfevents.1695278455.sc20.758883.3 | Bin 0 -> 193 bytes ...ents.out.tfevents.1695278458.sc20.758883.4 | Bin 0 -> 193 bytes ...ents.out.tfevents.1695278461.sc20.758883.5 | Bin 0 -> 134 bytes ...ents.out.tfevents.1695278466.sc20.758883.6 | Bin 0 -> 193 bytes ...ents.out.tfevents.1695278469.sc20.758883.7 | Bin 0 -> 193 bytes ...ents.out.tfevents.1695278472.sc20.758883.8 | Bin 0 -> 134 bytes ...ents.out.tfevents.1695278476.sc20.758883.9 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695278480.sc20.758883.10 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695278483.sc20.758883.11 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695278487.sc20.758883.12 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695278491.sc20.758883.13 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695278493.sc20.758883.14 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695278498.sc20.758883.15 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695278501.sc20.758883.16 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695278504.sc20.758883.17 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695278508.sc20.758883.18 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695278512.sc20.758883.19 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695279205.sc10.1505340.0 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695279210.sc10.1505340.1 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695279213.sc10.1505340.2 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695279216.sc10.1505340.3 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695279220.sc10.1505340.4 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695279224.sc10.1505340.5 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695279227.sc10.1505340.6 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695279231.sc10.1505340.7 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695279234.sc10.1505340.8 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695279237.sc10.1505340.9 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695279242.sc10.1505340.10 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695279245.sc10.1505340.11 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695279248.sc10.1505340.12 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695279252.sc10.1505340.13 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695279255.sc10.1505340.14 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695279258.sc10.1505340.15 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695279263.sc10.1505340.16 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695279266.sc10.1505340.17 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695279269.sc10.1505340.18 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695279273.sc10.1505340.19 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695279943.sc18.3937612.0 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695279946.sc18.3937612.1 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695279950.sc18.3937612.2 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695279954.sc18.3937612.3 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695279956.sc18.3937612.4 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695279961.sc18.3937612.5 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695279964.sc18.3937612.6 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695279967.sc18.3937612.7 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695279971.sc18.3937612.8 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695279975.sc18.3937612.9 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695279978.sc18.3937612.10 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695279982.sc18.3937612.11 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695279985.sc18.3937612.12 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695279988.sc18.3937612.13 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695279993.sc18.3937612.14 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695279996.sc18.3937612.15 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695279999.sc18.3937612.16 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695280003.sc18.3937612.17 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695280006.sc18.3937612.18 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695280009.sc18.3937612.19 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695280680.sc10.1562555.0 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695280682.sc10.1562555.1 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695280685.sc10.1562555.2 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695280689.sc10.1562555.3 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695280693.sc10.1562555.4 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695280696.sc10.1562555.5 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695280700.sc10.1562555.6 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695280704.sc10.1562555.7 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695280707.sc10.1562555.8 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695280711.sc10.1562555.9 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695280714.sc10.1562555.10 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695280717.sc10.1562555.11 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695280722.sc10.1562555.12 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695280725.sc10.1562555.13 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695280728.sc10.1562555.14 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695280733.sc10.1562555.15 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695280736.sc10.1562555.16 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695280739.sc10.1562555.17 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695280743.sc10.1562555.18 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695281501.sc13.2274570.0 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695281504.sc13.2274570.1 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695281509.sc13.2274570.2 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695281512.sc13.2274570.3 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695281515.sc13.2274570.4 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695281520.sc13.2274570.5 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695281523.sc13.2274570.6 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695281526.sc13.2274570.7 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695281530.sc13.2274570.8 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695281534.sc13.2274570.9 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695281537.sc13.2274570.10 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695281541.sc13.2274570.11 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695281545.sc13.2274570.12 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695281548.sc13.2274570.13 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695281552.sc13.2274570.14 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695281555.sc13.2274570.15 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695281558.sc13.2274570.16 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695281563.sc13.2274570.17 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695281566.sc13.2274570.18 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695282323.sc13.2296022.0 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695282328.sc13.2296022.1 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695282331.sc13.2296022.2 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695282334.sc13.2296022.3 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695282339.sc13.2296022.4 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695282342.sc13.2296022.5 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695282345.sc13.2296022.6 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695282350.sc13.2296022.7 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695282353.sc13.2296022.8 | Bin 0 -> 193 bytes ...nts.out.tfevents.1695282356.sc13.2296022.9 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695282361.sc13.2296022.10 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695282364.sc13.2296022.11 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695282369.sc13.2296022.12 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695282374.sc13.2296022.13 | Bin 0 -> 193 bytes ...ts.out.tfevents.1695282377.sc13.2296022.14 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695282381.sc13.2296022.15 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695282385.sc13.2296022.16 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695282389.sc13.2296022.17 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695282391.sc13.2296022.18 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695283062.sc16.2520987.0 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695283064.sc16.2520987.1 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695283067.sc16.2520987.2 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695283072.sc16.2520987.3 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695283075.sc16.2520987.4 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695283078.sc16.2520987.5 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695283083.sc16.2520987.6 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695283086.sc16.2520987.7 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695283089.sc16.2520987.8 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695283094.sc16.2520987.9 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695283097.sc16.2520987.10 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695283100.sc16.2520987.11 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695283104.sc16.2520987.12 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695283108.sc16.2520987.13 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695283110.sc16.2520987.14 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695283115.sc16.2520987.15 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695283118.sc16.2520987.16 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695283121.sc16.2520987.17 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695283125.sc16.2520987.18 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695283129.sc16.2520987.19 | Bin 0 -> 141 bytes ...nts.out.tfevents.1695283892.sc11.2190319.0 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695283895.sc11.2190319.1 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695283900.sc11.2190319.2 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695283903.sc11.2190319.3 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695283906.sc11.2190319.4 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695283910.sc11.2190319.5 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695283914.sc11.2190319.6 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695283917.sc11.2190319.7 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695283921.sc11.2190319.8 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695283925.sc11.2190319.9 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695283927.sc11.2190319.10 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695283932.sc11.2190319.11 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695283936.sc11.2190319.12 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695283938.sc11.2190319.13 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695283943.sc11.2190319.14 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695283946.sc11.2190319.15 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695283949.sc11.2190319.16 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695283953.sc11.2190319.17 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695283957.sc11.2190319.18 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695283960.sc11.2190319.19 | Bin 0 -> 134 bytes ...ents.out.tfevents.1695284628.sc12.184661.0 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695284630.sc12.184661.1 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695284633.sc12.184661.2 | Bin 0 -> 134 bytes ...ents.out.tfevents.1695284638.sc12.184661.3 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695284641.sc12.184661.4 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695284645.sc12.184661.5 | Bin 0 -> 134 bytes ...ents.out.tfevents.1695284650.sc12.184661.6 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695284653.sc12.184661.7 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695284656.sc12.184661.8 | Bin 0 -> 134 bytes ...ents.out.tfevents.1695284660.sc12.184661.9 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695284663.sc12.184661.10 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695284666.sc12.184661.11 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695284671.sc12.184661.12 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695284674.sc12.184661.13 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695284677.sc12.184661.14 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695284681.sc12.184661.15 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695284684.sc12.184661.16 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695284687.sc12.184661.17 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695284692.sc12.184661.18 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695285450.sc9.1183018.0 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695285453.sc9.1183018.1 | Bin 0 -> 134 bytes ...ents.out.tfevents.1695285457.sc9.1183018.2 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695285461.sc9.1183018.3 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695285463.sc9.1183018.4 | Bin 0 -> 134 bytes ...ents.out.tfevents.1695285468.sc9.1183018.5 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695285471.sc9.1183018.6 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695285474.sc9.1183018.7 | Bin 0 -> 134 bytes ...ents.out.tfevents.1695285478.sc9.1183018.8 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695285482.sc9.1183018.9 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695285484.sc9.1183018.10 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695285489.sc9.1183018.11 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695285492.sc9.1183018.12 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695285495.sc9.1183018.13 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695285499.sc9.1183018.14 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695285502.sc9.1183018.15 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695285505.sc9.1183018.16 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695285510.sc9.1183018.17 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695285513.sc9.1183018.18 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695285516.sc9.1183018.19 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695286186.sc10.1717960.0 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695286188.sc10.1717960.1 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695286190.sc10.1717960.2 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695286195.sc10.1717960.3 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695286198.sc10.1717960.4 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695286201.sc10.1717960.5 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695286205.sc10.1717960.6 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695286209.sc10.1717960.7 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695286211.sc10.1717960.8 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695286216.sc10.1717960.9 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695286219.sc10.1717960.10 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695286222.sc10.1717960.11 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695286226.sc10.1717960.12 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695286229.sc10.1717960.13 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695286232.sc10.1717960.14 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695286236.sc10.1717960.15 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695286240.sc10.1717960.16 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695286242.sc10.1717960.17 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695286247.sc10.1717960.18 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695286250.sc10.1717960.19 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695286253.sc10.1717960.20 | Bin 0 -> 134 bytes ...ents.out.tfevents.1695287096.sc9.1214828.0 | Bin 0 -> 134 bytes ...ents.out.tfevents.1695287101.sc9.1214828.1 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695287104.sc9.1214828.2 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695287107.sc9.1214828.3 | Bin 0 -> 134 bytes ...ents.out.tfevents.1695287111.sc9.1214828.4 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695287114.sc9.1214828.5 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695287117.sc9.1214828.6 | Bin 0 -> 134 bytes ...ents.out.tfevents.1695287122.sc9.1214828.7 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695287125.sc9.1214828.8 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695287128.sc9.1214828.9 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695287132.sc9.1214828.10 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695287136.sc9.1214828.11 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695287139.sc9.1214828.12 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695287143.sc9.1214828.13 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695287147.sc9.1214828.14 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695287150.sc9.1214828.15 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695287154.sc9.1214828.16 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695287158.sc9.1214828.17 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695287161.sc9.1214828.18 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695287165.sc9.1214828.19 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695287833.sc20.1362611.0 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695287835.sc20.1362611.1 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695287838.sc20.1362611.2 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695287842.sc20.1362611.3 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695287846.sc20.1362611.4 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695287849.sc20.1362611.5 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695287853.sc20.1362611.6 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695287857.sc20.1362611.7 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695287860.sc20.1362611.8 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695287864.sc20.1362611.9 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695287867.sc20.1362611.10 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695287870.sc20.1362611.11 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695287875.sc20.1362611.12 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695287878.sc20.1362611.13 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695287881.sc20.1362611.14 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695287886.sc20.1362611.15 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695287889.sc20.1362611.16 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695287892.sc20.1362611.17 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695287896.sc20.1362611.18 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695287900.sc20.1362611.19 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695288568.sc10.1768728.0 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695288573.sc10.1768728.1 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695288576.sc10.1768728.2 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695288579.sc10.1768728.3 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695288584.sc10.1768728.4 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695288587.sc10.1768728.5 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695288590.sc10.1768728.6 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695288594.sc10.1768728.7 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695288598.sc10.1768728.8 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695288601.sc10.1768728.9 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695288605.sc10.1768728.10 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695288609.sc10.1768728.11 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695288611.sc10.1768728.12 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695288616.sc10.1768728.13 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695288620.sc10.1768728.14 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695288622.sc10.1768728.15 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695288627.sc10.1768728.16 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695288631.sc10.1768728.17 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695288634.sc10.1768728.18 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695289303.sc10.1793187.0 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695289305.sc10.1793187.1 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695289307.sc10.1793187.2 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695289311.sc10.1793187.3 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695289315.sc10.1793187.4 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695289317.sc10.1793187.5 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695289322.sc10.1793187.6 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695289325.sc10.1793187.7 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695289328.sc10.1793187.8 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695289332.sc10.1793187.9 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695289335.sc10.1793187.10 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695289338.sc10.1793187.11 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695289342.sc10.1793187.12 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695289346.sc10.1793187.13 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695289348.sc10.1793187.14 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695289353.sc10.1793187.15 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695289356.sc10.1793187.16 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695289359.sc10.1793187.17 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695289363.sc10.1793187.18 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695289367.sc10.1793187.19 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695289369.sc10.1793187.20 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695290122.sc11.2375140.0 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695290126.sc11.2375140.1 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695290130.sc11.2375140.2 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695290133.sc11.2375140.3 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695290137.sc11.2375140.4 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695290140.sc11.2375140.5 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695290143.sc11.2375140.6 | Bin 0 -> 134 bytes ...nts.out.tfevents.1695290148.sc11.2375140.7 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695290151.sc11.2375140.8 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695290154.sc11.2375140.9 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695290158.sc11.2375140.10 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695290162.sc11.2375140.11 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695290164.sc11.2375140.12 | Bin 0 -> 134 bytes ...ts.out.tfevents.1695290169.sc11.2375140.13 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695290172.sc11.2375140.14 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695290175.sc11.2375140.15 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695290179.sc11.2375140.16 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695290183.sc11.2375140.17 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695290185.sc11.2375140.18 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695290190.sc11.2375140.19 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695290935.sc12.371669.0 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695290938.sc12.371669.1 | Bin 0 -> 135 bytes ...ents.out.tfevents.1695290942.sc12.371669.2 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695290946.sc12.371669.3 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695290948.sc12.371669.4 | Bin 0 -> 135 bytes ...ents.out.tfevents.1695290953.sc12.371669.5 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695290956.sc12.371669.6 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695290959.sc12.371669.7 | Bin 0 -> 135 bytes ...ents.out.tfevents.1695290964.sc12.371669.8 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695290967.sc12.371669.9 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695290970.sc12.371669.10 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695290974.sc12.371669.11 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695290978.sc12.371669.12 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695290980.sc12.371669.13 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695290985.sc12.371669.14 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695290988.sc12.371669.15 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695290991.sc12.371669.16 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695290995.sc12.371669.17 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695290998.sc12.371669.18 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695291826.sc16.2741235.0 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695291831.sc16.2741235.1 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695291834.sc16.2741235.2 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695291837.sc16.2741235.3 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695291841.sc16.2741235.4 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695291845.sc16.2741235.5 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695291848.sc16.2741235.6 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695291852.sc16.2741235.7 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695291856.sc16.2741235.8 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695291858.sc16.2741235.9 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695291863.sc16.2741235.10 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695291867.sc16.2741235.11 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695291870.sc16.2741235.12 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695291874.sc16.2741235.13 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695291878.sc16.2741235.14 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695291881.sc16.2741235.15 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695291885.sc16.2741235.16 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695291889.sc16.2741235.17 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695291891.sc16.2741235.18 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695291896.sc16.2741235.19 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695292972.sc9.1353776.0 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695292975.sc9.1353776.1 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695292977.sc9.1353776.2 | Bin 0 -> 135 bytes ...ents.out.tfevents.1695292982.sc9.1353776.3 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695292986.sc9.1353776.4 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695292988.sc9.1353776.5 | Bin 0 -> 135 bytes ...ents.out.tfevents.1695292993.sc9.1353776.6 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695292996.sc9.1353776.7 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695292999.sc9.1353776.8 | Bin 0 -> 135 bytes ...ents.out.tfevents.1695293004.sc9.1353776.9 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695293007.sc9.1353776.10 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695293010.sc9.1353776.11 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695293014.sc9.1353776.12 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695293017.sc9.1353776.13 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695293020.sc9.1353776.14 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695293025.sc9.1353776.15 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695293028.sc9.1353776.16 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695293031.sc9.1353776.17 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695293035.sc9.1353776.18 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695293865.sc14.2649097.0 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695293868.sc14.2649097.1 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695293872.sc14.2649097.2 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695293876.sc14.2649097.3 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695293879.sc14.2649097.4 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695293883.sc14.2649097.5 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695293887.sc14.2649097.6 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695293890.sc14.2649097.7 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695293894.sc14.2649097.8 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695293897.sc14.2649097.9 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695293900.sc14.2649097.10 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695293905.sc14.2649097.11 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695293908.sc14.2649097.12 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695293911.sc14.2649097.13 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695293916.sc14.2649097.14 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695293919.sc14.2649097.15 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695293922.sc14.2649097.16 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695293927.sc14.2649097.17 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695293930.sc14.2649097.18 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695294796.sc14.2671192.0 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695294801.sc14.2671192.1 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695294804.sc14.2671192.2 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695294807.sc14.2671192.3 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695294812.sc14.2671192.4 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695294816.sc14.2671192.5 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695294818.sc14.2671192.6 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695294823.sc14.2671192.7 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695294827.sc14.2671192.8 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695294830.sc14.2671192.9 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695294834.sc14.2671192.10 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695294838.sc14.2671192.11 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695294840.sc14.2671192.12 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695294845.sc14.2671192.13 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695294848.sc14.2671192.14 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695294851.sc14.2671192.15 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695294856.sc14.2671192.16 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695294859.sc14.2671192.17 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695294862.sc14.2671192.18 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695294866.sc14.2671192.19 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695295534.sc14.2696291.0 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695295536.sc14.2696291.1 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695295541.sc14.2696291.2 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695295544.sc14.2696291.3 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695295547.sc14.2696291.4 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695295551.sc14.2696291.5 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695295555.sc14.2696291.6 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695295558.sc14.2696291.7 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695295562.sc14.2696291.8 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695295565.sc14.2696291.9 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695295568.sc14.2696291.10 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695295573.sc14.2696291.11 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695295576.sc14.2696291.12 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695295579.sc14.2696291.13 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695295584.sc14.2696291.14 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695295587.sc14.2696291.15 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695295590.sc14.2696291.16 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695295594.sc14.2696291.17 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695295598.sc14.2696291.18 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695295601.sc14.2696291.19 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695296480.sc14.2714192.0 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695296482.sc14.2714192.1 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695296485.sc14.2714192.2 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695296489.sc14.2714192.3 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695296493.sc14.2714192.4 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695296496.sc14.2714192.5 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695296500.sc14.2714192.6 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695296504.sc14.2714192.7 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695296507.sc14.2714192.8 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695296511.sc14.2714192.9 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695296515.sc14.2714192.10 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695296518.sc14.2714192.11 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695296522.sc14.2714192.12 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695296525.sc14.2714192.13 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695296528.sc14.2714192.14 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695296533.sc14.2714192.15 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695296536.sc14.2714192.16 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695296539.sc14.2714192.17 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695296544.sc14.2714192.18 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695297315.sc14.2733087.0 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695297318.sc14.2733087.1 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695297323.sc14.2733087.2 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695297326.sc14.2733087.3 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695297329.sc14.2733087.4 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695297334.sc14.2733087.5 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695297337.sc14.2733087.6 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695297340.sc14.2733087.7 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695297345.sc14.2733087.8 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695297348.sc14.2733087.9 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695297350.sc14.2733087.10 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695297355.sc14.2733087.11 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695297358.sc14.2733087.12 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695297361.sc14.2733087.13 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695297366.sc14.2733087.14 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695297369.sc14.2733087.15 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695297372.sc14.2733087.16 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695297376.sc14.2733087.17 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695297380.sc14.2733087.18 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695297383.sc14.2733087.19 | Bin 0 -> 135 bytes ...ents.out.tfevents.1695298153.sc9.1458633.0 | Bin 0 -> 135 bytes ...ents.out.tfevents.1695298158.sc9.1458633.1 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695298161.sc9.1458633.2 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695298164.sc9.1458633.3 | Bin 0 -> 135 bytes ...ents.out.tfevents.1695298168.sc9.1458633.4 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695298172.sc9.1458633.5 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695298174.sc9.1458633.6 | Bin 0 -> 135 bytes ...ents.out.tfevents.1695298179.sc9.1458633.7 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695298183.sc9.1458633.8 | Bin 0 -> 195 bytes ...ents.out.tfevents.1695298185.sc9.1458633.9 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695298190.sc9.1458633.10 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695298193.sc9.1458633.11 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695298196.sc9.1458633.12 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695298201.sc9.1458633.13 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695298205.sc9.1458633.14 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695298207.sc9.1458633.15 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695298212.sc9.1458633.16 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695298216.sc9.1458633.17 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695298218.sc9.1458633.18 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695298994.sc14.2774085.0 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695298996.sc14.2774085.1 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695298999.sc14.2774085.2 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695299003.sc14.2774085.3 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695299007.sc14.2774085.4 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695299010.sc14.2774085.5 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695299014.sc14.2774085.6 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695299018.sc14.2774085.7 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695299020.sc14.2774085.8 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695299025.sc14.2774085.9 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695299029.sc14.2774085.10 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695299032.sc14.2774085.11 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695299036.sc14.2774085.12 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695299040.sc14.2774085.13 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695299042.sc14.2774085.14 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695299047.sc14.2774085.15 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695299050.sc14.2774085.16 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695299053.sc14.2774085.17 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695299058.sc14.2774085.18 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695299061.sc14.2774085.19 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695299832.sc11.2623573.0 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695299835.sc11.2623573.1 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695299840.sc11.2623573.2 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695299843.sc11.2623573.3 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695299846.sc11.2623573.4 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695299851.sc11.2623573.5 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695299854.sc11.2623573.6 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695299857.sc11.2623573.7 | Bin 0 -> 135 bytes ...nts.out.tfevents.1695299862.sc11.2623573.8 | Bin 0 -> 195 bytes ...nts.out.tfevents.1695299865.sc11.2623573.9 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695299868.sc11.2623573.10 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695299873.sc11.2623573.11 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695299876.sc11.2623573.12 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695299879.sc11.2623573.13 | Bin 0 -> 135 bytes ...ts.out.tfevents.1695299883.sc11.2623573.14 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695299887.sc11.2623573.15 | Bin 0 -> 195 bytes ...ts.out.tfevents.1695299890.sc11.2623573.16 | Bin 0 -> 135 bytes tv-detection/maskrcnn_resnet101_fpn.isc | 2 +- tv-detection/retinanet_resnet101_fpn.isc | 2 +- .../deeplabv3_mobilenet_v3_large.isc | 2 +- tv-segmentation/fcn_resnet101.isc | 2 +- 624 files changed, 26031 insertions(+), 6 deletions(-) create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_0.txt create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_1.txt create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_10.txt create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_2.txt create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_3.txt create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_4.txt create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_5.txt create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_6.txt create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_7.txt create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_8.txt create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_9.txt create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275223.sc14.2163737.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275225.sc14.2163737.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275228.sc14.2163737.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275232.sc14.2163737.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275235.sc14.2163737.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275238.sc14.2163737.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275242.sc14.2163737.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275246.sc14.2163737.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275249.sc14.2163737.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275253.sc14.2163737.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275257.sc14.2163737.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275259.sc14.2163737.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275264.sc14.2163737.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275267.sc14.2163737.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275270.sc14.2163737.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275274.sc14.2163737.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275278.sc14.2163737.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275280.sc14.2163737.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275285.sc14.2163737.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275288.sc14.2163737.19 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275975.sc12.3473404.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275980.sc12.3473404.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275983.sc12.3473404.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275986.sc12.3473404.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275991.sc12.3473404.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275995.sc12.3473404.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275997.sc12.3473404.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276002.sc12.3473404.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276006.sc12.3473404.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276009.sc12.3473404.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276013.sc12.3473404.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276016.sc12.3473404.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276019.sc12.3473404.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276024.sc12.3473404.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276027.sc12.3473404.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276030.sc12.3473404.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276035.sc12.3473404.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276038.sc12.3473404.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276041.sc12.3473404.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276756.sc13.1777137.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276758.sc13.1777137.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276761.sc13.1777137.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276765.sc13.1777137.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276769.sc13.1777137.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276772.sc13.1777137.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276776.sc13.1777137.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276780.sc13.1777137.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276783.sc13.1777137.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276787.sc13.1777137.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276791.sc13.1777137.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276793.sc13.1777137.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276798.sc13.1777137.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276801.sc13.1777137.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276804.sc13.1777137.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276809.sc13.1777137.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276812.sc13.1777137.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276815.sc13.1777137.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276819.sc13.1777137.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276822.sc13.1777137.19 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277599.sc12.4183255.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277604.sc12.4183255.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277608.sc12.4183255.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277610.sc12.4183255.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277615.sc12.4183255.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277619.sc12.4183255.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277622.sc12.4183255.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277626.sc12.4183255.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277630.sc12.4183255.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277632.sc12.4183255.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277637.sc12.4183255.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277641.sc12.4183255.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277644.sc12.4183255.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277648.sc12.4183255.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277652.sc12.4183255.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277655.sc12.4183255.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277659.sc12.4183255.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277663.sc12.4183255.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277666.sc12.4183255.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278445.sc20.758883.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278447.sc20.758883.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278450.sc20.758883.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278455.sc20.758883.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278458.sc20.758883.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278461.sc20.758883.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278466.sc20.758883.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278469.sc20.758883.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278472.sc20.758883.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278476.sc20.758883.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278480.sc20.758883.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278483.sc20.758883.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278487.sc20.758883.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278491.sc20.758883.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278493.sc20.758883.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278498.sc20.758883.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278501.sc20.758883.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278504.sc20.758883.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278508.sc20.758883.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278512.sc20.758883.19 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279205.sc10.1505340.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279210.sc10.1505340.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279213.sc10.1505340.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279216.sc10.1505340.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279220.sc10.1505340.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279224.sc10.1505340.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279227.sc10.1505340.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279231.sc10.1505340.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279234.sc10.1505340.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279237.sc10.1505340.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279242.sc10.1505340.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279245.sc10.1505340.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279248.sc10.1505340.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279252.sc10.1505340.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279255.sc10.1505340.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279258.sc10.1505340.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279263.sc10.1505340.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279266.sc10.1505340.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279269.sc10.1505340.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279273.sc10.1505340.19 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279943.sc18.3937612.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279946.sc18.3937612.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279950.sc18.3937612.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279954.sc18.3937612.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279956.sc18.3937612.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279961.sc18.3937612.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279964.sc18.3937612.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279967.sc18.3937612.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279971.sc18.3937612.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279975.sc18.3937612.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279978.sc18.3937612.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279982.sc18.3937612.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279985.sc18.3937612.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279988.sc18.3937612.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279993.sc18.3937612.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279996.sc18.3937612.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279999.sc18.3937612.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280003.sc18.3937612.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280006.sc18.3937612.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280009.sc18.3937612.19 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280680.sc10.1562555.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280682.sc10.1562555.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280685.sc10.1562555.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280689.sc10.1562555.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280693.sc10.1562555.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280696.sc10.1562555.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280700.sc10.1562555.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280704.sc10.1562555.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280707.sc10.1562555.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280711.sc10.1562555.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280714.sc10.1562555.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280717.sc10.1562555.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280722.sc10.1562555.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280725.sc10.1562555.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280728.sc10.1562555.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280733.sc10.1562555.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280736.sc10.1562555.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280739.sc10.1562555.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280743.sc10.1562555.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281501.sc13.2274570.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281504.sc13.2274570.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281509.sc13.2274570.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281512.sc13.2274570.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281515.sc13.2274570.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281520.sc13.2274570.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281523.sc13.2274570.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281526.sc13.2274570.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281530.sc13.2274570.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281534.sc13.2274570.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281537.sc13.2274570.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281541.sc13.2274570.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281545.sc13.2274570.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281548.sc13.2274570.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281552.sc13.2274570.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281555.sc13.2274570.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281558.sc13.2274570.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281563.sc13.2274570.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281566.sc13.2274570.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282323.sc13.2296022.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282328.sc13.2296022.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282331.sc13.2296022.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282334.sc13.2296022.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282339.sc13.2296022.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282342.sc13.2296022.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282345.sc13.2296022.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282350.sc13.2296022.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282353.sc13.2296022.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282356.sc13.2296022.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282361.sc13.2296022.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282364.sc13.2296022.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282369.sc13.2296022.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282374.sc13.2296022.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282377.sc13.2296022.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282381.sc13.2296022.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282385.sc13.2296022.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282389.sc13.2296022.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282391.sc13.2296022.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283062.sc16.2520987.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283064.sc16.2520987.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283067.sc16.2520987.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283072.sc16.2520987.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283075.sc16.2520987.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283078.sc16.2520987.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283083.sc16.2520987.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283086.sc16.2520987.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283089.sc16.2520987.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283094.sc16.2520987.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283097.sc16.2520987.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283100.sc16.2520987.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283104.sc16.2520987.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283108.sc16.2520987.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283110.sc16.2520987.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283115.sc16.2520987.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283118.sc16.2520987.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283121.sc16.2520987.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283125.sc16.2520987.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283129.sc16.2520987.19 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283892.sc11.2190319.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283895.sc11.2190319.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283900.sc11.2190319.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283903.sc11.2190319.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283906.sc11.2190319.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283910.sc11.2190319.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283914.sc11.2190319.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283917.sc11.2190319.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283921.sc11.2190319.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283925.sc11.2190319.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283927.sc11.2190319.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283932.sc11.2190319.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283936.sc11.2190319.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283938.sc11.2190319.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283943.sc11.2190319.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283946.sc11.2190319.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283949.sc11.2190319.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283953.sc11.2190319.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283957.sc11.2190319.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283960.sc11.2190319.19 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284628.sc12.184661.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284630.sc12.184661.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284633.sc12.184661.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284638.sc12.184661.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284641.sc12.184661.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284645.sc12.184661.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284650.sc12.184661.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284653.sc12.184661.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284656.sc12.184661.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284660.sc12.184661.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284663.sc12.184661.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284666.sc12.184661.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284671.sc12.184661.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284674.sc12.184661.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284677.sc12.184661.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284681.sc12.184661.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284684.sc12.184661.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284687.sc12.184661.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284692.sc12.184661.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285450.sc9.1183018.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285453.sc9.1183018.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285457.sc9.1183018.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285461.sc9.1183018.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285463.sc9.1183018.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285468.sc9.1183018.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285471.sc9.1183018.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285474.sc9.1183018.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285478.sc9.1183018.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285482.sc9.1183018.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285484.sc9.1183018.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285489.sc9.1183018.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285492.sc9.1183018.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285495.sc9.1183018.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285499.sc9.1183018.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285502.sc9.1183018.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285505.sc9.1183018.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285510.sc9.1183018.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285513.sc9.1183018.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285516.sc9.1183018.19 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286186.sc10.1717960.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286188.sc10.1717960.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286190.sc10.1717960.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286195.sc10.1717960.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286198.sc10.1717960.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286201.sc10.1717960.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286205.sc10.1717960.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286209.sc10.1717960.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286211.sc10.1717960.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286216.sc10.1717960.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286219.sc10.1717960.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286222.sc10.1717960.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286226.sc10.1717960.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286229.sc10.1717960.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286232.sc10.1717960.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286236.sc10.1717960.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286240.sc10.1717960.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286242.sc10.1717960.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286247.sc10.1717960.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286250.sc10.1717960.19 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286253.sc10.1717960.20 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287096.sc9.1214828.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287101.sc9.1214828.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287104.sc9.1214828.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287107.sc9.1214828.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287111.sc9.1214828.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287114.sc9.1214828.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287117.sc9.1214828.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287122.sc9.1214828.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287125.sc9.1214828.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287128.sc9.1214828.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287132.sc9.1214828.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287136.sc9.1214828.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287139.sc9.1214828.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287143.sc9.1214828.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287147.sc9.1214828.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287150.sc9.1214828.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287154.sc9.1214828.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287158.sc9.1214828.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287161.sc9.1214828.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287165.sc9.1214828.19 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287833.sc20.1362611.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287835.sc20.1362611.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287838.sc20.1362611.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287842.sc20.1362611.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287846.sc20.1362611.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287849.sc20.1362611.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287853.sc20.1362611.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287857.sc20.1362611.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287860.sc20.1362611.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287864.sc20.1362611.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287867.sc20.1362611.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287870.sc20.1362611.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287875.sc20.1362611.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287878.sc20.1362611.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287881.sc20.1362611.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287886.sc20.1362611.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287889.sc20.1362611.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287892.sc20.1362611.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287896.sc20.1362611.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287900.sc20.1362611.19 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288568.sc10.1768728.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288573.sc10.1768728.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288576.sc10.1768728.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288579.sc10.1768728.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288584.sc10.1768728.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288587.sc10.1768728.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288590.sc10.1768728.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288594.sc10.1768728.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288598.sc10.1768728.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288601.sc10.1768728.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288605.sc10.1768728.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288609.sc10.1768728.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288611.sc10.1768728.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288616.sc10.1768728.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288620.sc10.1768728.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288622.sc10.1768728.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288627.sc10.1768728.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288631.sc10.1768728.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288634.sc10.1768728.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289303.sc10.1793187.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289305.sc10.1793187.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289307.sc10.1793187.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289311.sc10.1793187.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289315.sc10.1793187.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289317.sc10.1793187.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289322.sc10.1793187.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289325.sc10.1793187.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289328.sc10.1793187.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289332.sc10.1793187.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289335.sc10.1793187.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289338.sc10.1793187.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289342.sc10.1793187.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289346.sc10.1793187.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289348.sc10.1793187.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289353.sc10.1793187.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289356.sc10.1793187.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289359.sc10.1793187.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289363.sc10.1793187.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289367.sc10.1793187.19 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289369.sc10.1793187.20 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290122.sc11.2375140.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290126.sc11.2375140.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290130.sc11.2375140.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290133.sc11.2375140.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290137.sc11.2375140.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290140.sc11.2375140.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290143.sc11.2375140.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290148.sc11.2375140.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290151.sc11.2375140.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290154.sc11.2375140.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290158.sc11.2375140.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290162.sc11.2375140.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290164.sc11.2375140.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290169.sc11.2375140.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290172.sc11.2375140.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290175.sc11.2375140.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290179.sc11.2375140.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290183.sc11.2375140.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290185.sc11.2375140.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290190.sc11.2375140.19 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290935.sc12.371669.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290938.sc12.371669.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290942.sc12.371669.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290946.sc12.371669.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290948.sc12.371669.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290953.sc12.371669.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290956.sc12.371669.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290959.sc12.371669.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290964.sc12.371669.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290967.sc12.371669.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290970.sc12.371669.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290974.sc12.371669.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290978.sc12.371669.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290980.sc12.371669.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290985.sc12.371669.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290988.sc12.371669.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290991.sc12.371669.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290995.sc12.371669.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290998.sc12.371669.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291826.sc16.2741235.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291831.sc16.2741235.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291834.sc16.2741235.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291837.sc16.2741235.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291841.sc16.2741235.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291845.sc16.2741235.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291848.sc16.2741235.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291852.sc16.2741235.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291856.sc16.2741235.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291858.sc16.2741235.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291863.sc16.2741235.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291867.sc16.2741235.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291870.sc16.2741235.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291874.sc16.2741235.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291878.sc16.2741235.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291881.sc16.2741235.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291885.sc16.2741235.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291889.sc16.2741235.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291891.sc16.2741235.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291896.sc16.2741235.19 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292972.sc9.1353776.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292975.sc9.1353776.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292977.sc9.1353776.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292982.sc9.1353776.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292986.sc9.1353776.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292988.sc9.1353776.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292993.sc9.1353776.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292996.sc9.1353776.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292999.sc9.1353776.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293004.sc9.1353776.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293007.sc9.1353776.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293010.sc9.1353776.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293014.sc9.1353776.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293017.sc9.1353776.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293020.sc9.1353776.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293025.sc9.1353776.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293028.sc9.1353776.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293031.sc9.1353776.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293035.sc9.1353776.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293865.sc14.2649097.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293868.sc14.2649097.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293872.sc14.2649097.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293876.sc14.2649097.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293879.sc14.2649097.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293883.sc14.2649097.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293887.sc14.2649097.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293890.sc14.2649097.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293894.sc14.2649097.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293897.sc14.2649097.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293900.sc14.2649097.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293905.sc14.2649097.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293908.sc14.2649097.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293911.sc14.2649097.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293916.sc14.2649097.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293919.sc14.2649097.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293922.sc14.2649097.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293927.sc14.2649097.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293930.sc14.2649097.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294796.sc14.2671192.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294801.sc14.2671192.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294804.sc14.2671192.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294807.sc14.2671192.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294812.sc14.2671192.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294816.sc14.2671192.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294818.sc14.2671192.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294823.sc14.2671192.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294827.sc14.2671192.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294830.sc14.2671192.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294834.sc14.2671192.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294838.sc14.2671192.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294840.sc14.2671192.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294845.sc14.2671192.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294848.sc14.2671192.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294851.sc14.2671192.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294856.sc14.2671192.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294859.sc14.2671192.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294862.sc14.2671192.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294866.sc14.2671192.19 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295534.sc14.2696291.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295536.sc14.2696291.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295541.sc14.2696291.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295544.sc14.2696291.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295547.sc14.2696291.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295551.sc14.2696291.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295555.sc14.2696291.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295558.sc14.2696291.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295562.sc14.2696291.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295565.sc14.2696291.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295568.sc14.2696291.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295573.sc14.2696291.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295576.sc14.2696291.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295579.sc14.2696291.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295584.sc14.2696291.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295587.sc14.2696291.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295590.sc14.2696291.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295594.sc14.2696291.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295598.sc14.2696291.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295601.sc14.2696291.19 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296480.sc14.2714192.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296482.sc14.2714192.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296485.sc14.2714192.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296489.sc14.2714192.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296493.sc14.2714192.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296496.sc14.2714192.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296500.sc14.2714192.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296504.sc14.2714192.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296507.sc14.2714192.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296511.sc14.2714192.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296515.sc14.2714192.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296518.sc14.2714192.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296522.sc14.2714192.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296525.sc14.2714192.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296528.sc14.2714192.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296533.sc14.2714192.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296536.sc14.2714192.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296539.sc14.2714192.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296544.sc14.2714192.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297315.sc14.2733087.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297318.sc14.2733087.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297323.sc14.2733087.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297326.sc14.2733087.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297329.sc14.2733087.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297334.sc14.2733087.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297337.sc14.2733087.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297340.sc14.2733087.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297345.sc14.2733087.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297348.sc14.2733087.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297350.sc14.2733087.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297355.sc14.2733087.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297358.sc14.2733087.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297361.sc14.2733087.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297366.sc14.2733087.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297369.sc14.2733087.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297372.sc14.2733087.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297376.sc14.2733087.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297380.sc14.2733087.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297383.sc14.2733087.19 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298153.sc9.1458633.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298158.sc9.1458633.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298161.sc9.1458633.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298164.sc9.1458633.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298168.sc9.1458633.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298172.sc9.1458633.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298174.sc9.1458633.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298179.sc9.1458633.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298183.sc9.1458633.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298185.sc9.1458633.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298190.sc9.1458633.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298193.sc9.1458633.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298196.sc9.1458633.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298201.sc9.1458633.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298205.sc9.1458633.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298207.sc9.1458633.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298212.sc9.1458633.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298216.sc9.1458633.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298218.sc9.1458633.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298994.sc14.2774085.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298996.sc14.2774085.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298999.sc14.2774085.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299003.sc14.2774085.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299007.sc14.2774085.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299010.sc14.2774085.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299014.sc14.2774085.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299018.sc14.2774085.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299020.sc14.2774085.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299025.sc14.2774085.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299029.sc14.2774085.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299032.sc14.2774085.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299036.sc14.2774085.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299040.sc14.2774085.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299042.sc14.2774085.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299047.sc14.2774085.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299050.sc14.2774085.16 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299053.sc14.2774085.17 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299058.sc14.2774085.18 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299061.sc14.2774085.19 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299832.sc11.2623573.0 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299835.sc11.2623573.1 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299840.sc11.2623573.2 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299843.sc11.2623573.3 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299846.sc11.2623573.4 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299851.sc11.2623573.5 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299854.sc11.2623573.6 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299857.sc11.2623573.7 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299862.sc11.2623573.8 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299865.sc11.2623573.9 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299868.sc11.2623573.10 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299873.sc11.2623573.11 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299876.sc11.2623573.12 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299879.sc11.2623573.13 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299883.sc11.2623573.14 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299887.sc11.2623573.15 create mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299890.sc11.2623573.16 diff --git a/monai_brats_mri_2d/brats_mri_2d_diff.isc b/monai_brats_mri_2d/brats_mri_2d_diff.isc index 3220f07b..5f100352 100644 --- a/monai_brats_mri_2d/brats_mri_2d_diff.isc +++ b/monai_brats_mri_2d/brats_mri_2d_diff.isc @@ -3,4 +3,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_brats_mri_2d_diff" -command="train_cycling_diff.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc, --gen-load-path ~/output_brats_mri_2d_gen/exp_1791/checkpoint.isc --tboard-path ~/output_brats_mri_2d_diff/tb" \ No newline at end of file +command="train_cycling_diff.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc, --gen-load-path ~/output_brats_mri_2d_gen/exp_1855/checkpoint.isc --tboard-path $OUTPUT_PATH/tb" \ No newline at end of file diff --git a/monai_brats_mri_2d/brats_mri_2d_gen.isc b/monai_brats_mri_2d/brats_mri_2d_gen.isc index df84c2f3..802596ee 100644 --- a/monai_brats_mri_2d/brats_mri_2d_gen.isc +++ b/monai_brats_mri_2d/brats_mri_2d_gen.isc @@ -3,4 +3,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_brats_mri_2d_gen" -command="train_cycling_gen.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_brats_mri_2d_gen/tb" \ No newline at end of file +command="train_cycling_gen.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc --tboard-path $OUTPUT_PATH/tb" \ No newline at end of file diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_0.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_0.txt new file mode 100644 index 00000000..414e5290 --- /dev/null +++ b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_0.txt @@ -0,0 +1,12925 @@ +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 05:46:44 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 05:46:44 ] Completed importing Timer 0.024 ms, 0.00 s total +[ 2023-09-21 05:46:49 ] Completed importing everything else 4,759.109 ms, 4.76 s total +| distributed init (rank 5): env:// +| distributed init (rank 3): env:// +| distributed init (rank 1): env:// +| distributed init (rank 2): env:// +| distributed init (rank 4): env:// +| distributed init (rank 0): env:// +[ 2023-09-21 05:46:56 ] Completed preliminaries 7,462.666 ms, 12.22 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 05:46:56 ] Completed build datasets 11.769 ms, 12.23 s total +[ 2023-09-21 05:46:56 ] Completed build samplers 0.047 ms, 12.23 s total +[ 2023-09-21 05:46:56 ] Completed build dataloaders 0.134 ms, 12.23 s total +[ 2023-09-21 05:46:56 ] Completed generator to device 79.883 ms, 12.31 s total +[ 2023-09-21 05:46:56 ] Completed discriminator to device 7.031 ms, 12.32 s total +[ 2023-09-21 05:46:57 ] Completed loss functions 503.801 ms, 12.82 s total +[ 2023-09-21 05:46:57 ] Completed models prepped for distribution 38.852 ms, 12.86 s total +[ 2023-09-21 05:46:57 ] Completed optimizers 0.669 ms, 12.86 s total +[ 2023-09-21 05:46:57 ] Completed grad scalers 0.025 ms, 12.86 s total +[ 2023-09-21 05:46:57 ] Completed checkpoint retrieval 0.275 ms, 12.86 s total + + +EPOCH :: 0 + + +[ 2023-09-21 05:46:57 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 0 from training step 0 + +[ 2023-09-21 05:47:00 ] Completed train batch 0 to device 3,602.622 ms, 3.60 s total +[ 2023-09-21 05:47:02 ] Completed train batch 0 generator forward 1,421.216 ms, 5.02 s total +[ 2023-09-21 05:47:02 ] Completed train batch 0 generator loss: 2.480 87.080 ms, 5.11 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 05:47:03 ] Completed train batch 0 generator backward 1,006.777 ms, 6.12 s total +[ 2023-09-21 05:47:03 ] Completed train batch 0 discriminator loss 0.342 56.426 ms, 6.17 s total +[ 2023-09-21 05:47:03 ] Completed train batch 0 discriminator backward 101.381 ms, 6.28 s total +Epoch [0] Step [0/2], gen_loss: 0.827, disc_loss: 0.114 +[ 2023-09-21 05:47:03 ] Completed train batch 0 metrics update 6.901 ms, 6.28 s total +Saving checkpoint at epoch 0 train batch 0 +[ 2023-09-21 05:47:04 ] Completed saving temp checkpoint 433.835 ms, 6.72 s total +[ 2023-09-21 05:47:04 ] Completed replacing temp checkpoint with checkpoint 6.076 ms, 6.72 s total +[ 2023-09-21 05:47:04 ] Completed train batch 1 to device 125.666 ms, 6.85 s total +[ 2023-09-21 05:47:04 ] Completed train batch 1 generator forward 27.611 ms, 6.88 s total +[ 2023-09-21 05:47:04 ] Completed train batch 1 generator loss: 2.426 63.994 ms, 6.94 s total +[ 2023-09-21 05:47:05 ] Completed train batch 1 generator backward 1,094.595 ms, 8.03 s total +[ 2023-09-21 05:47:05 ] Completed train batch 1 discriminator loss 0.195 62.031 ms, 8.10 s total +[ 2023-09-21 05:47:05 ] Completed train batch 1 discriminator backward 58.162 ms, 8.15 s total +Epoch [0] Step [1/2], gen_loss: 0.807, disc_loss: 0.064 +[ 2023-09-21 05:47:05 ] Completed train batch 1 metrics update 5.524 ms, 8.16 s total +Saving checkpoint at epoch 0 train batch 1 +[ 2023-09-21 05:47:05 ] Completed saving temp checkpoint 476.097 ms, 8.64 s total +[ 2023-09-21 05:47:05 ] Completed replacing temp checkpoint with checkpoint 24.204 ms, 8.66 s total +Epoch [0] :: gen_loss: 0.817, disc_loss: 0.089 +[ 2023-09-21 05:47:06 ] Completed training generator for epoch 0 64.098 ms, 8.72 s total +[ 2023-09-21 05:47:06 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 0 from eval step 0 + +[ 2023-09-21 05:47:07 ] Completed eval batch 0 to device 1,133.913 ms, 1.13 s total +[ 2023-09-21 05:47:07 ] Completed eval batch 0 forward 44.514 ms, 1.18 s total +[ 2023-09-21 05:47:07 ] Completed eval batch 0 recons_loss 0.319 ms, 1.18 s total +[ 2023-09-21 05:47:07 ] Completed eval batch 0 metrics update 0.576 ms, 1.18 s total +Saving checkpoint at epoch 0 val batch 0 +[ 2023-09-21 05:47:07 ] Completed saving temp checkpoint 411.608 ms, 1.59 s total +[ 2023-09-21 05:47:07 ] Completed replacing temp checkpoint with checkpoint 25.754 ms, 1.62 s total +[ 2023-09-21 05:47:08 ] Completed eval batch 1 to device 364.721 ms, 1.98 s total +[ 2023-09-21 05:47:08 ] Completed eval batch 1 forward 39.759 ms, 2.02 s total +[ 2023-09-21 05:47:08 ] Completed eval batch 1 recons_loss 0.551 ms, 2.02 s total +[ 2023-09-21 05:47:08 ] Completed eval batch 1 metrics update 50.717 ms, 2.07 s total +Saving checkpoint at epoch 0 val batch 1 +Epoch 0 val loss: 0.4807 +[ 2023-09-21 05:47:08 ] Completed saving temp checkpoint 467.206 ms, 2.54 s total +[ 2023-09-21 05:47:08 ] Completed replacing temp checkpoint with checkpoint 21.521 ms, 2.56 s total +[ 2023-09-21 05:47:08 ] Completed evaluating generator for epoch 0 47.881 ms, 2.61 s total + + +EPOCH :: 1 + + +[ 2023-09-21 05:47:08 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 1 from training step 0 + +[ 2023-09-21 05:47:11 ] Completed train batch 0 to device 2,972.115 ms, 2.97 s total +[ 2023-09-21 05:47:11 ] Completed train batch 0 generator forward 33.176 ms, 3.01 s total +[ 2023-09-21 05:47:11 ] Completed train batch 0 generator loss: 2.501 66.624 ms, 3.07 s total +[ 2023-09-21 05:47:12 ] Completed train batch 0 generator backward 563.043 ms, 3.63 s total +[ 2023-09-21 05:47:12 ] Completed train batch 0 discriminator loss 0.142 74.985 ms, 3.71 s total +[ 2023-09-21 05:47:12 ] Completed train batch 0 discriminator backward 77.776 ms, 3.79 s total +Epoch [1] Step [0/2], gen_loss: 0.837, disc_loss: 0.047 +[ 2023-09-21 05:47:12 ] Completed train batch 0 metrics update 5.447 ms, 3.79 s total +Saving checkpoint at epoch 1 train batch 0 +[ 2023-09-21 05:47:12 ] Completed saving temp checkpoint 396.407 ms, 4.19 s total +[ 2023-09-21 05:47:12 ] Completed replacing temp checkpoint with checkpoint 30.443 ms, 4.22 s total +[ 2023-09-21 05:47:14 ] Completed train batch 1 to device 1,651.137 ms, 5.87 s total +[ 2023-09-21 05:47:14 ] Completed train batch 1 generator forward 41.709 ms, 5.91 s total +[ 2023-09-21 05:47:14 ] Completed train batch 1 generator loss: 2.302 69.503 ms, 5.98 s total +[ 2023-09-21 05:47:15 ] Completed train batch 1 generator backward 1,028.466 ms, 7.01 s total +[ 2023-09-21 05:47:15 ] Completed train batch 1 discriminator loss 0.128 56.712 ms, 7.07 s total +[ 2023-09-21 05:47:15 ] Completed train batch 1 discriminator backward 71.761 ms, 7.14 s total +Epoch [1] Step [1/2], gen_loss: 0.766, disc_loss: 0.042 +[ 2023-09-21 05:47:15 ] Completed train batch 1 metrics update 5.404 ms, 7.14 s total +Saving checkpoint at epoch 1 train batch 1 +[ 2023-09-21 05:47:16 ] Completed saving temp checkpoint 449.338 ms, 7.59 s total +[ 2023-09-21 05:47:16 ] Completed replacing temp checkpoint with checkpoint 32.826 ms, 7.63 s total +Epoch [1] :: gen_loss: 0.801, disc_loss: 0.045 +[ 2023-09-21 05:47:16 ] Completed training generator for epoch 1 52.284 ms, 7.68 s total +[ 2023-09-21 05:47:16 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 1 from eval step 0 + +[ 2023-09-21 05:47:17 ] Completed eval batch 0 to device 1,123.771 ms, 1.12 s total +[ 2023-09-21 05:47:17 ] Completed eval batch 0 forward 43.463 ms, 1.17 s total +[ 2023-09-21 05:47:17 ] Completed eval batch 0 recons_loss 0.361 ms, 1.17 s total +[ 2023-09-21 05:47:17 ] Completed eval batch 0 metrics update 0.577 ms, 1.17 s total +Saving checkpoint at epoch 1 val batch 0 +[ 2023-09-21 05:47:17 ] Completed saving temp checkpoint 393.795 ms, 1.56 s total +[ 2023-09-21 05:47:17 ] Completed replacing temp checkpoint with checkpoint 29.655 ms, 1.59 s total +[ 2023-09-21 05:47:18 ] Completed eval batch 1 to device 413.625 ms, 2.01 s total +[ 2023-09-21 05:47:18 ] Completed eval batch 1 forward 39.775 ms, 2.05 s total +[ 2023-09-21 05:47:18 ] Completed eval batch 1 recons_loss 0.605 ms, 2.05 s total +[ 2023-09-21 05:47:18 ] Completed eval batch 1 metrics update 134.619 ms, 2.18 s total +Saving checkpoint at epoch 1 val batch 1 +Epoch 1 val loss: 0.3936 +[ 2023-09-21 05:47:18 ] Completed saving temp checkpoint 470.263 ms, 2.65 s total +[ 2023-09-21 05:47:19 ] Completed replacing temp checkpoint with checkpoint 20.152 ms, 2.67 s total +[ 2023-09-21 05:47:19 ] Completed evaluating generator for epoch 1 45.420 ms, 2.72 s total + + +EPOCH :: 2 + + +[ 2023-09-21 05:47:19 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 2 from training step 0 + +[ 2023-09-21 05:47:22 ] Completed train batch 0 to device 2,986.176 ms, 2.99 s total +[ 2023-09-21 05:47:22 ] Completed train batch 0 generator forward 45.010 ms, 3.03 s total +[ 2023-09-21 05:47:22 ] Completed train batch 0 generator loss: nan 66.154 ms, 3.10 s total +[ 2023-09-21 05:47:22 ] Completed train batch 0 generator backward 546.607 ms, 3.64 s total +[ 2023-09-21 05:47:22 ] Completed train batch 0 discriminator loss 0.122 56.990 ms, 3.70 s total +[ 2023-09-21 05:47:22 ] Completed train batch 0 discriminator backward 93.706 ms, 3.79 s total +Epoch [2] Step [0/2], gen_loss: nan, disc_loss: 0.039 +[ 2023-09-21 05:47:22 ] Completed train batch 0 metrics update 12.546 ms, 3.81 s total +Saving checkpoint at epoch 2 train batch 0 +[ 2023-09-21 05:47:23 ] Completed saving temp checkpoint 483.371 ms, 4.29 s total +[ 2023-09-21 05:47:23 ] Completed replacing temp checkpoint with checkpoint 20.450 ms, 4.31 s total +[ 2023-09-21 05:47:24 ] Completed train batch 1 to device 1,409.171 ms, 5.72 s total +[ 2023-09-21 05:47:24 ] Completed train batch 1 generator forward 40.428 ms, 5.76 s total +[ 2023-09-21 05:47:24 ] Completed train batch 1 generator loss: nan 63.653 ms, 5.82 s total +[ 2023-09-21 05:47:26 ] Completed train batch 1 generator backward 1,392.355 ms, 7.22 s total +[ 2023-09-21 05:47:26 ] Completed train batch 1 discriminator loss 0.102 56.399 ms, 7.27 s total +[ 2023-09-21 05:47:26 ] Completed train batch 1 discriminator backward 66.305 ms, 7.34 s total +Epoch [2] Step [1/2], gen_loss: nan, disc_loss: 0.034 +[ 2023-09-21 05:47:26 ] Completed train batch 1 metrics update 5.850 ms, 7.35 s total +Saving checkpoint at epoch 2 train batch 1 +[ 2023-09-21 05:47:26 ] Completed saving temp checkpoint 482.316 ms, 7.83 s total +[ 2023-09-21 05:47:26 ] Completed replacing temp checkpoint with checkpoint 35.606 ms, 7.86 s total +Epoch [2] :: gen_loss: nan, disc_loss: 0.036 +[ 2023-09-21 05:47:26 ] Completed training generator for epoch 2 66.120 ms, 7.93 s total +[ 2023-09-21 05:47:26 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 2 from eval step 0 + +[ 2023-09-21 05:47:28 ] Completed eval batch 0 to device 1,094.025 ms, 1.09 s total +[ 2023-09-21 05:47:28 ] Completed eval batch 0 forward 28.024 ms, 1.12 s total +[ 2023-09-21 05:47:28 ] Completed eval batch 0 recons_loss 0.311 ms, 1.12 s total +[ 2023-09-21 05:47:28 ] Completed eval batch 0 metrics update 0.551 ms, 1.12 s total +Saving checkpoint at epoch 2 val batch 0 +[ 2023-09-21 05:47:28 ] Completed saving temp checkpoint 494.025 ms, 1.62 s total +[ 2023-09-21 05:47:28 ] Completed replacing temp checkpoint with checkpoint 33.083 ms, 1.65 s total +[ 2023-09-21 05:47:28 ] Completed eval batch 1 to device 308.368 ms, 1.96 s total +[ 2023-09-21 05:47:28 ] Completed eval batch 1 forward 39.712 ms, 2.00 s total +[ 2023-09-21 05:47:28 ] Completed eval batch 1 recons_loss 0.557 ms, 2.00 s total +[ 2023-09-21 05:47:29 ] Completed eval batch 1 metrics update 236.971 ms, 2.24 s total +Saving checkpoint at epoch 2 val batch 1 +Epoch 2 val loss: 0.3945 +[ 2023-09-21 05:47:29 ] Completed saving temp checkpoint 683.919 ms, 2.92 s total +[ 2023-09-21 05:47:29 ] Completed replacing temp checkpoint with checkpoint 41.794 ms, 2.96 s total +[ 2023-09-21 05:47:30 ] Completed evaluating generator for epoch 2 64.710 ms, 3.03 s total + + +EPOCH :: 3 + + +[ 2023-09-21 05:47:30 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 3 from training step 0 + +[ 2023-09-21 05:47:32 ] Completed train batch 0 to device 2,987.141 ms, 2.99 s total +[ 2023-09-21 05:47:33 ] Completed train batch 0 generator forward 29.920 ms, 3.02 s total +[ 2023-09-21 05:47:33 ] Completed train batch 0 generator loss: nan 66.475 ms, 3.08 s total +[ 2023-09-21 05:47:33 ] Completed train batch 0 generator backward 483.431 ms, 3.57 s total +[ 2023-09-21 05:47:33 ] Completed train batch 0 discriminator loss 0.097 64.411 ms, 3.63 s total +[ 2023-09-21 05:47:33 ] Completed train batch 0 discriminator backward 84.491 ms, 3.72 s total +Epoch [3] Step [0/2], gen_loss: nan, disc_loss: 0.031 +[ 2023-09-21 05:47:33 ] Completed train batch 0 metrics update 4.911 ms, 3.72 s total +Saving checkpoint at epoch 3 train batch 0 +[ 2023-09-21 05:47:34 ] Completed saving temp checkpoint 441.548 ms, 4.16 s total +[ 2023-09-21 05:47:34 ] Completed replacing temp checkpoint with checkpoint 17.724 ms, 4.18 s total +[ 2023-09-21 05:47:35 ] Completed train batch 1 to device 1,596.127 ms, 5.78 s total +[ 2023-09-21 05:47:35 ] Completed train batch 1 generator forward 24.086 ms, 5.80 s total +[ 2023-09-21 05:47:35 ] Completed train batch 1 generator loss: nan 64.102 ms, 5.86 s total +[ 2023-09-21 05:47:37 ] Completed train batch 1 generator backward 1,181.406 ms, 7.05 s total +[ 2023-09-21 05:47:37 ] Completed train batch 1 discriminator loss 0.074 56.884 ms, 7.10 s total +[ 2023-09-21 05:47:37 ] Completed train batch 1 discriminator backward 77.488 ms, 7.18 s total +Epoch [3] Step [1/2], gen_loss: nan, disc_loss: 0.026 +[ 2023-09-21 05:47:37 ] Completed train batch 1 metrics update 5.155 ms, 7.19 s total +Saving checkpoint at epoch 3 train batch 1 +[ 2023-09-21 05:47:37 ] Completed saving temp checkpoint 473.955 ms, 7.66 s total +[ 2023-09-21 05:47:37 ] Completed replacing temp checkpoint with checkpoint 18.245 ms, 7.68 s total +Epoch [3] :: gen_loss: nan, disc_loss: 0.028 +[ 2023-09-21 05:47:37 ] Completed training generator for epoch 3 52.778 ms, 7.73 s total +[ 2023-09-21 05:47:37 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 3 from eval step 0 + +[ 2023-09-21 05:47:38 ] Completed eval batch 0 to device 1,113.652 ms, 1.11 s total +[ 2023-09-21 05:47:38 ] Completed eval batch 0 forward 28.587 ms, 1.14 s total +[ 2023-09-21 05:47:38 ] Completed eval batch 0 recons_loss 0.305 ms, 1.14 s total +[ 2023-09-21 05:47:38 ] Completed eval batch 0 metrics update 0.564 ms, 1.14 s total +Saving checkpoint at epoch 3 val batch 0 +[ 2023-09-21 05:47:39 ] Completed saving temp checkpoint 435.592 ms, 1.58 s total +[ 2023-09-21 05:47:39 ] Completed replacing temp checkpoint with checkpoint 18.484 ms, 1.60 s total +[ 2023-09-21 05:47:39 ] Completed eval batch 1 to device 452.839 ms, 2.05 s total +[ 2023-09-21 05:47:39 ] Completed eval batch 1 forward 21.929 ms, 2.07 s total +[ 2023-09-21 05:47:39 ] Completed eval batch 1 recons_loss 0.401 ms, 2.07 s total +[ 2023-09-21 05:47:39 ] Completed eval batch 1 metrics update 154.161 ms, 2.23 s total +Saving checkpoint at epoch 3 val batch 1 +Epoch 3 val loss: 0.3858 +[ 2023-09-21 05:47:40 ] Completed saving temp checkpoint 452.450 ms, 2.68 s total +[ 2023-09-21 05:47:40 ] Completed replacing temp checkpoint with checkpoint 25.877 ms, 2.70 s total +[ 2023-09-21 05:47:40 ] Completed evaluating generator for epoch 3 44.876 ms, 2.75 s total + + +EPOCH :: 4 + + +[ 2023-09-21 05:47:40 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 4 from training step 0 + +[ 2023-09-21 05:47:43 ] Completed train batch 0 to device 3,034.499 ms, 3.03 s total +[ 2023-09-21 05:47:43 ] Completed train batch 0 generator forward 34.966 ms, 3.07 s total +[ 2023-09-21 05:47:43 ] Completed train batch 0 generator loss: nan 77.904 ms, 3.15 s total +[ 2023-09-21 05:47:44 ] Completed train batch 0 generator backward 625.718 ms, 3.77 s total +[ 2023-09-21 05:47:44 ] Completed train batch 0 discriminator loss 0.063 63.370 ms, 3.84 s total +[ 2023-09-21 05:47:44 ] Completed train batch 0 discriminator backward 87.465 ms, 3.92 s total +Epoch [4] Step [0/2], gen_loss: nan, disc_loss: 0.021 +[ 2023-09-21 05:47:44 ] Completed train batch 0 metrics update 5.202 ms, 3.93 s total +Saving checkpoint at epoch 4 train batch 0 +[ 2023-09-21 05:47:44 ] Completed saving temp checkpoint 461.724 ms, 4.39 s total +[ 2023-09-21 05:47:44 ] Completed replacing temp checkpoint with checkpoint 33.478 ms, 4.42 s total +[ 2023-09-21 05:47:46 ] Completed train batch 1 to device 1,426.859 ms, 5.85 s total +[ 2023-09-21 05:47:46 ] Completed train batch 1 generator forward 41.331 ms, 5.89 s total +[ 2023-09-21 05:47:46 ] Completed train batch 1 generator loss: nan 64.118 ms, 5.96 s total +[ 2023-09-21 05:47:47 ] Completed train batch 1 generator backward 833.377 ms, 6.79 s total +[ 2023-09-21 05:47:47 ] Completed train batch 1 discriminator loss 0.047 56.337 ms, 6.85 s total +[ 2023-09-21 05:47:47 ] Completed train batch 1 discriminator backward 76.802 ms, 6.92 s total +Epoch [4] Step [1/2], gen_loss: nan, disc_loss: 0.016 +[ 2023-09-21 05:47:47 ] Completed train batch 1 metrics update 5.243 ms, 6.93 s total +Saving checkpoint at epoch 4 train batch 1 +[ 2023-09-21 05:47:47 ] Completed saving temp checkpoint 485.188 ms, 7.41 s total +[ 2023-09-21 05:47:47 ] Completed replacing temp checkpoint with checkpoint 32.691 ms, 7.45 s total +Epoch [4] :: gen_loss: nan, disc_loss: 0.018 +[ 2023-09-21 05:47:47 ] Completed training generator for epoch 4 54.131 ms, 7.50 s total +[ 2023-09-21 05:47:47 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 4 from eval step 0 + +[ 2023-09-21 05:47:49 ] Completed eval batch 0 to device 1,057.164 ms, 1.06 s total +[ 2023-09-21 05:47:49 ] Completed eval batch 0 forward 24.136 ms, 1.08 s total +[ 2023-09-21 05:47:49 ] Completed eval batch 0 recons_loss 0.227 ms, 1.08 s total +[ 2023-09-21 05:47:49 ] Completed eval batch 0 metrics update 0.487 ms, 1.08 s total +Saving checkpoint at epoch 4 val batch 0 +[ 2023-09-21 05:47:49 ] Completed saving temp checkpoint 459.566 ms, 1.54 s total +[ 2023-09-21 05:47:49 ] Completed replacing temp checkpoint with checkpoint 27.068 ms, 1.57 s total +[ 2023-09-21 05:47:49 ] Completed eval batch 1 to device 371.450 ms, 1.94 s total +[ 2023-09-21 05:47:49 ] Completed eval batch 1 forward 37.941 ms, 1.98 s total +[ 2023-09-21 05:47:49 ] Completed eval batch 1 recons_loss 0.344 ms, 1.98 s total +[ 2023-09-21 05:47:50 ] Completed eval batch 1 metrics update 266.220 ms, 2.24 s total +Saving checkpoint at epoch 4 val batch 1 +Epoch 4 val loss: 0.3746 +[ 2023-09-21 05:47:50 ] Completed saving temp checkpoint 455.324 ms, 2.70 s total +[ 2023-09-21 05:47:50 ] Completed replacing temp checkpoint with checkpoint 25.711 ms, 2.73 s total +[ 2023-09-21 05:47:50 ] Completed evaluating generator for epoch 4 51.014 ms, 2.78 s total + + +EPOCH :: 5 + + +[ 2023-09-21 05:47:50 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 5 from training step 0 + +[ 2023-09-21 05:47:53 ] Completed train batch 0 to device 2,978.069 ms, 2.98 s total +[ 2023-09-21 05:47:53 ] Completed train batch 0 generator forward 44.741 ms, 3.02 s total +[ 2023-09-21 05:47:53 ] Completed train batch 0 generator loss: nan 66.353 ms, 3.09 s total +[ 2023-09-21 05:47:54 ] Completed train batch 0 generator backward 607.318 ms, 3.70 s total +[ 2023-09-21 05:47:54 ] Completed train batch 0 discriminator loss 0.040 61.459 ms, 3.76 s total +[ 2023-09-21 05:47:54 ] Completed train batch 0 discriminator backward 84.707 ms, 3.84 s total +Epoch [5] Step [0/2], gen_loss: nan, disc_loss: 0.013 +[ 2023-09-21 05:47:54 ] Completed train batch 0 metrics update 6.009 ms, 3.85 s total +Saving checkpoint at epoch 5 train batch 0 +[ 2023-09-21 05:47:55 ] Completed saving temp checkpoint 486.331 ms, 4.33 s total +[ 2023-09-21 05:47:55 ] Completed replacing temp checkpoint with checkpoint 33.656 ms, 4.37 s total +[ 2023-09-21 05:47:56 ] Completed train batch 1 to device 1,476.606 ms, 5.85 s total +[ 2023-09-21 05:47:56 ] Completed train batch 1 generator forward 42.024 ms, 5.89 s total +[ 2023-09-21 05:47:56 ] Completed train batch 1 generator loss: nan 63.460 ms, 5.95 s total +[ 2023-09-21 05:47:57 ] Completed train batch 1 generator backward 1,190.381 ms, 7.14 s total +[ 2023-09-21 05:47:57 ] Completed train batch 1 discriminator loss 0.036 56.555 ms, 7.20 s total +[ 2023-09-21 05:47:58 ] Completed train batch 1 discriminator backward 63.022 ms, 7.26 s total +Epoch [5] Step [1/2], gen_loss: nan, disc_loss: 0.012 +[ 2023-09-21 05:47:58 ] Completed train batch 1 metrics update 1.791 ms, 7.26 s total +Saving checkpoint at epoch 5 train batch 1 +[ 2023-09-21 05:47:58 ] Completed saving temp checkpoint 769.802 ms, 8.03 s total +[ 2023-09-21 05:47:58 ] Completed replacing temp checkpoint with checkpoint 31.174 ms, 8.06 s total +Epoch [5] :: gen_loss: nan, disc_loss: 0.013 +[ 2023-09-21 05:47:58 ] Completed training generator for epoch 5 54.946 ms, 8.12 s total +[ 2023-09-21 05:47:58 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 5 from eval step 0 + +[ 2023-09-21 05:48:00 ] Completed eval batch 0 to device 1,164.036 ms, 1.16 s total +[ 2023-09-21 05:48:00 ] Completed eval batch 0 forward 41.934 ms, 1.21 s total +[ 2023-09-21 05:48:00 ] Completed eval batch 0 recons_loss 0.216 ms, 1.21 s total +[ 2023-09-21 05:48:00 ] Completed eval batch 0 metrics update 0.484 ms, 1.21 s total +Saving checkpoint at epoch 5 val batch 0 +[ 2023-09-21 05:48:00 ] Completed saving temp checkpoint 405.120 ms, 1.61 s total +[ 2023-09-21 05:48:00 ] Completed replacing temp checkpoint with checkpoint 27.864 ms, 1.64 s total +[ 2023-09-21 05:48:00 ] Completed eval batch 1 to device 341.629 ms, 1.98 s total +[ 2023-09-21 05:48:00 ] Completed eval batch 1 forward 27.358 ms, 2.01 s total +[ 2023-09-21 05:48:00 ] Completed eval batch 1 recons_loss 0.365 ms, 2.01 s total +[ 2023-09-21 05:48:00 ] Completed eval batch 1 metrics update 0.319 ms, 2.01 s total +Saving checkpoint at epoch 5 val batch 1 +Epoch 5 val loss: 0.3652 +[ 2023-09-21 05:48:01 ] Completed saving temp checkpoint 459.710 ms, 2.47 s total +[ 2023-09-21 05:48:01 ] Completed replacing temp checkpoint with checkpoint 19.135 ms, 2.49 s total +[ 2023-09-21 05:48:01 ] Completed evaluating generator for epoch 5 51.450 ms, 2.54 s total + + +EPOCH :: 6 + + +[ 2023-09-21 05:48:01 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 6 from training step 0 + +[ 2023-09-21 05:48:04 ] Completed train batch 0 to device 3,051.714 ms, 3.05 s total +[ 2023-09-21 05:48:04 ] Completed train batch 0 generator forward 27.535 ms, 3.08 s total +[ 2023-09-21 05:48:04 ] Completed train batch 0 generator loss: 2.529 67.139 ms, 3.15 s total +[ 2023-09-21 05:48:05 ] Completed train batch 0 generator backward 565.106 ms, 3.71 s total +[ 2023-09-21 05:48:05 ] Completed train batch 0 discriminator loss 0.033 79.844 ms, 3.79 s total +[ 2023-09-21 05:48:05 ] Completed train batch 0 discriminator backward 69.594 ms, 3.86 s total +Epoch [6] Step [0/2], gen_loss: nan, disc_loss: 0.011 +[ 2023-09-21 05:48:05 ] Completed train batch 0 metrics update 0.817 ms, 3.86 s total +Saving checkpoint at epoch 6 train batch 0 +[ 2023-09-21 05:48:05 ] Completed saving temp checkpoint 455.497 ms, 4.32 s total +[ 2023-09-21 05:48:05 ] Completed replacing temp checkpoint with checkpoint 17.827 ms, 4.34 s total +[ 2023-09-21 05:48:07 ] Completed train batch 1 to device 1,413.333 ms, 5.75 s total +[ 2023-09-21 05:48:07 ] Completed train batch 1 generator forward 22.135 ms, 5.77 s total +[ 2023-09-21 05:48:07 ] Completed train batch 1 generator loss: 2.525 63.923 ms, 5.83 s total +[ 2023-09-21 05:48:08 ] Completed train batch 1 generator backward 1,410.301 ms, 7.24 s total +[ 2023-09-21 05:48:08 ] Completed train batch 1 discriminator loss 0.036 57.864 ms, 7.30 s total +[ 2023-09-21 05:48:08 ] Completed train batch 1 discriminator backward 56.641 ms, 7.36 s total +Epoch [6] Step [1/2], gen_loss: 0.840, disc_loss: 0.012 +[ 2023-09-21 05:48:08 ] Completed train batch 1 metrics update 0.922 ms, 7.36 s total +Saving checkpoint at epoch 6 train batch 1 +[ 2023-09-21 05:48:09 ] Completed saving temp checkpoint 460.122 ms, 7.82 s total +[ 2023-09-21 05:48:09 ] Completed replacing temp checkpoint with checkpoint 27.414 ms, 7.85 s total +Epoch [6] :: gen_loss: nan, disc_loss: 0.012 +[ 2023-09-21 05:48:09 ] Completed training generator for epoch 6 67.314 ms, 7.92 s total +[ 2023-09-21 05:48:09 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 6 from eval step 0 + +[ 2023-09-21 05:48:10 ] Completed eval batch 0 to device 1,080.185 ms, 1.08 s total +[ 2023-09-21 05:48:10 ] Completed eval batch 0 forward 43.646 ms, 1.12 s total +[ 2023-09-21 05:48:10 ] Completed eval batch 0 recons_loss 0.311 ms, 1.12 s total +[ 2023-09-21 05:48:10 ] Completed eval batch 0 metrics update 0.599 ms, 1.12 s total +Saving checkpoint at epoch 6 val batch 0 +[ 2023-09-21 05:48:10 ] Completed saving temp checkpoint 428.917 ms, 1.55 s total +[ 2023-09-21 05:48:10 ] Completed replacing temp checkpoint with checkpoint 35.399 ms, 1.59 s total +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 05:59:18 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 05:59:18 ] Completed importing Timer 0.023 ms, 0.00 s total +[ 2023-09-21 05:59:23 ] Completed importing everything else 4,955.559 ms, 4.96 s total +| distributed init (rank 0): env:// +| distributed init (rank 3): env:// +| distributed init (rank 2): env:// +| distributed init (rank 1): env:// +| distributed init (rank 4): env:// +| distributed init (rank 5): env:// +[ 2023-09-21 05:59:31 ] Completed preliminaries 7,795.846 ms, 12.75 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 05:59:31 ] Completed build datasets 11.824 ms, 12.76 s total +[ 2023-09-21 05:59:31 ] Completed build samplers 0.046 ms, 12.76 s total +[ 2023-09-21 05:59:31 ] Completed build dataloaders 0.134 ms, 12.76 s total +[ 2023-09-21 05:59:31 ] Completed generator to device 74.621 ms, 12.84 s total +[ 2023-09-21 05:59:31 ] Completed discriminator to device 7.130 ms, 12.85 s total +[ 2023-09-21 05:59:32 ] Completed loss functions 412.476 ms, 13.26 s total +[ 2023-09-21 05:59:32 ] Completed models prepped for distribution 69.399 ms, 13.33 s total +[ 2023-09-21 05:59:32 ] Completed optimizers 0.682 ms, 13.33 s total +[ 2023-09-21 05:59:32 ] Completed grad scalers 0.037 ms, 13.33 s total +[ 2023-09-21 05:59:32 ] Completed checkpoint retrieval 438.620 ms, 13.77 s total + + +EPOCH :: 6 + + +[ 2023-09-21 05:59:32 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 6 from training step 2 + +Epoch [6] :: gen_loss: nan, disc_loss: 0.012 +[ 2023-09-21 05:59:32 ] Completed training generator for epoch 6 132.440 ms, 0.13 s total +[ 2023-09-21 05:59:32 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 6 from eval step 1 + +[ 2023-09-21 05:59:34 ] Completed eval batch 1 to device 1,321.682 ms, 1.32 s total +[ 2023-09-21 05:59:35 ] Completed eval batch 1 forward 1,167.208 ms, 2.49 s total +[ 2023-09-21 05:59:35 ] Completed eval batch 1 recons_loss 0.266 ms, 2.49 s total +[ 2023-09-21 05:59:35 ] Completed eval batch 1 metrics update 404.749 ms, 2.89 s total +Saving checkpoint at epoch 6 val batch 1 +Epoch 6 val loss: 0.2630 +[ 2023-09-21 05:59:36 ] Completed saving temp checkpoint 466.935 ms, 3.36 s total +[ 2023-09-21 05:59:36 ] Completed replacing temp checkpoint with checkpoint 134.088 ms, 3.49 s total +[ 2023-09-21 05:59:36 ] Completed evaluating generator for epoch 6 48.808 ms, 3.54 s total + + +EPOCH :: 7 + + +[ 2023-09-21 05:59:36 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 7 from training step 0 + +[ 2023-09-21 05:59:39 ] Completed train batch 0 to device 3,009.194 ms, 3.01 s total +[ 2023-09-21 05:59:39 ] Completed train batch 0 generator forward 70.285 ms, 3.08 s total +[ 2023-09-21 05:59:39 ] Completed train batch 0 generator loss: 2.295 81.576 ms, 3.16 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 05:59:40 ] Completed train batch 0 generator backward 685.762 ms, 3.85 s total +[ 2023-09-21 05:59:40 ] Completed train batch 0 discriminator loss 0.120 58.438 ms, 3.91 s total +[ 2023-09-21 05:59:40 ] Completed train batch 0 discriminator backward 135.834 ms, 4.04 s total +Epoch [7] Step [0/2], gen_loss: 0.769, disc_loss: 0.040 +[ 2023-09-21 05:59:40 ] Completed train batch 0 metrics update 0.744 ms, 4.04 s total +Saving checkpoint at epoch 7 train batch 0 +[ 2023-09-21 05:59:40 ] Completed saving temp checkpoint 399.034 ms, 4.44 s total +[ 2023-09-21 05:59:40 ] Completed replacing temp checkpoint with checkpoint 21.614 ms, 4.46 s total +[ 2023-09-21 05:59:42 ] Completed train batch 1 to device 1,383.704 ms, 5.85 s total +[ 2023-09-21 05:59:42 ] Completed train batch 1 generator forward 41.533 ms, 5.89 s total +[ 2023-09-21 05:59:42 ] Completed train batch 1 generator loss: 2.237 63.407 ms, 5.95 s total +[ 2023-09-21 05:59:43 ] Completed train batch 1 generator backward 1,477.580 ms, 7.43 s total +[ 2023-09-21 05:59:43 ] Completed train batch 1 discriminator loss 0.294 56.625 ms, 7.49 s total +[ 2023-09-21 05:59:43 ] Completed train batch 1 discriminator backward 77.874 ms, 7.56 s total +Epoch [7] Step [1/2], gen_loss: 0.750, disc_loss: 0.098 +[ 2023-09-21 05:59:43 ] Completed train batch 1 metrics update 2.224 ms, 7.57 s total +Saving checkpoint at epoch 7 train batch 1 +[ 2023-09-21 05:59:44 ] Completed saving temp checkpoint 421.899 ms, 7.99 s total +[ 2023-09-21 05:59:44 ] Completed replacing temp checkpoint with checkpoint 19.280 ms, 8.01 s total +Epoch [7] :: gen_loss: 0.759, disc_loss: 0.069 +[ 2023-09-21 05:59:44 ] Completed training generator for epoch 7 47.638 ms, 8.05 s total +[ 2023-09-21 05:59:44 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 7 from eval step 0 + +[ 2023-09-21 05:59:45 ] Completed eval batch 0 to device 1,177.301 ms, 1.18 s total +[ 2023-09-21 05:59:45 ] Completed eval batch 0 forward 30.266 ms, 1.21 s total +[ 2023-09-21 05:59:45 ] Completed eval batch 0 recons_loss 0.335 ms, 1.21 s total +[ 2023-09-21 05:59:45 ] Completed eval batch 0 metrics update 0.606 ms, 1.21 s total +Saving checkpoint at epoch 7 val batch 0 +[ 2023-09-21 05:59:46 ] Completed saving temp checkpoint 398.766 ms, 1.61 s total +[ 2023-09-21 05:59:46 ] Completed replacing temp checkpoint with checkpoint 21.775 ms, 1.63 s total +[ 2023-09-21 05:59:46 ] Completed eval batch 1 to device 447.385 ms, 2.08 s total +[ 2023-09-21 05:59:46 ] Completed eval batch 1 forward 40.043 ms, 2.12 s total +[ 2023-09-21 05:59:46 ] Completed eval batch 1 recons_loss 0.591 ms, 2.12 s total +[ 2023-09-21 05:59:46 ] Completed eval batch 1 metrics update 297.923 ms, 2.41 s total +Saving checkpoint at epoch 7 val batch 1 +Epoch 7 val loss: 0.2485 +[ 2023-09-21 05:59:47 ] Completed saving temp checkpoint 415.167 ms, 2.83 s total +[ 2023-09-21 05:59:47 ] Completed replacing temp checkpoint with checkpoint 31.136 ms, 2.86 s total +[ 2023-09-21 05:59:47 ] Completed evaluating generator for epoch 7 48.140 ms, 2.91 s total + + +EPOCH :: 8 + + +[ 2023-09-21 05:59:47 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 8 from training step 0 + +[ 2023-09-21 05:59:50 ] Completed train batch 0 to device 3,099.081 ms, 3.10 s total +[ 2023-09-21 05:59:50 ] Completed train batch 0 generator forward 44.707 ms, 3.14 s total +[ 2023-09-21 05:59:50 ] Completed train batch 0 generator loss: 2.214 66.533 ms, 3.21 s total +[ 2023-09-21 05:59:51 ] Completed train batch 0 generator backward 701.440 ms, 3.91 s total +[ 2023-09-21 05:59:51 ] Completed train batch 0 discriminator loss 0.248 58.150 ms, 3.97 s total +[ 2023-09-21 05:59:51 ] Completed train batch 0 discriminator backward 106.197 ms, 4.08 s total +Epoch [8] Step [0/2], gen_loss: 0.730, disc_loss: 0.083 +[ 2023-09-21 05:59:51 ] Completed train batch 0 metrics update 3.011 ms, 4.08 s total +Saving checkpoint at epoch 8 train batch 0 +[ 2023-09-21 05:59:51 ] Completed saving temp checkpoint 405.351 ms, 4.48 s total +[ 2023-09-21 05:59:51 ] Completed replacing temp checkpoint with checkpoint 23.332 ms, 4.51 s total +[ 2023-09-21 05:59:53 ] Completed train batch 1 to device 1,541.309 ms, 6.05 s total +[ 2023-09-21 05:59:53 ] Completed train batch 1 generator forward 22.959 ms, 6.07 s total +[ 2023-09-21 05:59:53 ] Completed train batch 1 generator loss: 2.356 64.554 ms, 6.14 s total +[ 2023-09-21 05:59:54 ] Completed train batch 1 generator backward 1,474.490 ms, 7.61 s total +[ 2023-09-21 05:59:55 ] Completed train batch 1 discriminator loss 0.197 56.802 ms, 7.67 s total +[ 2023-09-21 05:59:55 ] Completed train batch 1 discriminator backward 82.041 ms, 7.75 s total +Epoch [8] Step [1/2], gen_loss: 0.782, disc_loss: 0.066 +[ 2023-09-21 05:59:55 ] Completed train batch 1 metrics update 5.189 ms, 7.76 s total +Saving checkpoint at epoch 8 train batch 1 +[ 2023-09-21 05:59:55 ] Completed saving temp checkpoint 422.054 ms, 8.18 s total +[ 2023-09-21 05:59:55 ] Completed replacing temp checkpoint with checkpoint 15.549 ms, 8.19 s total +Epoch [8] :: gen_loss: 0.756, disc_loss: 0.074 +[ 2023-09-21 05:59:55 ] Completed training generator for epoch 8 54.933 ms, 8.25 s total +[ 2023-09-21 05:59:55 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 8 from eval step 0 + +[ 2023-09-21 05:59:56 ] Completed eval batch 0 to device 1,257.212 ms, 1.26 s total +[ 2023-09-21 05:59:56 ] Completed eval batch 0 forward 24.573 ms, 1.28 s total +[ 2023-09-21 05:59:56 ] Completed eval batch 0 recons_loss 0.271 ms, 1.28 s total +[ 2023-09-21 05:59:56 ] Completed eval batch 0 metrics update 0.557 ms, 1.28 s total +Saving checkpoint at epoch 8 val batch 0 +[ 2023-09-21 05:59:57 ] Completed saving temp checkpoint 393.026 ms, 1.68 s total +[ 2023-09-21 05:59:57 ] Completed replacing temp checkpoint with checkpoint 24.425 ms, 1.70 s total +[ 2023-09-21 05:59:57 ] Completed eval batch 1 to device 572.233 ms, 2.27 s total +[ 2023-09-21 05:59:57 ] Completed eval batch 1 forward 22.035 ms, 2.29 s total +[ 2023-09-21 05:59:57 ] Completed eval batch 1 recons_loss 0.386 ms, 2.29 s total +[ 2023-09-21 05:59:57 ] Completed eval batch 1 metrics update 82.575 ms, 2.38 s total +Saving checkpoint at epoch 8 val batch 1 +Epoch 8 val loss: 0.4310 +[ 2023-09-21 05:59:58 ] Completed saving temp checkpoint 419.567 ms, 2.80 s total +[ 2023-09-21 05:59:58 ] Completed replacing temp checkpoint with checkpoint 25.248 ms, 2.82 s total +[ 2023-09-21 05:59:58 ] Completed evaluating generator for epoch 8 48.283 ms, 2.87 s total + + +EPOCH :: 9 + + +[ 2023-09-21 05:59:58 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 9 from training step 0 + +[ 2023-09-21 06:00:01 ] Completed train batch 0 to device 3,289.524 ms, 3.29 s total +[ 2023-09-21 06:00:01 ] Completed train batch 0 generator forward 27.593 ms, 3.32 s total +[ 2023-09-21 06:00:01 ] Completed train batch 0 generator loss: 2.384 67.243 ms, 3.38 s total +[ 2023-09-21 06:00:02 ] Completed train batch 0 generator backward 583.331 ms, 3.97 s total +[ 2023-09-21 06:00:02 ] Completed train batch 0 discriminator loss 0.172 57.930 ms, 4.03 s total +[ 2023-09-21 06:00:02 ] Completed train batch 0 discriminator backward 97.708 ms, 4.12 s total +Epoch [9] Step [0/2], gen_loss: 0.793, disc_loss: 0.057 +[ 2023-09-21 06:00:02 ] Completed train batch 0 metrics update 7.794 ms, 4.13 s total +Saving checkpoint at epoch 9 train batch 0 +[ 2023-09-21 06:00:03 ] Completed saving temp checkpoint 453.248 ms, 4.58 s total +[ 2023-09-21 06:00:03 ] Completed replacing temp checkpoint with checkpoint 19.509 ms, 4.60 s total +[ 2023-09-21 06:00:04 ] Completed train batch 1 to device 1,836.372 ms, 6.44 s total +[ 2023-09-21 06:00:04 ] Completed train batch 1 generator forward 22.679 ms, 6.46 s total +[ 2023-09-21 06:00:05 ] Completed train batch 1 generator loss: 2.349 64.442 ms, 6.53 s total +[ 2023-09-21 06:00:06 ] Completed train batch 1 generator backward 1,083.678 ms, 7.61 s total +[ 2023-09-21 06:00:06 ] Completed train batch 1 discriminator loss 0.156 57.158 ms, 7.67 s total +[ 2023-09-21 06:00:06 ] Completed train batch 1 discriminator backward 75.780 ms, 7.74 s total +Epoch [9] Step [1/2], gen_loss: 0.793, disc_loss: 0.051 +[ 2023-09-21 06:00:06 ] Completed train batch 1 metrics update 1.662 ms, 7.75 s total +Saving checkpoint at epoch 9 train batch 1 +[ 2023-09-21 06:00:06 ] Completed saving temp checkpoint 427.739 ms, 8.17 s total +[ 2023-09-21 06:00:06 ] Completed replacing temp checkpoint with checkpoint 18.060 ms, 8.19 s total +Epoch [9] :: gen_loss: 0.794, disc_loss: 0.054 +[ 2023-09-21 06:00:06 ] Completed training generator for epoch 9 50.404 ms, 8.24 s total +[ 2023-09-21 06:00:06 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 9 from eval step 0 + +[ 2023-09-21 06:00:07 ] Completed eval batch 0 to device 1,182.060 ms, 1.18 s total +[ 2023-09-21 06:00:07 ] Completed eval batch 0 forward 28.041 ms, 1.21 s total +[ 2023-09-21 06:00:07 ] Completed eval batch 0 recons_loss 0.260 ms, 1.21 s total +[ 2023-09-21 06:00:07 ] Completed eval batch 0 metrics update 0.622 ms, 1.21 s total +Saving checkpoint at epoch 9 val batch 0 +[ 2023-09-21 06:00:08 ] Completed saving temp checkpoint 397.001 ms, 1.61 s total +[ 2023-09-21 06:00:08 ] Completed replacing temp checkpoint with checkpoint 34.948 ms, 1.64 s total +[ 2023-09-21 06:00:08 ] Completed eval batch 1 to device 608.771 ms, 2.25 s total +[ 2023-09-21 06:00:09 ] Completed eval batch 1 forward 22.189 ms, 2.27 s total +[ 2023-09-21 06:00:09 ] Completed eval batch 1 recons_loss 0.378 ms, 2.27 s total +[ 2023-09-21 06:00:09 ] Completed eval batch 1 metrics update 10.141 ms, 2.28 s total +Saving checkpoint at epoch 9 val batch 1 +Epoch 9 val loss: 0.2821 +[ 2023-09-21 06:00:09 ] Completed saving temp checkpoint 431.807 ms, 2.72 s total +[ 2023-09-21 06:00:09 ] Completed replacing temp checkpoint with checkpoint 22.822 ms, 2.74 s total +[ 2023-09-21 06:00:09 ] Completed evaluating generator for epoch 9 48.145 ms, 2.79 s total + + +EPOCH :: 10 + + +[ 2023-09-21 06:00:09 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 10 from training step 0 + +[ 2023-09-21 06:00:12 ] Completed train batch 0 to device 3,370.513 ms, 3.37 s total +[ 2023-09-21 06:00:12 ] Completed train batch 0 generator forward 26.396 ms, 3.40 s total +[ 2023-09-21 06:00:12 ] Completed train batch 0 generator loss: 2.241 65.611 ms, 3.46 s total +[ 2023-09-21 06:00:13 ] Completed train batch 0 generator backward 432.115 ms, 3.89 s total +[ 2023-09-21 06:00:13 ] Completed train batch 0 discriminator loss 0.153 57.791 ms, 3.95 s total +[ 2023-09-21 06:00:13 ] Completed train batch 0 discriminator backward 88.302 ms, 4.04 s total +Epoch [10] Step [0/2], gen_loss: 0.747, disc_loss: 0.053 +[ 2023-09-21 06:00:13 ] Completed train batch 0 metrics update 2.264 ms, 4.04 s total +Saving checkpoint at epoch 10 train batch 0 +[ 2023-09-21 06:00:13 ] Completed saving temp checkpoint 407.946 ms, 4.45 s total +[ 2023-09-21 06:00:13 ] Completed replacing temp checkpoint with checkpoint 23.869 ms, 4.47 s total +[ 2023-09-21 06:00:16 ] Completed train batch 1 to device 2,290.220 ms, 6.77 s total +[ 2023-09-21 06:00:16 ] Completed train batch 1 generator forward 41.284 ms, 6.81 s total +[ 2023-09-21 06:00:16 ] Completed train batch 1 generator loss: 2.050 63.915 ms, 6.87 s total +[ 2023-09-21 06:00:16 ] Completed train batch 1 generator backward 311.262 ms, 7.18 s total +[ 2023-09-21 06:00:16 ] Completed train batch 1 discriminator loss 0.152 56.414 ms, 7.24 s total +[ 2023-09-21 06:00:16 ] Completed train batch 1 discriminator backward 94.204 ms, 7.33 s total +Epoch [10] Step [1/2], gen_loss: 0.697, disc_loss: 0.051 +[ 2023-09-21 06:00:16 ] Completed train batch 1 metrics update 0.516 ms, 7.33 s total +Saving checkpoint at epoch 10 train batch 1 +[ 2023-09-21 06:00:17 ] Completed saving temp checkpoint 455.703 ms, 7.79 s total +[ 2023-09-21 06:00:17 ] Completed replacing temp checkpoint with checkpoint 31.564 ms, 7.82 s total +Epoch [10] :: gen_loss: 0.722, disc_loss: 0.052 +[ 2023-09-21 06:00:17 ] Completed training generator for epoch 10 53.745 ms, 7.87 s total +[ 2023-09-21 06:00:17 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 10 from eval step 0 + +[ 2023-09-21 06:00:18 ] Completed eval batch 0 to device 1,054.950 ms, 1.05 s total +[ 2023-09-21 06:00:18 ] Completed eval batch 0 forward 24.740 ms, 1.08 s total +[ 2023-09-21 06:00:18 ] Completed eval batch 0 recons_loss 0.241 ms, 1.08 s total +[ 2023-09-21 06:00:18 ] Completed eval batch 0 metrics update 0.545 ms, 1.08 s total +Saving checkpoint at epoch 10 val batch 0 +[ 2023-09-21 06:00:18 ] Completed saving temp checkpoint 377.685 ms, 1.46 s total +[ 2023-09-21 06:00:18 ] Completed replacing temp checkpoint with checkpoint 29.832 ms, 1.49 s total +[ 2023-09-21 06:00:19 ] Completed eval batch 1 to device 439.378 ms, 1.93 s total +[ 2023-09-21 06:00:19 ] Completed eval batch 1 forward 40.508 ms, 1.97 s total +[ 2023-09-21 06:00:19 ] Completed eval batch 1 recons_loss 0.378 ms, 1.97 s total +[ 2023-09-21 06:00:19 ] Completed eval batch 1 metrics update 293.240 ms, 2.26 s total +Saving checkpoint at epoch 10 val batch 1 +Epoch 10 val loss: 0.2217 +[ 2023-09-21 06:00:20 ] Completed saving temp checkpoint 384.638 ms, 2.65 s total +[ 2023-09-21 06:00:20 ] Completed replacing temp checkpoint with checkpoint 15.443 ms, 2.66 s total +[ 2023-09-21 06:00:20 ] Completed evaluating generator for epoch 10 66.866 ms, 2.73 s total + + +EPOCH :: 11 + + +[ 2023-09-21 06:00:20 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 11 from training step 0 + +[ 2023-09-21 06:00:23 ] Completed train batch 0 to device 3,233.901 ms, 3.23 s total +[ 2023-09-21 06:00:23 ] Completed train batch 0 generator forward 43.434 ms, 3.28 s total +[ 2023-09-21 06:00:23 ] Completed train batch 0 generator loss: 2.221 78.511 ms, 3.36 s total +[ 2023-09-21 06:00:24 ] Completed train batch 0 generator backward 542.500 ms, 3.90 s total +[ 2023-09-21 06:00:24 ] Completed train batch 0 discriminator loss 0.131 62.383 ms, 3.96 s total +[ 2023-09-21 06:00:24 ] Completed train batch 0 discriminator backward 101.983 ms, 4.06 s total +Epoch [11] Step [0/2], gen_loss: 0.747, disc_loss: 0.045 +[ 2023-09-21 06:00:24 ] Completed train batch 0 metrics update 7.883 ms, 4.07 s total +Saving checkpoint at epoch 11 train batch 0 +[ 2023-09-21 06:00:24 ] Completed saving temp checkpoint 457.511 ms, 4.53 s total +[ 2023-09-21 06:00:24 ] Completed replacing temp checkpoint with checkpoint 23.285 ms, 4.55 s total +[ 2023-09-21 06:00:26 ] Completed train batch 1 to device 2,242.198 ms, 6.79 s total +[ 2023-09-21 06:00:26 ] Completed train batch 1 generator forward 27.855 ms, 6.82 s total +[ 2023-09-21 06:00:27 ] Completed train batch 1 generator loss: 2.115 64.395 ms, 6.89 s total +[ 2023-09-21 06:00:27 ] Completed train batch 1 generator backward 565.879 ms, 7.45 s total +[ 2023-09-21 06:00:27 ] Completed train batch 1 discriminator loss 0.110 56.976 ms, 7.51 s total +[ 2023-09-21 06:00:27 ] Completed train batch 1 discriminator backward 73.999 ms, 7.58 s total +Epoch [11] Step [1/2], gen_loss: 0.690, disc_loss: 0.037 +[ 2023-09-21 06:00:27 ] Completed train batch 1 metrics update 3.654 ms, 7.59 s total +Saving checkpoint at epoch 11 train batch 1 +[ 2023-09-21 06:00:28 ] Completed saving temp checkpoint 467.824 ms, 8.05 s total +[ 2023-09-21 06:00:28 ] Completed replacing temp checkpoint with checkpoint 27.784 ms, 8.08 s total +Epoch [11] :: gen_loss: 0.718, disc_loss: 0.041 +[ 2023-09-21 06:00:28 ] Completed training generator for epoch 11 56.057 ms, 8.14 s total +[ 2023-09-21 06:00:28 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 11 from eval step 0 + +[ 2023-09-21 06:00:29 ] Completed eval batch 0 to device 1,188.366 ms, 1.19 s total +[ 2023-09-21 06:00:29 ] Completed eval batch 0 forward 43.755 ms, 1.23 s total +[ 2023-09-21 06:00:29 ] Completed eval batch 0 recons_loss 0.322 ms, 1.23 s total +[ 2023-09-21 06:00:29 ] Completed eval batch 0 metrics update 0.622 ms, 1.23 s total +Saving checkpoint at epoch 11 val batch 0 +[ 2023-09-21 06:00:29 ] Completed saving temp checkpoint 416.438 ms, 1.65 s total +[ 2023-09-21 06:00:29 ] Completed replacing temp checkpoint with checkpoint 31.206 ms, 1.68 s total +[ 2023-09-21 06:00:30 ] Completed eval batch 1 to device 403.993 ms, 2.08 s total +[ 2023-09-21 06:00:30 ] Completed eval batch 1 forward 38.467 ms, 2.12 s total +[ 2023-09-21 06:00:30 ] Completed eval batch 1 recons_loss 0.384 ms, 2.12 s total +[ 2023-09-21 06:00:30 ] Completed eval batch 1 metrics update 128.345 ms, 2.25 s total +Saving checkpoint at epoch 11 val batch 1 +Epoch 11 val loss: 0.2417 +[ 2023-09-21 06:00:30 ] Completed saving temp checkpoint 439.265 ms, 2.69 s total +[ 2023-09-21 06:00:30 ] Completed replacing temp checkpoint with checkpoint 41.616 ms, 2.73 s total +[ 2023-09-21 06:00:31 ] Completed evaluating generator for epoch 11 64.451 ms, 2.80 s total + + +EPOCH :: 12 + + +[ 2023-09-21 06:00:31 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 12 from training step 0 + +[ 2023-09-21 06:00:34 ] Completed train batch 0 to device 3,088.916 ms, 3.09 s total +[ 2023-09-21 06:00:34 ] Completed train batch 0 generator forward 43.973 ms, 3.13 s total +[ 2023-09-21 06:00:34 ] Completed train batch 0 generator loss: 2.049 64.833 ms, 3.20 s total +[ 2023-09-21 06:00:34 ] Completed train batch 0 generator backward 617.891 ms, 3.82 s total +[ 2023-09-21 06:00:34 ] Completed train batch 0 discriminator loss 0.122 57.490 ms, 3.87 s total +[ 2023-09-21 06:00:35 ] Completed train batch 0 discriminator backward 114.083 ms, 3.99 s total +Epoch [12] Step [0/2], gen_loss: 0.696, disc_loss: 0.041 +[ 2023-09-21 06:00:35 ] Completed train batch 0 metrics update 19.901 ms, 4.01 s total +Saving checkpoint at epoch 12 train batch 0 +[ 2023-09-21 06:00:35 ] Completed saving temp checkpoint 413.645 ms, 4.42 s total +[ 2023-09-21 06:00:35 ] Completed replacing temp checkpoint with checkpoint 24.526 ms, 4.45 s total +[ 2023-09-21 06:00:36 ] Completed train batch 1 to device 1,346.627 ms, 5.79 s total +[ 2023-09-21 06:00:36 ] Completed train batch 1 generator forward 41.295 ms, 5.83 s total +[ 2023-09-21 06:00:36 ] Completed train batch 1 generator loss: 2.142 64.071 ms, 5.90 s total +[ 2023-09-21 06:00:38 ] Completed train batch 1 generator backward 1,254.846 ms, 7.15 s total +[ 2023-09-21 06:00:38 ] Completed train batch 1 discriminator loss 0.140 56.630 ms, 7.21 s total +[ 2023-09-21 06:00:38 ] Completed train batch 1 discriminator backward 84.973 ms, 7.29 s total +Epoch [12] Step [1/2], gen_loss: 0.702, disc_loss: 0.045 +[ 2023-09-21 06:00:38 ] Completed train batch 1 metrics update 2.192 ms, 7.30 s total +Saving checkpoint at epoch 12 train batch 1 +[ 2023-09-21 06:00:38 ] Completed saving temp checkpoint 430.741 ms, 7.73 s total +[ 2023-09-21 06:00:38 ] Completed replacing temp checkpoint with checkpoint 24.532 ms, 7.75 s total +Epoch [12] :: gen_loss: 0.699, disc_loss: 0.043 +[ 2023-09-21 06:00:38 ] Completed training generator for epoch 12 61.222 ms, 7.81 s total +[ 2023-09-21 06:00:38 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 12 from eval step 0 + +[ 2023-09-21 06:00:40 ] Completed eval batch 0 to device 1,194.088 ms, 1.19 s total +[ 2023-09-21 06:00:40 ] Completed eval batch 0 forward 30.534 ms, 1.22 s total +[ 2023-09-21 06:00:40 ] Completed eval batch 0 recons_loss 0.290 ms, 1.22 s total +[ 2023-09-21 06:00:40 ] Completed eval batch 0 metrics update 0.693 ms, 1.23 s total +Saving checkpoint at epoch 12 val batch 0 +[ 2023-09-21 06:00:40 ] Completed saving temp checkpoint 521.137 ms, 1.75 s total +[ 2023-09-21 06:00:40 ] Completed replacing temp checkpoint with checkpoint 23.377 ms, 1.77 s total +[ 2023-09-21 06:00:41 ] Completed eval batch 1 to device 468.025 ms, 2.24 s total +[ 2023-09-21 06:00:41 ] Completed eval batch 1 forward 21.786 ms, 2.26 s total +[ 2023-09-21 06:00:41 ] Completed eval batch 1 recons_loss 0.373 ms, 2.26 s total +[ 2023-09-21 06:00:41 ] Completed eval batch 1 metrics update 65.120 ms, 2.33 s total +Saving checkpoint at epoch 12 val batch 1 +Epoch 12 val loss: 0.2570 +[ 2023-09-21 06:00:41 ] Completed saving temp checkpoint 735.990 ms, 3.06 s total +[ 2023-09-21 06:00:41 ] Completed replacing temp checkpoint with checkpoint 31.567 ms, 3.09 s total +[ 2023-09-21 06:00:42 ] Completed evaluating generator for epoch 12 66.252 ms, 3.16 s total + + +EPOCH :: 13 + + +[ 2023-09-21 06:00:42 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 13 from training step 0 + +[ 2023-09-21 06:00:45 ] Completed train batch 0 to device 3,123.151 ms, 3.12 s total +[ 2023-09-21 06:00:45 ] Completed train batch 0 generator forward 30.793 ms, 3.15 s total +[ 2023-09-21 06:00:45 ] Completed train batch 0 generator loss: 2.079 65.580 ms, 3.22 s total +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 06:12:16 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 06:12:16 ] Completed importing Timer 0.023 ms, 0.00 s total +[ 2023-09-21 06:12:21 ] Completed importing everything else 4,545.767 ms, 4.55 s total +| distributed init (rank 1): env:// +| distributed init (rank 3): env:// +| distributed init (rank 4): env:// +| distributed init (rank 0): env:// +| distributed init (rank 5): env:// +| distributed init (rank 2): env:// +[ 2023-09-21 06:12:28 ] Completed preliminaries 7,524.959 ms, 12.07 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 06:12:28 ] Completed build datasets 10.408 ms, 12.08 s total +[ 2023-09-21 06:12:28 ] Completed build samplers 0.042 ms, 12.08 s total +[ 2023-09-21 06:12:28 ] Completed build dataloaders 0.127 ms, 12.08 s total +[ 2023-09-21 06:12:28 ] Completed generator to device 73.793 ms, 12.16 s total +[ 2023-09-21 06:12:28 ] Completed discriminator to device 7.053 ms, 12.16 s total +[ 2023-09-21 06:12:29 ] Completed loss functions 425.567 ms, 12.59 s total +[ 2023-09-21 06:12:29 ] Completed models prepped for distribution 68.759 ms, 12.66 s total +[ 2023-09-21 06:12:29 ] Completed optimizers 0.679 ms, 12.66 s total +[ 2023-09-21 06:12:29 ] Completed grad scalers 0.027 ms, 12.66 s total +[ 2023-09-21 06:12:29 ] Completed checkpoint retrieval 469.977 ms, 13.13 s total + + +EPOCH :: 12 + + +[ 2023-09-21 06:12:29 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 12 from training step 2 + +Epoch [12] :: gen_loss: 0.699, disc_loss: 0.043 +[ 2023-09-21 06:12:29 ] Completed training generator for epoch 12 129.347 ms, 0.13 s total +[ 2023-09-21 06:12:29 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 12 from eval step 2 + +[ 2023-09-21 06:12:29 ] Completed evaluating generator for epoch 12 129.344 ms, 0.13 s total + + +EPOCH :: 13 + + +[ 2023-09-21 06:12:29 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 13 from training step 0 + +[ 2023-09-21 06:12:33 ] Completed train batch 0 to device 3,636.977 ms, 3.64 s total +[ 2023-09-21 06:12:34 ] Completed train batch 0 generator forward 1,311.772 ms, 4.95 s total +[ 2023-09-21 06:12:34 ] Completed train batch 0 generator loss: 2.058 75.786 ms, 5.02 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 06:12:36 ] Completed train batch 0 generator backward 1,183.120 ms, 6.21 s total +[ 2023-09-21 06:12:36 ] Completed train batch 0 discriminator loss 0.143 55.956 ms, 6.26 s total +[ 2023-09-21 06:12:36 ] Completed train batch 0 discriminator backward 98.642 ms, 6.36 s total +Epoch [13] Step [0/2], gen_loss: 0.685, disc_loss: 0.049 +[ 2023-09-21 06:12:36 ] Completed train batch 0 metrics update 7.430 ms, 6.37 s total +Saving checkpoint at epoch 13 train batch 0 +[ 2023-09-21 06:12:36 ] Completed saving temp checkpoint 415.744 ms, 6.79 s total +[ 2023-09-21 06:12:36 ] Completed replacing temp checkpoint with checkpoint 119.098 ms, 6.90 s total +[ 2023-09-21 06:12:37 ] Completed train batch 1 to device 344.769 ms, 7.25 s total +[ 2023-09-21 06:12:37 ] Completed train batch 1 generator forward 22.179 ms, 7.27 s total +[ 2023-09-21 06:12:37 ] Completed train batch 1 generator loss: 1.970 62.843 ms, 7.33 s total +[ 2023-09-21 06:12:38 ] Completed train batch 1 generator backward 1,143.028 ms, 8.48 s total +[ 2023-09-21 06:12:38 ] Completed train batch 1 discriminator loss 0.151 56.105 ms, 8.53 s total +[ 2023-09-21 06:12:38 ] Completed train batch 1 discriminator backward 71.245 ms, 8.60 s total +Epoch [13] Step [1/2], gen_loss: 0.665, disc_loss: 0.050 +[ 2023-09-21 06:12:38 ] Completed train batch 1 metrics update 3.547 ms, 8.61 s total +Saving checkpoint at epoch 13 train batch 1 +[ 2023-09-21 06:12:38 ] Completed saving temp checkpoint 438.367 ms, 9.05 s total +[ 2023-09-21 06:12:38 ] Completed replacing temp checkpoint with checkpoint 24.197 ms, 9.07 s total +Epoch [13] :: gen_loss: 0.675, disc_loss: 0.050 +[ 2023-09-21 06:12:38 ] Completed training generator for epoch 13 33.344 ms, 9.10 s total +[ 2023-09-21 06:12:38 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 13 from eval step 0 + +[ 2023-09-21 06:12:40 ] Completed eval batch 0 to device 1,076.729 ms, 1.08 s total +[ 2023-09-21 06:12:40 ] Completed eval batch 0 forward 25.197 ms, 1.10 s total +[ 2023-09-21 06:12:40 ] Completed eval batch 0 recons_loss 0.303 ms, 1.10 s total +[ 2023-09-21 06:12:40 ] Completed eval batch 0 metrics update 0.586 ms, 1.10 s total +Saving checkpoint at epoch 13 val batch 0 +[ 2023-09-21 06:12:40 ] Completed saving temp checkpoint 481.183 ms, 1.58 s total +[ 2023-09-21 06:12:40 ] Completed replacing temp checkpoint with checkpoint 25.016 ms, 1.61 s total +[ 2023-09-21 06:12:40 ] Completed eval batch 1 to device 329.505 ms, 1.94 s total +[ 2023-09-21 06:12:40 ] Completed eval batch 1 forward 40.561 ms, 1.98 s total +[ 2023-09-21 06:12:40 ] Completed eval batch 1 recons_loss 0.337 ms, 1.98 s total +[ 2023-09-21 06:12:41 ] Completed eval batch 1 metrics update 259.834 ms, 2.24 s total +Saving checkpoint at epoch 13 val batch 1 +Epoch 13 val loss: 0.1831 +[ 2023-09-21 06:12:41 ] Completed saving temp checkpoint 551.265 ms, 2.79 s total +[ 2023-09-21 06:12:41 ] Completed replacing temp checkpoint with checkpoint 30.475 ms, 2.82 s total +[ 2023-09-21 06:12:41 ] Completed evaluating generator for epoch 13 63.648 ms, 2.88 s total + + +EPOCH :: 14 + + +[ 2023-09-21 06:12:41 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 14 from training step 0 + +[ 2023-09-21 06:12:44 ] Completed train batch 0 to device 2,881.983 ms, 2.88 s total +[ 2023-09-21 06:12:44 ] Completed train batch 0 generator forward 26.457 ms, 2.91 s total +[ 2023-09-21 06:12:44 ] Completed train batch 0 generator loss: 1.837 66.258 ms, 2.97 s total +[ 2023-09-21 06:12:45 ] Completed train batch 0 generator backward 774.953 ms, 3.75 s total +[ 2023-09-21 06:12:45 ] Completed train batch 0 discriminator loss 0.158 59.696 ms, 3.81 s total +[ 2023-09-21 06:12:45 ] Completed train batch 0 discriminator backward 108.043 ms, 3.92 s total +Epoch [14] Step [0/2], gen_loss: 0.607, disc_loss: 0.054 +[ 2023-09-21 06:12:45 ] Completed train batch 0 metrics update 19.270 ms, 3.94 s total +Saving checkpoint at epoch 14 train batch 0 +[ 2023-09-21 06:12:46 ] Completed saving temp checkpoint 414.018 ms, 4.35 s total +[ 2023-09-21 06:12:46 ] Completed replacing temp checkpoint with checkpoint 26.998 ms, 4.38 s total +[ 2023-09-21 06:12:47 ] Completed train batch 1 to device 1,238.225 ms, 5.62 s total +[ 2023-09-21 06:12:47 ] Completed train batch 1 generator forward 40.425 ms, 5.66 s total +[ 2023-09-21 06:12:47 ] Completed train batch 1 generator loss: 1.742 63.544 ms, 5.72 s total +[ 2023-09-21 06:12:49 ] Completed train batch 1 generator backward 1,616.803 ms, 7.34 s total +[ 2023-09-21 06:12:49 ] Completed train batch 1 discriminator loss 0.163 55.897 ms, 7.39 s total +[ 2023-09-21 06:12:49 ] Completed train batch 1 discriminator backward 75.497 ms, 7.47 s total +Epoch [14] Step [1/2], gen_loss: 0.598, disc_loss: 0.054 +[ 2023-09-21 06:12:49 ] Completed train batch 1 metrics update 2.217 ms, 7.47 s total +Saving checkpoint at epoch 14 train batch 1 +[ 2023-09-21 06:12:49 ] Completed saving temp checkpoint 503.225 ms, 7.97 s total +[ 2023-09-21 06:12:49 ] Completed replacing temp checkpoint with checkpoint 24.537 ms, 8.00 s total +Epoch [14] :: gen_loss: 0.602, disc_loss: 0.054 +[ 2023-09-21 06:12:49 ] Completed training generator for epoch 14 49.569 ms, 8.05 s total +[ 2023-09-21 06:12:49 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 14 from eval step 0 + +[ 2023-09-21 06:12:51 ] Completed eval batch 0 to device 1,105.344 ms, 1.11 s total +[ 2023-09-21 06:12:51 ] Completed eval batch 0 forward 41.653 ms, 1.15 s total +[ 2023-09-21 06:12:51 ] Completed eval batch 0 recons_loss 0.293 ms, 1.15 s total +[ 2023-09-21 06:12:51 ] Completed eval batch 0 metrics update 0.564 ms, 1.15 s total +Saving checkpoint at epoch 14 val batch 0 +[ 2023-09-21 06:12:51 ] Completed saving temp checkpoint 425.916 ms, 1.57 s total +[ 2023-09-21 06:12:51 ] Completed replacing temp checkpoint with checkpoint 28.912 ms, 1.60 s total +[ 2023-09-21 06:12:51 ] Completed eval batch 1 to device 383.356 ms, 1.99 s total +[ 2023-09-21 06:12:51 ] Completed eval batch 1 forward 38.152 ms, 2.02 s total +[ 2023-09-21 06:12:51 ] Completed eval batch 1 recons_loss 0.332 ms, 2.02 s total +[ 2023-09-21 06:12:52 ] Completed eval batch 1 metrics update 163.543 ms, 2.19 s total +Saving checkpoint at epoch 14 val batch 1 +Epoch 14 val loss: 0.2282 +[ 2023-09-21 06:12:52 ] Completed saving temp checkpoint 433.918 ms, 2.62 s total +[ 2023-09-21 06:12:52 ] Completed replacing temp checkpoint with checkpoint 20.315 ms, 2.64 s total +[ 2023-09-21 06:12:52 ] Completed evaluating generator for epoch 14 55.441 ms, 2.70 s total + + +EPOCH :: 15 + + +[ 2023-09-21 06:12:52 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 15 from training step 0 + +[ 2023-09-21 06:12:55 ] Completed train batch 0 to device 2,934.298 ms, 2.93 s total +[ 2023-09-21 06:12:55 ] Completed train batch 0 generator forward 29.177 ms, 2.96 s total +[ 2023-09-21 06:12:55 ] Completed train batch 0 generator loss: 2.037 66.560 ms, 3.03 s total +[ 2023-09-21 06:12:56 ] Completed train batch 0 generator backward 798.036 ms, 3.83 s total +[ 2023-09-21 06:12:56 ] Completed train batch 0 discriminator loss 0.150 57.090 ms, 3.89 s total +[ 2023-09-21 06:12:56 ] Completed train batch 0 discriminator backward 98.159 ms, 3.98 s total +Epoch [15] Step [0/2], gen_loss: 0.676, disc_loss: 0.051 +[ 2023-09-21 06:12:56 ] Completed train batch 0 metrics update 3.074 ms, 3.99 s total +Saving checkpoint at epoch 15 train batch 0 +[ 2023-09-21 06:12:56 ] Completed saving temp checkpoint 395.024 ms, 4.38 s total +[ 2023-09-21 06:12:57 ] Completed replacing temp checkpoint with checkpoint 24.384 ms, 4.41 s total +[ 2023-09-21 06:12:58 ] Completed train batch 1 to device 1,150.190 ms, 5.56 s total +[ 2023-09-21 06:12:58 ] Completed train batch 1 generator forward 40.896 ms, 5.60 s total +[ 2023-09-21 06:12:58 ] Completed train batch 1 generator loss: 1.981 62.045 ms, 5.66 s total +[ 2023-09-21 06:13:00 ] Completed train batch 1 generator backward 1,869.957 ms, 7.53 s total +[ 2023-09-21 06:13:00 ] Completed train batch 1 discriminator loss 0.134 56.303 ms, 7.59 s total +[ 2023-09-21 06:13:00 ] Completed train batch 1 discriminator backward 71.053 ms, 7.66 s total +Epoch [15] Step [1/2], gen_loss: 0.676, disc_loss: 0.045 +[ 2023-09-21 06:13:00 ] Completed train batch 1 metrics update 5.735 ms, 7.66 s total +Saving checkpoint at epoch 15 train batch 1 +[ 2023-09-21 06:13:00 ] Completed saving temp checkpoint 455.769 ms, 8.12 s total +[ 2023-09-21 06:13:00 ] Completed replacing temp checkpoint with checkpoint 23.246 ms, 8.14 s total +Epoch [15] :: gen_loss: 0.676, disc_loss: 0.048 +[ 2023-09-21 06:13:00 ] Completed training generator for epoch 15 57.371 ms, 8.20 s total +[ 2023-09-21 06:13:00 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 15 from eval step 0 + +[ 2023-09-21 06:13:01 ] Completed eval batch 0 to device 1,133.617 ms, 1.13 s total +[ 2023-09-21 06:13:01 ] Completed eval batch 0 forward 43.453 ms, 1.18 s total +[ 2023-09-21 06:13:01 ] Completed eval batch 0 recons_loss 0.327 ms, 1.18 s total +[ 2023-09-21 06:13:01 ] Completed eval batch 0 metrics update 0.612 ms, 1.18 s total +Saving checkpoint at epoch 15 val batch 0 +[ 2023-09-21 06:13:02 ] Completed saving temp checkpoint 432.473 ms, 1.61 s total +[ 2023-09-21 06:13:02 ] Completed replacing temp checkpoint with checkpoint 35.602 ms, 1.65 s total +[ 2023-09-21 06:13:02 ] Completed eval batch 1 to device 397.637 ms, 2.04 s total +[ 2023-09-21 06:13:02 ] Completed eval batch 1 forward 39.920 ms, 2.08 s total +[ 2023-09-21 06:13:02 ] Completed eval batch 1 recons_loss 0.565 ms, 2.08 s total +[ 2023-09-21 06:13:03 ] Completed eval batch 1 metrics update 152.794 ms, 2.24 s total +Saving checkpoint at epoch 15 val batch 1 +Epoch 15 val loss: 0.2527 +[ 2023-09-21 06:13:03 ] Completed saving temp checkpoint 629.026 ms, 2.87 s total +[ 2023-09-21 06:13:03 ] Completed replacing temp checkpoint with checkpoint 39.706 ms, 2.91 s total +[ 2023-09-21 06:13:03 ] Completed evaluating generator for epoch 15 57.582 ms, 2.96 s total + + +EPOCH :: 16 + + +[ 2023-09-21 06:13:03 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 16 from training step 0 + +[ 2023-09-21 06:13:06 ] Completed train batch 0 to device 3,045.157 ms, 3.05 s total +[ 2023-09-21 06:13:06 ] Completed train batch 0 generator forward 25.900 ms, 3.07 s total +[ 2023-09-21 06:13:06 ] Completed train batch 0 generator loss: 2.156 66.812 ms, 3.14 s total +[ 2023-09-21 06:13:07 ] Completed train batch 0 generator backward 507.921 ms, 3.65 s total +[ 2023-09-21 06:13:07 ] Completed train batch 0 discriminator loss 0.123 57.292 ms, 3.70 s total +[ 2023-09-21 06:13:07 ] Completed train batch 0 discriminator backward 104.515 ms, 3.81 s total +Epoch [16] Step [0/2], gen_loss: 0.729, disc_loss: 0.043 +[ 2023-09-21 06:13:07 ] Completed train batch 0 metrics update 5.730 ms, 3.81 s total +Saving checkpoint at epoch 16 train batch 0 +[ 2023-09-21 06:13:08 ] Completed saving temp checkpoint 414.706 ms, 4.23 s total +[ 2023-09-21 06:13:08 ] Completed replacing temp checkpoint with checkpoint 22.189 ms, 4.25 s total +[ 2023-09-21 06:13:09 ] Completed train batch 1 to device 1,712.595 ms, 5.96 s total +[ 2023-09-21 06:13:09 ] Completed train batch 1 generator forward 22.413 ms, 5.99 s total +[ 2023-09-21 06:13:09 ] Completed train batch 1 generator loss: 2.354 63.482 ms, 6.05 s total +[ 2023-09-21 06:13:11 ] Completed train batch 1 generator backward 1,195.438 ms, 7.24 s total +[ 2023-09-21 06:13:11 ] Completed train batch 1 discriminator loss 0.112 57.134 ms, 7.30 s total +[ 2023-09-21 06:13:11 ] Completed train batch 1 discriminator backward 73.378 ms, 7.37 s total +Epoch [16] Step [1/2], gen_loss: 0.765, disc_loss: 0.038 +[ 2023-09-21 06:13:11 ] Completed train batch 1 metrics update 1.040 ms, 7.38 s total +Saving checkpoint at epoch 16 train batch 1 +[ 2023-09-21 06:13:11 ] Completed saving temp checkpoint 436.627 ms, 7.81 s total +[ 2023-09-21 06:13:11 ] Completed replacing temp checkpoint with checkpoint 18.214 ms, 7.83 s total +Epoch [16] :: gen_loss: 0.747, disc_loss: 0.040 +[ 2023-09-21 06:13:11 ] Completed training generator for epoch 16 54.639 ms, 7.89 s total +[ 2023-09-21 06:13:11 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 16 from eval step 0 + +[ 2023-09-21 06:13:12 ] Completed eval batch 0 to device 1,164.517 ms, 1.16 s total +[ 2023-09-21 06:13:12 ] Completed eval batch 0 forward 26.908 ms, 1.19 s total +[ 2023-09-21 06:13:12 ] Completed eval batch 0 recons_loss 0.378 ms, 1.19 s total +[ 2023-09-21 06:13:12 ] Completed eval batch 0 metrics update 0.625 ms, 1.19 s total +Saving checkpoint at epoch 16 val batch 0 +[ 2023-09-21 06:13:13 ] Completed saving temp checkpoint 391.904 ms, 1.58 s total +[ 2023-09-21 06:13:13 ] Completed replacing temp checkpoint with checkpoint 26.002 ms, 1.61 s total +[ 2023-09-21 06:13:13 ] Completed eval batch 1 to device 408.181 ms, 2.02 s total +[ 2023-09-21 06:13:13 ] Completed eval batch 1 forward 21.611 ms, 2.04 s total +[ 2023-09-21 06:13:13 ] Completed eval batch 1 recons_loss 0.369 ms, 2.04 s total +[ 2023-09-21 06:13:13 ] Completed eval batch 1 metrics update 171.785 ms, 2.21 s total +Saving checkpoint at epoch 16 val batch 1 +Epoch 16 val loss: 0.3175 +[ 2023-09-21 06:13:14 ] Completed saving temp checkpoint 446.802 ms, 2.66 s total +[ 2023-09-21 06:13:14 ] Completed replacing temp checkpoint with checkpoint 26.578 ms, 2.69 s total +[ 2023-09-21 06:13:14 ] Completed evaluating generator for epoch 16 55.591 ms, 2.74 s total + + +EPOCH :: 17 + + +[ 2023-09-21 06:13:14 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 17 from training step 0 + +[ 2023-09-21 06:13:17 ] Completed train batch 0 to device 3,089.605 ms, 3.09 s total +[ 2023-09-21 06:13:17 ] Completed train batch 0 generator forward 31.533 ms, 3.12 s total +[ 2023-09-21 06:13:17 ] Completed train batch 0 generator loss: 2.296 66.255 ms, 3.19 s total +[ 2023-09-21 06:13:18 ] Completed train batch 0 generator backward 627.823 ms, 3.82 s total +[ 2023-09-21 06:13:18 ] Completed train batch 0 discriminator loss 0.108 56.206 ms, 3.87 s total +[ 2023-09-21 06:13:18 ] Completed train batch 0 discriminator backward 115.541 ms, 3.99 s total +Epoch [17] Step [0/2], gen_loss: 0.776, disc_loss: 0.036 +[ 2023-09-21 06:13:18 ] Completed train batch 0 metrics update 6.672 ms, 3.99 s total +Saving checkpoint at epoch 17 train batch 0 +[ 2023-09-21 06:13:18 ] Completed saving temp checkpoint 394.360 ms, 4.39 s total +[ 2023-09-21 06:13:18 ] Completed replacing temp checkpoint with checkpoint 31.702 ms, 4.42 s total +[ 2023-09-21 06:13:20 ] Completed train batch 1 to device 1,497.831 ms, 5.92 s total +[ 2023-09-21 06:13:20 ] Completed train batch 1 generator forward 22.256 ms, 5.94 s total +[ 2023-09-21 06:13:20 ] Completed train batch 1 generator loss: 2.291 63.235 ms, 6.00 s total +[ 2023-09-21 06:13:21 ] Completed train batch 1 generator backward 1,321.630 ms, 7.32 s total +[ 2023-09-21 06:13:21 ] Completed train batch 1 discriminator loss 0.102 56.259 ms, 7.38 s total +[ 2023-09-21 06:13:21 ] Completed train batch 1 discriminator backward 74.014 ms, 7.45 s total +Epoch [17] Step [1/2], gen_loss: 0.768, disc_loss: 0.035 +[ 2023-09-21 06:13:21 ] Completed train batch 1 metrics update 5.670 ms, 7.46 s total +Saving checkpoint at epoch 17 train batch 1 +[ 2023-09-21 06:13:22 ] Completed saving temp checkpoint 464.422 ms, 7.93 s total +[ 2023-09-21 06:13:22 ] Completed replacing temp checkpoint with checkpoint 22.264 ms, 7.95 s total +Epoch [17] :: gen_loss: 0.772, disc_loss: 0.035 +[ 2023-09-21 06:13:22 ] Completed training generator for epoch 17 52.477 ms, 8.00 s total +[ 2023-09-21 06:13:22 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 17 from eval step 0 + +[ 2023-09-21 06:13:23 ] Completed eval batch 0 to device 1,060.799 ms, 1.06 s total +[ 2023-09-21 06:13:23 ] Completed eval batch 0 forward 25.276 ms, 1.09 s total +[ 2023-09-21 06:13:23 ] Completed eval batch 0 recons_loss 0.319 ms, 1.09 s total +[ 2023-09-21 06:13:23 ] Completed eval batch 0 metrics update 0.548 ms, 1.09 s total +Saving checkpoint at epoch 17 val batch 0 +[ 2023-09-21 06:13:23 ] Completed saving temp checkpoint 367.484 ms, 1.45 s total +[ 2023-09-21 06:13:23 ] Completed replacing temp checkpoint with checkpoint 29.368 ms, 1.48 s total +[ 2023-09-21 06:13:24 ] Completed eval batch 1 to device 439.374 ms, 1.92 s total +[ 2023-09-21 06:13:24 ] Completed eval batch 1 forward 38.815 ms, 1.96 s total +[ 2023-09-21 06:13:24 ] Completed eval batch 1 recons_loss 0.527 ms, 1.96 s total +[ 2023-09-21 06:13:24 ] Completed eval batch 1 metrics update 274.837 ms, 2.24 s total +Saving checkpoint at epoch 17 val batch 1 +Epoch 17 val loss: 0.2978 +[ 2023-09-21 06:13:25 ] Completed saving temp checkpoint 489.065 ms, 2.73 s total +[ 2023-09-21 06:13:25 ] Completed replacing temp checkpoint with checkpoint 27.289 ms, 2.75 s total +[ 2023-09-21 06:13:25 ] Completed evaluating generator for epoch 17 53.463 ms, 2.81 s total + + +EPOCH :: 18 + + +[ 2023-09-21 06:13:25 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 18 from training step 0 + +[ 2023-09-21 06:13:28 ] Completed train batch 0 to device 3,044.956 ms, 3.04 s total +[ 2023-09-21 06:13:28 ] Completed train batch 0 generator forward 30.691 ms, 3.08 s total +[ 2023-09-21 06:13:28 ] Completed train batch 0 generator loss: 2.224 66.440 ms, 3.14 s total +[ 2023-09-21 06:13:28 ] Completed train batch 0 generator backward 606.919 ms, 3.75 s total +[ 2023-09-21 06:13:29 ] Completed train batch 0 discriminator loss 0.112 57.587 ms, 3.81 s total +[ 2023-09-21 06:13:29 ] Completed train batch 0 discriminator backward 107.442 ms, 3.91 s total +Epoch [18] Step [0/2], gen_loss: 0.738, disc_loss: 0.038 +[ 2023-09-21 06:13:29 ] Completed train batch 0 metrics update 4.891 ms, 3.92 s total +Saving checkpoint at epoch 18 train batch 0 +[ 2023-09-21 06:13:29 ] Completed saving temp checkpoint 417.542 ms, 4.34 s total +[ 2023-09-21 06:13:29 ] Completed replacing temp checkpoint with checkpoint 23.134 ms, 4.36 s total +[ 2023-09-21 06:13:31 ] Completed train batch 1 to device 1,530.888 ms, 5.89 s total +[ 2023-09-21 06:13:31 ] Completed train batch 1 generator forward 42.228 ms, 5.93 s total +[ 2023-09-21 06:13:31 ] Completed train batch 1 generator loss: 2.190 63.661 ms, 6.00 s total +[ 2023-09-21 06:13:32 ] Completed train batch 1 generator backward 1,037.285 ms, 7.03 s total +[ 2023-09-21 06:13:32 ] Completed train batch 1 discriminator loss 0.131 61.955 ms, 7.10 s total +[ 2023-09-21 06:13:32 ] Completed train batch 1 discriminator backward 57.661 ms, 7.15 s total +Epoch [18] Step [1/2], gen_loss: 0.721, disc_loss: 0.042 +[ 2023-09-21 06:13:32 ] Completed train batch 1 metrics update 5.421 ms, 7.16 s total +Saving checkpoint at epoch 18 train batch 1 +[ 2023-09-21 06:13:32 ] Completed saving temp checkpoint 456.546 ms, 7.62 s total +[ 2023-09-21 06:13:32 ] Completed replacing temp checkpoint with checkpoint 23.140 ms, 7.64 s total +Epoch [18] :: gen_loss: 0.729, disc_loss: 0.040 +[ 2023-09-21 06:13:32 ] Completed training generator for epoch 18 63.905 ms, 7.70 s total +[ 2023-09-21 06:13:32 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 18 from eval step 0 + +[ 2023-09-21 06:13:33 ] Completed eval batch 0 to device 1,078.770 ms, 1.08 s total +[ 2023-09-21 06:13:34 ] Completed eval batch 0 forward 24.651 ms, 1.10 s total +[ 2023-09-21 06:13:34 ] Completed eval batch 0 recons_loss 0.311 ms, 1.10 s total +[ 2023-09-21 06:13:34 ] Completed eval batch 0 metrics update 0.547 ms, 1.10 s total +Saving checkpoint at epoch 18 val batch 0 +[ 2023-09-21 06:13:34 ] Completed saving temp checkpoint 421.878 ms, 1.53 s total +[ 2023-09-21 06:13:34 ] Completed replacing temp checkpoint with checkpoint 27.372 ms, 1.55 s total +[ 2023-09-21 06:13:34 ] Completed eval batch 1 to device 364.133 ms, 1.92 s total +[ 2023-09-21 06:13:34 ] Completed eval batch 1 forward 39.126 ms, 1.96 s total +[ 2023-09-21 06:13:34 ] Completed eval batch 1 recons_loss 0.329 ms, 1.96 s total +[ 2023-09-21 06:13:35 ] Completed eval batch 1 metrics update 263.351 ms, 2.22 s total +Saving checkpoint at epoch 18 val batch 1 +Epoch 18 val loss: 0.2379 +[ 2023-09-21 06:13:35 ] Completed saving temp checkpoint 420.844 ms, 2.64 s total +[ 2023-09-21 06:13:35 ] Completed replacing temp checkpoint with checkpoint 22.810 ms, 2.66 s total +[ 2023-09-21 06:13:35 ] Completed evaluating generator for epoch 18 54.664 ms, 2.72 s total + + +EPOCH :: 19 + + +[ 2023-09-21 06:13:35 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 19 from training step 0 + +[ 2023-09-21 06:13:38 ] Completed train batch 0 to device 3,042.992 ms, 3.04 s total +[ 2023-09-21 06:13:38 ] Completed train batch 0 generator forward 33.362 ms, 3.08 s total +[ 2023-09-21 06:13:38 ] Completed train batch 0 generator loss: 2.083 76.342 ms, 3.15 s total +[ 2023-09-21 06:13:39 ] Completed train batch 0 generator backward 636.184 ms, 3.79 s total +[ 2023-09-21 06:13:39 ] Completed train batch 0 discriminator loss 0.151 57.366 ms, 3.85 s total +[ 2023-09-21 06:13:39 ] Completed train batch 0 discriminator backward 118.933 ms, 3.97 s total +Epoch [19] Step [0/2], gen_loss: 0.696, disc_loss: 0.050 +[ 2023-09-21 06:13:39 ] Completed train batch 0 metrics update 5.173 ms, 3.97 s total +Saving checkpoint at epoch 19 train batch 0 +[ 2023-09-21 06:13:40 ] Completed saving temp checkpoint 405.401 ms, 4.38 s total +[ 2023-09-21 06:13:40 ] Completed replacing temp checkpoint with checkpoint 22.087 ms, 4.40 s total +[ 2023-09-21 06:13:41 ] Completed train batch 1 to device 1,430.871 ms, 5.83 s total +[ 2023-09-21 06:13:41 ] Completed train batch 1 generator forward 40.708 ms, 5.87 s total +[ 2023-09-21 06:13:41 ] Completed train batch 1 generator loss: 1.992 64.195 ms, 5.93 s total +[ 2023-09-21 06:13:42 ] Completed train batch 1 generator backward 1,245.921 ms, 7.18 s total +[ 2023-09-21 06:13:42 ] Completed train batch 1 discriminator loss 0.153 56.157 ms, 7.24 s total +[ 2023-09-21 06:13:42 ] Completed train batch 1 discriminator backward 75.378 ms, 7.31 s total +Epoch [19] Step [1/2], gen_loss: 0.666, disc_loss: 0.052 +[ 2023-09-21 06:13:42 ] Completed train batch 1 metrics update 0.859 ms, 7.31 s total +Saving checkpoint at epoch 19 train batch 1 +[ 2023-09-21 06:13:43 ] Completed saving temp checkpoint 409.781 ms, 7.72 s total +[ 2023-09-21 06:13:43 ] Completed replacing temp checkpoint with checkpoint 26.296 ms, 7.75 s total +Epoch [19] :: gen_loss: 0.681, disc_loss: 0.051 +[ 2023-09-21 06:13:43 ] Completed training generator for epoch 19 53.171 ms, 7.80 s total +[ 2023-09-21 06:13:43 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 19 from eval step 0 + +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 06:26:21 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 06:26:21 ] Completed importing Timer 0.024 ms, 0.00 s total +[ 2023-09-21 06:26:26 ] Completed importing everything else 4,907.589 ms, 4.91 s total +| distributed init (rank 5): env:// +| distributed init (rank 3): env:// +| distributed init (rank 4): env:// +| distributed init (rank 1): env:// +| distributed init (rank 0): env:// +| distributed init (rank 2): env:// +[ 2023-09-21 06:26:34 ] Completed preliminaries 7,723.475 ms, 12.63 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 06:26:34 ] Completed build datasets 10.425 ms, 12.64 s total +[ 2023-09-21 06:26:34 ] Completed build samplers 0.052 ms, 12.64 s total +[ 2023-09-21 06:26:34 ] Completed build dataloaders 0.139 ms, 12.64 s total +[ 2023-09-21 06:26:34 ] Completed generator to device 76.185 ms, 12.72 s total +[ 2023-09-21 06:26:34 ] Completed discriminator to device 7.759 ms, 12.73 s total +[ 2023-09-21 06:26:34 ] Completed loss functions 455.803 ms, 13.18 s total +[ 2023-09-21 06:26:34 ] Completed models prepped for distribution 83.918 ms, 13.27 s total +[ 2023-09-21 06:26:34 ] Completed optimizers 0.748 ms, 13.27 s total +[ 2023-09-21 06:26:34 ] Completed grad scalers 0.035 ms, 13.27 s total +[ 2023-09-21 06:26:35 ] Completed checkpoint retrieval 528.489 ms, 13.79 s total + + +EPOCH :: 19 + + +[ 2023-09-21 06:26:35 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 19 from training step 2 + +Epoch [19] :: gen_loss: 0.681, disc_loss: 0.051 +[ 2023-09-21 06:26:35 ] Completed training generator for epoch 19 138.221 ms, 0.14 s total +[ 2023-09-21 06:26:35 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 19 from eval step 0 + +[ 2023-09-21 06:26:37 ] Completed eval batch 0 to device 1,645.299 ms, 1.65 s total +[ 2023-09-21 06:26:39 ] Completed eval batch 0 forward 1,734.666 ms, 3.38 s total +[ 2023-09-21 06:26:39 ] Completed eval batch 0 recons_loss 0.349 ms, 3.38 s total +[ 2023-09-21 06:26:39 ] Completed eval batch 0 metrics update 305.799 ms, 3.69 s total +Saving checkpoint at epoch 19 val batch 0 +[ 2023-09-21 06:26:39 ] Completed saving temp checkpoint 421.883 ms, 4.11 s total +[ 2023-09-21 06:26:39 ] Completed replacing temp checkpoint with checkpoint 124.821 ms, 4.23 s total +[ 2023-09-21 06:26:39 ] Completed eval batch 1 to device 11.775 ms, 4.24 s total +[ 2023-09-21 06:26:39 ] Completed eval batch 1 forward 21.206 ms, 4.27 s total +[ 2023-09-21 06:26:39 ] Completed eval batch 1 recons_loss 0.342 ms, 4.27 s total +[ 2023-09-21 06:26:39 ] Completed eval batch 1 metrics update 0.307 ms, 4.27 s total +Saving checkpoint at epoch 19 val batch 1 +Epoch 19 val loss: 0.1705 +[ 2023-09-21 06:26:40 ] Completed saving temp checkpoint 433.925 ms, 4.70 s total +[ 2023-09-21 06:26:40 ] Completed replacing temp checkpoint with checkpoint 18.297 ms, 4.72 s total +[ 2023-09-21 06:26:40 ] Completed evaluating generator for epoch 19 40.113 ms, 4.76 s total + + +EPOCH :: 20 + + +[ 2023-09-21 06:26:40 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 20 from training step 0 + +[ 2023-09-21 06:26:44 ] Completed train batch 0 to device 3,606.015 ms, 3.61 s total +[ 2023-09-21 06:26:44 ] Completed train batch 0 generator forward 48.470 ms, 3.65 s total +[ 2023-09-21 06:26:44 ] Completed train batch 0 generator loss: 1.809 86.878 ms, 3.74 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 06:26:44 ] Completed train batch 0 generator backward 333.596 ms, 4.07 s total +[ 2023-09-21 06:26:44 ] Completed train batch 0 discriminator loss 0.165 57.774 ms, 4.13 s total +[ 2023-09-21 06:26:44 ] Completed train batch 0 discriminator backward 109.061 ms, 4.24 s total +Epoch [20] Step [0/2], gen_loss: 0.611, disc_loss: 0.055 +[ 2023-09-21 06:26:44 ] Completed train batch 0 metrics update 4.774 ms, 4.25 s total +Saving checkpoint at epoch 20 train batch 0 +[ 2023-09-21 06:26:45 ] Completed saving temp checkpoint 421.583 ms, 4.67 s total +[ 2023-09-21 06:26:45 ] Completed replacing temp checkpoint with checkpoint 17.248 ms, 4.69 s total +[ 2023-09-21 06:26:47 ] Completed train batch 1 to device 2,276.392 ms, 6.96 s total +[ 2023-09-21 06:26:47 ] Completed train batch 1 generator forward 22.244 ms, 6.98 s total +[ 2023-09-21 06:26:47 ] Completed train batch 1 generator loss: 1.595 63.413 ms, 7.05 s total +[ 2023-09-21 06:26:47 ] Completed train batch 1 generator backward 554.514 ms, 7.60 s total +[ 2023-09-21 06:26:48 ] Completed train batch 1 discriminator loss 0.166 56.843 ms, 7.66 s total +[ 2023-09-21 06:26:48 ] Completed train batch 1 discriminator backward 70.543 ms, 7.73 s total +Epoch [20] Step [1/2], gen_loss: 0.548, disc_loss: 0.055 +[ 2023-09-21 06:26:48 ] Completed train batch 1 metrics update 2.029 ms, 7.73 s total +Saving checkpoint at epoch 20 train batch 1 +[ 2023-09-21 06:26:48 ] Completed saving temp checkpoint 471.295 ms, 8.20 s total +[ 2023-09-21 06:26:48 ] Completed replacing temp checkpoint with checkpoint 21.383 ms, 8.22 s total +Epoch [20] :: gen_loss: 0.579, disc_loss: 0.055 +[ 2023-09-21 06:26:48 ] Completed training generator for epoch 20 48.211 ms, 8.27 s total +[ 2023-09-21 06:26:48 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 20 from eval step 0 + +[ 2023-09-21 06:26:49 ] Completed eval batch 0 to device 1,173.850 ms, 1.17 s total +[ 2023-09-21 06:26:49 ] Completed eval batch 0 forward 31.760 ms, 1.21 s total +[ 2023-09-21 06:26:49 ] Completed eval batch 0 recons_loss 0.385 ms, 1.21 s total +[ 2023-09-21 06:26:49 ] Completed eval batch 0 metrics update 0.681 ms, 1.21 s total +Saving checkpoint at epoch 20 val batch 0 +[ 2023-09-21 06:26:50 ] Completed saving temp checkpoint 395.185 ms, 1.60 s total +[ 2023-09-21 06:26:50 ] Completed replacing temp checkpoint with checkpoint 21.265 ms, 1.62 s total +[ 2023-09-21 06:26:50 ] Completed eval batch 1 to device 482.913 ms, 2.11 s total +[ 2023-09-21 06:26:50 ] Completed eval batch 1 forward 22.572 ms, 2.13 s total +[ 2023-09-21 06:26:50 ] Completed eval batch 1 recons_loss 0.346 ms, 2.13 s total +[ 2023-09-21 06:26:50 ] Completed eval batch 1 metrics update 172.256 ms, 2.30 s total +Saving checkpoint at epoch 20 val batch 1 +Epoch 20 val loss: 0.1752 +[ 2023-09-21 06:26:51 ] Completed saving temp checkpoint 416.413 ms, 2.72 s total +[ 2023-09-21 06:26:51 ] Completed replacing temp checkpoint with checkpoint 18.386 ms, 2.74 s total +[ 2023-09-21 06:26:51 ] Completed evaluating generator for epoch 20 47.010 ms, 2.78 s total + + +EPOCH :: 21 + + +[ 2023-09-21 06:26:51 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 21 from training step 0 + +[ 2023-09-21 06:26:54 ] Completed train batch 0 to device 3,303.875 ms, 3.30 s total +[ 2023-09-21 06:26:54 ] Completed train batch 0 generator forward 43.259 ms, 3.35 s total +[ 2023-09-21 06:26:54 ] Completed train batch 0 generator loss: 1.710 66.738 ms, 3.41 s total +[ 2023-09-21 06:26:55 ] Completed train batch 0 generator backward 595.003 ms, 4.01 s total +[ 2023-09-21 06:26:55 ] Completed train batch 0 discriminator loss 0.158 57.620 ms, 4.07 s total +[ 2023-09-21 06:26:55 ] Completed train batch 0 discriminator backward 88.025 ms, 4.15 s total +Epoch [21] Step [0/2], gen_loss: 0.575, disc_loss: 0.054 +[ 2023-09-21 06:26:55 ] Completed train batch 0 metrics update 4.955 ms, 4.16 s total +Saving checkpoint at epoch 21 train batch 0 +[ 2023-09-21 06:26:56 ] Completed saving temp checkpoint 547.404 ms, 4.71 s total +[ 2023-09-21 06:26:56 ] Completed replacing temp checkpoint with checkpoint 42.150 ms, 4.75 s total +[ 2023-09-21 06:26:57 ] Completed train batch 1 to device 1,760.675 ms, 6.51 s total +[ 2023-09-21 06:26:57 ] Completed train batch 1 generator forward 38.238 ms, 6.55 s total +[ 2023-09-21 06:26:58 ] Completed train batch 1 generator loss: 1.687 63.445 ms, 6.61 s total +[ 2023-09-21 06:26:59 ] Completed train batch 1 generator backward 1,065.147 ms, 7.68 s total +[ 2023-09-21 06:26:59 ] Completed train batch 1 discriminator loss 0.157 56.694 ms, 7.73 s total +[ 2023-09-21 06:26:59 ] Completed train batch 1 discriminator backward 69.972 ms, 7.80 s total +Epoch [21] Step [1/2], gen_loss: 0.544, disc_loss: 0.051 +[ 2023-09-21 06:26:59 ] Completed train batch 1 metrics update 1.997 ms, 7.81 s total +Saving checkpoint at epoch 21 train batch 1 +[ 2023-09-21 06:26:59 ] Completed saving temp checkpoint 714.721 ms, 8.52 s total +[ 2023-09-21 06:27:00 ] Completed replacing temp checkpoint with checkpoint 31.463 ms, 8.55 s total +Epoch [21] :: gen_loss: 0.560, disc_loss: 0.052 +[ 2023-09-21 06:27:00 ] Completed training generator for epoch 21 59.078 ms, 8.61 s total +[ 2023-09-21 06:27:00 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 21 from eval step 0 + +[ 2023-09-21 06:27:01 ] Completed eval batch 0 to device 1,159.042 ms, 1.16 s total +[ 2023-09-21 06:27:01 ] Completed eval batch 0 forward 41.340 ms, 1.20 s total +[ 2023-09-21 06:27:01 ] Completed eval batch 0 recons_loss 0.227 ms, 1.20 s total +[ 2023-09-21 06:27:01 ] Completed eval batch 0 metrics update 0.537 ms, 1.20 s total +Saving checkpoint at epoch 21 val batch 0 +[ 2023-09-21 06:27:01 ] Completed saving temp checkpoint 418.159 ms, 1.62 s total +[ 2023-09-21 06:27:01 ] Completed replacing temp checkpoint with checkpoint 25.561 ms, 1.64 s total +[ 2023-09-21 06:27:02 ] Completed eval batch 1 to device 395.809 ms, 2.04 s total +[ 2023-09-21 06:27:02 ] Completed eval batch 1 forward 27.756 ms, 2.07 s total +[ 2023-09-21 06:27:02 ] Completed eval batch 1 recons_loss 0.353 ms, 2.07 s total +[ 2023-09-21 06:27:02 ] Completed eval batch 1 metrics update 0.319 ms, 2.07 s total +Saving checkpoint at epoch 21 val batch 1 +Epoch 21 val loss: 0.1330 +[ 2023-09-21 06:27:02 ] Completed saving temp checkpoint 473.613 ms, 2.54 s total +[ 2023-09-21 06:27:02 ] Completed replacing temp checkpoint with checkpoint 25.098 ms, 2.57 s total +[ 2023-09-21 06:27:02 ] Completed evaluating generator for epoch 21 56.063 ms, 2.62 s total + + +EPOCH :: 22 + + +[ 2023-09-21 06:27:02 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 22 from training step 0 + +[ 2023-09-21 06:27:05 ] Completed train batch 0 to device 3,079.879 ms, 3.08 s total +[ 2023-09-21 06:27:05 ] Completed train batch 0 generator forward 30.760 ms, 3.11 s total +[ 2023-09-21 06:27:05 ] Completed train batch 0 generator loss: 1.689 66.481 ms, 3.18 s total +[ 2023-09-21 06:27:06 ] Completed train batch 0 generator backward 693.006 ms, 3.87 s total +[ 2023-09-21 06:27:06 ] Completed train batch 0 discriminator loss 0.149 57.735 ms, 3.93 s total +[ 2023-09-21 06:27:06 ] Completed train batch 0 discriminator backward 110.747 ms, 4.04 s total +Epoch [22] Step [0/2], gen_loss: 0.559, disc_loss: 0.049 +[ 2023-09-21 06:27:06 ] Completed train batch 0 metrics update 6.854 ms, 4.05 s total +Saving checkpoint at epoch 22 train batch 0 +[ 2023-09-21 06:27:07 ] Completed saving temp checkpoint 452.375 ms, 4.50 s total +[ 2023-09-21 06:27:07 ] Completed replacing temp checkpoint with checkpoint 24.200 ms, 4.52 s total +[ 2023-09-21 06:27:08 ] Completed train batch 1 to device 1,447.933 ms, 5.97 s total +[ 2023-09-21 06:27:08 ] Completed train batch 1 generator forward 40.817 ms, 6.01 s total +[ 2023-09-21 06:27:08 ] Completed train batch 1 generator loss: 1.550 63.452 ms, 6.07 s total +[ 2023-09-21 06:27:09 ] Completed train batch 1 generator backward 1,222.880 ms, 7.30 s total +[ 2023-09-21 06:27:10 ] Completed train batch 1 discriminator loss 0.135 56.834 ms, 7.35 s total +[ 2023-09-21 06:27:10 ] Completed train batch 1 discriminator backward 89.915 ms, 7.44 s total +Epoch [22] Step [1/2], gen_loss: 0.545, disc_loss: 0.047 +[ 2023-09-21 06:27:10 ] Completed train batch 1 metrics update 5.352 ms, 7.45 s total +Saving checkpoint at epoch 22 train batch 1 +[ 2023-09-21 06:27:10 ] Completed saving temp checkpoint 493.356 ms, 7.94 s total +[ 2023-09-21 06:27:10 ] Completed replacing temp checkpoint with checkpoint 18.979 ms, 7.96 s total +Epoch [22] :: gen_loss: 0.552, disc_loss: 0.048 +[ 2023-09-21 06:27:10 ] Completed training generator for epoch 22 56.861 ms, 8.02 s total +[ 2023-09-21 06:27:10 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 22 from eval step 0 + +[ 2023-09-21 06:27:11 ] Completed eval batch 0 to device 1,162.579 ms, 1.16 s total +[ 2023-09-21 06:27:11 ] Completed eval batch 0 forward 25.488 ms, 1.19 s total +[ 2023-09-21 06:27:11 ] Completed eval batch 0 recons_loss 0.334 ms, 1.19 s total +[ 2023-09-21 06:27:11 ] Completed eval batch 0 metrics update 1.050 ms, 1.19 s total +Saving checkpoint at epoch 22 val batch 0 +[ 2023-09-21 06:27:12 ] Completed saving temp checkpoint 445.171 ms, 1.63 s total +[ 2023-09-21 06:27:12 ] Completed replacing temp checkpoint with checkpoint 28.860 ms, 1.66 s total +[ 2023-09-21 06:27:12 ] Completed eval batch 1 to device 426.425 ms, 2.09 s total +[ 2023-09-21 06:27:12 ] Completed eval batch 1 forward 40.212 ms, 2.13 s total +[ 2023-09-21 06:27:12 ] Completed eval batch 1 recons_loss 0.595 ms, 2.13 s total +[ 2023-09-21 06:27:12 ] Completed eval batch 1 metrics update 103.060 ms, 2.23 s total +Saving checkpoint at epoch 22 val batch 1 +Epoch 22 val loss: 0.1237 +[ 2023-09-21 06:27:13 ] Completed saving temp checkpoint 468.910 ms, 2.70 s total +[ 2023-09-21 06:27:13 ] Completed replacing temp checkpoint with checkpoint 22.051 ms, 2.72 s total +[ 2023-09-21 06:27:13 ] Completed evaluating generator for epoch 22 54.260 ms, 2.78 s total + + +EPOCH :: 23 + + +[ 2023-09-21 06:27:13 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 23 from training step 0 + +[ 2023-09-21 06:27:16 ] Completed train batch 0 to device 3,157.251 ms, 3.16 s total +[ 2023-09-21 06:27:16 ] Completed train batch 0 generator forward 27.059 ms, 3.18 s total +[ 2023-09-21 06:27:16 ] Completed train batch 0 generator loss: 1.640 67.300 ms, 3.25 s total +[ 2023-09-21 06:27:17 ] Completed train batch 0 generator backward 651.430 ms, 3.90 s total +[ 2023-09-21 06:27:17 ] Completed train batch 0 discriminator loss 0.143 59.277 ms, 3.96 s total +[ 2023-09-21 06:27:17 ] Completed train batch 0 discriminator backward 95.777 ms, 4.06 s total +Epoch [23] Step [0/2], gen_loss: 0.543, disc_loss: 0.047 +[ 2023-09-21 06:27:17 ] Completed train batch 0 metrics update 5.464 ms, 4.06 s total +Saving checkpoint at epoch 23 train batch 0 +[ 2023-09-21 06:27:18 ] Completed saving temp checkpoint 477.886 ms, 4.54 s total +[ 2023-09-21 06:27:18 ] Completed replacing temp checkpoint with checkpoint 24.215 ms, 4.57 s total +[ 2023-09-21 06:27:19 ] Completed train batch 1 to device 1,536.806 ms, 6.10 s total +[ 2023-09-21 06:27:19 ] Completed train batch 1 generator forward 22.219 ms, 6.12 s total +[ 2023-09-21 06:27:19 ] Completed train batch 1 generator loss: 1.557 64.394 ms, 6.19 s total +[ 2023-09-21 06:27:21 ] Completed train batch 1 generator backward 1,388.185 ms, 7.58 s total +[ 2023-09-21 06:27:21 ] Completed train batch 1 discriminator loss 0.143 56.936 ms, 7.63 s total +[ 2023-09-21 06:27:21 ] Completed train batch 1 discriminator backward 84.745 ms, 7.72 s total +Epoch [23] Step [1/2], gen_loss: 0.526, disc_loss: 0.046 +[ 2023-09-21 06:27:21 ] Completed train batch 1 metrics update 0.900 ms, 7.72 s total +Saving checkpoint at epoch 23 train batch 1 +[ 2023-09-21 06:27:21 ] Completed saving temp checkpoint 474.567 ms, 8.19 s total +[ 2023-09-21 06:27:21 ] Completed replacing temp checkpoint with checkpoint 27.882 ms, 8.22 s total +Epoch [23] :: gen_loss: 0.535, disc_loss: 0.047 +[ 2023-09-21 06:27:21 ] Completed training generator for epoch 23 52.338 ms, 8.27 s total +[ 2023-09-21 06:27:21 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 23 from eval step 0 + +[ 2023-09-21 06:27:22 ] Completed eval batch 0 to device 1,174.747 ms, 1.17 s total +[ 2023-09-21 06:27:22 ] Completed eval batch 0 forward 42.867 ms, 1.22 s total +[ 2023-09-21 06:27:22 ] Completed eval batch 0 recons_loss 0.346 ms, 1.22 s total +[ 2023-09-21 06:27:22 ] Completed eval batch 0 metrics update 0.860 ms, 1.22 s total +Saving checkpoint at epoch 23 val batch 0 +[ 2023-09-21 06:27:23 ] Completed saving temp checkpoint 417.071 ms, 1.64 s total +[ 2023-09-21 06:27:23 ] Completed replacing temp checkpoint with checkpoint 22.505 ms, 1.66 s total +[ 2023-09-21 06:27:23 ] Completed eval batch 1 to device 436.314 ms, 2.09 s total +[ 2023-09-21 06:27:23 ] Completed eval batch 1 forward 37.607 ms, 2.13 s total +[ 2023-09-21 06:27:23 ] Completed eval batch 1 recons_loss 0.338 ms, 2.13 s total +[ 2023-09-21 06:27:24 ] Completed eval batch 1 metrics update 179.521 ms, 2.31 s total +Saving checkpoint at epoch 23 val batch 1 +Epoch 23 val loss: 0.1415 +[ 2023-09-21 06:27:24 ] Completed saving temp checkpoint 435.266 ms, 2.75 s total +[ 2023-09-21 06:27:24 ] Completed replacing temp checkpoint with checkpoint 32.591 ms, 2.78 s total +[ 2023-09-21 06:27:24 ] Completed evaluating generator for epoch 23 51.149 ms, 2.83 s total + + +EPOCH :: 24 + + +[ 2023-09-21 06:27:24 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 24 from training step 0 + +[ 2023-09-21 06:27:27 ] Completed train batch 0 to device 3,104.204 ms, 3.10 s total +[ 2023-09-21 06:27:27 ] Completed train batch 0 generator forward 26.786 ms, 3.13 s total +[ 2023-09-21 06:27:27 ] Completed train batch 0 generator loss: 1.764 66.464 ms, 3.20 s total +[ 2023-09-21 06:27:28 ] Completed train batch 0 generator backward 760.081 ms, 3.96 s total +[ 2023-09-21 06:27:28 ] Completed train batch 0 discriminator loss 0.134 57.371 ms, 4.01 s total +[ 2023-09-21 06:27:28 ] Completed train batch 0 discriminator backward 107.273 ms, 4.12 s total +Epoch [24] Step [0/2], gen_loss: 0.601, disc_loss: 0.044 +[ 2023-09-21 06:27:28 ] Completed train batch 0 metrics update 7.452 ms, 4.13 s total +Saving checkpoint at epoch 24 train batch 0 +[ 2023-09-21 06:27:29 ] Completed saving temp checkpoint 521.662 ms, 4.65 s total +[ 2023-09-21 06:27:29 ] Completed replacing temp checkpoint with checkpoint 34.244 ms, 4.69 s total +[ 2023-09-21 06:27:30 ] Completed train batch 1 to device 1,274.558 ms, 5.96 s total +[ 2023-09-21 06:27:30 ] Completed train batch 1 generator forward 42.187 ms, 6.00 s total +[ 2023-09-21 06:27:30 ] Completed train batch 1 generator loss: 1.679 65.342 ms, 6.07 s total +[ 2023-09-21 06:27:32 ] Completed train batch 1 generator backward 1,778.728 ms, 7.85 s total +[ 2023-09-21 06:27:32 ] Completed train batch 1 discriminator loss 0.127 57.495 ms, 7.90 s total +[ 2023-09-21 06:27:32 ] Completed train batch 1 discriminator backward 73.192 ms, 7.98 s total +Epoch [24] Step [1/2], gen_loss: 0.578, disc_loss: 0.043 +[ 2023-09-21 06:27:32 ] Completed train batch 1 metrics update 0.788 ms, 7.98 s total +Saving checkpoint at epoch 24 train batch 1 +[ 2023-09-21 06:27:33 ] Completed saving temp checkpoint 568.349 ms, 8.55 s total +[ 2023-09-21 06:27:33 ] Completed replacing temp checkpoint with checkpoint 33.552 ms, 8.58 s total +Epoch [24] :: gen_loss: 0.589, disc_loss: 0.044 +[ 2023-09-21 06:27:33 ] Completed training generator for epoch 24 72.822 ms, 8.65 s total +[ 2023-09-21 06:27:33 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 24 from eval step 0 + +[ 2023-09-21 06:27:34 ] Completed eval batch 0 to device 1,170.881 ms, 1.17 s total +[ 2023-09-21 06:27:34 ] Completed eval batch 0 forward 25.051 ms, 1.20 s total +[ 2023-09-21 06:27:34 ] Completed eval batch 0 recons_loss 0.329 ms, 1.20 s total +[ 2023-09-21 06:27:34 ] Completed eval batch 0 metrics update 0.640 ms, 1.20 s total +Saving checkpoint at epoch 24 val batch 0 +[ 2023-09-21 06:27:35 ] Completed saving temp checkpoint 844.709 ms, 2.04 s total +[ 2023-09-21 06:27:35 ] Completed replacing temp checkpoint with checkpoint 54.410 ms, 2.10 s total +[ 2023-09-21 06:27:35 ] Completed eval batch 1 to device 36.711 ms, 2.13 s total +[ 2023-09-21 06:27:35 ] Completed eval batch 1 forward 36.725 ms, 2.17 s total +[ 2023-09-21 06:27:35 ] Completed eval batch 1 recons_loss 0.341 ms, 2.17 s total +[ 2023-09-21 06:27:35 ] Completed eval batch 1 metrics update 0.301 ms, 2.17 s total +Saving checkpoint at epoch 24 val batch 1 +Epoch 24 val loss: 0.1357 +[ 2023-09-21 06:27:36 ] Completed saving temp checkpoint 752.492 ms, 2.92 s total +[ 2023-09-21 06:27:36 ] Completed replacing temp checkpoint with checkpoint 38.980 ms, 2.96 s total +[ 2023-09-21 06:27:36 ] Completed evaluating generator for epoch 24 59.398 ms, 3.02 s total + + +EPOCH :: 25 + + +[ 2023-09-21 06:27:36 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 25 from training step 0 + +[ 2023-09-21 06:27:39 ] Completed train batch 0 to device 3,152.129 ms, 3.15 s total +[ 2023-09-21 06:27:39 ] Completed train batch 0 generator forward 26.215 ms, 3.18 s total +[ 2023-09-21 06:27:39 ] Completed train batch 0 generator loss: 1.660 65.716 ms, 3.24 s total +[ 2023-09-21 06:27:39 ] Completed train batch 0 generator backward 315.891 ms, 3.56 s total +[ 2023-09-21 06:27:39 ] Completed train batch 0 discriminator loss 0.133 63.613 ms, 3.62 s total +[ 2023-09-21 06:27:39 ] Completed train batch 0 discriminator backward 83.277 ms, 3.71 s total +Epoch [25] Step [0/2], gen_loss: 0.544, disc_loss: 0.045 +[ 2023-09-21 06:27:39 ] Completed train batch 0 metrics update 3.741 ms, 3.71 s total +Saving checkpoint at epoch 25 train batch 0 +[ 2023-09-21 06:27:40 ] Completed saving temp checkpoint 447.755 ms, 4.16 s total +[ 2023-09-21 06:27:40 ] Completed replacing temp checkpoint with checkpoint 20.648 ms, 4.18 s total +[ 2023-09-21 06:27:42 ] Completed train batch 1 to device 1,977.280 ms, 6.16 s total +[ 2023-09-21 06:27:42 ] Completed train batch 1 generator forward 41.714 ms, 6.20 s total +[ 2023-09-21 06:27:42 ] Completed train batch 1 generator loss: 1.537 63.780 ms, 6.26 s total +[ 2023-09-21 06:27:43 ] Completed train batch 1 generator backward 666.693 ms, 6.93 s total +[ 2023-09-21 06:27:43 ] Completed train batch 1 discriminator loss 0.130 61.063 ms, 6.99 s total +[ 2023-09-21 06:27:43 ] Completed train batch 1 discriminator backward 61.070 ms, 7.05 s total +Epoch [25] Step [1/2], gen_loss: 0.535, disc_loss: 0.042 +[ 2023-09-21 06:27:43 ] Completed train batch 1 metrics update 0.637 ms, 7.05 s total +Saving checkpoint at epoch 25 train batch 1 +[ 2023-09-21 06:27:43 ] Completed saving temp checkpoint 489.122 ms, 7.54 s total +[ 2023-09-21 06:27:43 ] Completed replacing temp checkpoint with checkpoint 30.998 ms, 7.57 s total +Epoch [25] :: gen_loss: 0.539, disc_loss: 0.044 +[ 2023-09-21 06:27:43 ] Completed training generator for epoch 25 55.724 ms, 7.63 s total +[ 2023-09-21 06:27:43 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 25 from eval step 0 + +[ 2023-09-21 06:27:45 ] Completed eval batch 0 to device 1,179.917 ms, 1.18 s total +[ 2023-09-21 06:27:45 ] Completed eval batch 0 forward 31.153 ms, 1.21 s total +[ 2023-09-21 06:27:45 ] Completed eval batch 0 recons_loss 0.323 ms, 1.21 s total +[ 2023-09-21 06:27:45 ] Completed eval batch 0 metrics update 0.630 ms, 1.21 s total +Saving checkpoint at epoch 25 val batch 0 +[ 2023-09-21 06:27:45 ] Completed saving temp checkpoint 417.929 ms, 1.63 s total +[ 2023-09-21 06:27:45 ] Completed replacing temp checkpoint with checkpoint 32.709 ms, 1.66 s total +[ 2023-09-21 06:27:45 ] Completed eval batch 1 to device 434.132 ms, 2.10 s total +[ 2023-09-21 06:27:46 ] Completed eval batch 1 forward 39.890 ms, 2.14 s total +[ 2023-09-21 06:27:46 ] Completed eval batch 1 recons_loss 0.566 ms, 2.14 s total +[ 2023-09-21 06:27:46 ] Completed eval batch 1 metrics update 139.134 ms, 2.28 s total +Saving checkpoint at epoch 25 val batch 1 +Epoch 25 val loss: 0.1180 +[ 2023-09-21 06:27:46 ] Completed saving temp checkpoint 469.231 ms, 2.75 s total +[ 2023-09-21 06:27:46 ] Completed replacing temp checkpoint with checkpoint 23.951 ms, 2.77 s total +[ 2023-09-21 06:27:46 ] Completed evaluating generator for epoch 25 48.793 ms, 2.82 s total + + +EPOCH :: 26 + + +[ 2023-09-21 06:27:46 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 26 from training step 0 + +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 06:40:25 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 06:40:25 ] Completed importing Timer 0.028 ms, 0.00 s total +[ 2023-09-21 06:40:30 ] Completed importing everything else 4,714.302 ms, 4.71 s total +| distributed init (rank 4): env:// +| distributed init (rank 3): env:// +| distributed init (rank 1): env:// +| distributed init (rank 5): env:// +| distributed init (rank 0): env:// +| distributed init (rank 2): env:// +[ 2023-09-21 06:40:38 ] Completed preliminaries 7,634.946 ms, 12.35 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 06:40:38 ] Completed build datasets 11.098 ms, 12.36 s total +[ 2023-09-21 06:40:38 ] Completed build samplers 0.044 ms, 12.36 s total +[ 2023-09-21 06:40:38 ] Completed build dataloaders 0.134 ms, 12.36 s total +[ 2023-09-21 06:40:38 ] Completed generator to device 75.857 ms, 12.44 s total +[ 2023-09-21 06:40:38 ] Completed discriminator to device 6.927 ms, 12.44 s total +[ 2023-09-21 06:40:38 ] Completed loss functions 397.873 ms, 12.84 s total +[ 2023-09-21 06:40:38 ] Completed models prepped for distribution 134.753 ms, 12.98 s total +[ 2023-09-21 06:40:38 ] Completed optimizers 0.684 ms, 12.98 s total +[ 2023-09-21 06:40:38 ] Completed grad scalers 0.023 ms, 12.98 s total +[ 2023-09-21 06:40:39 ] Completed checkpoint retrieval 486.976 ms, 13.46 s total + + +EPOCH :: 25 + + +[ 2023-09-21 06:40:39 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 25 from training step 2 + +Epoch [25] :: gen_loss: 0.539, disc_loss: 0.044 +[ 2023-09-21 06:40:39 ] Completed training generator for epoch 25 135.213 ms, 0.14 s total +[ 2023-09-21 06:40:39 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 25 from eval step 2 + +[ 2023-09-21 06:40:39 ] Completed evaluating generator for epoch 25 128.298 ms, 0.13 s total + + +EPOCH :: 26 + + +[ 2023-09-21 06:40:39 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 26 from training step 0 + +[ 2023-09-21 06:40:42 ] Completed train batch 0 to device 3,463.167 ms, 3.46 s total +[ 2023-09-21 06:40:44 ] Completed train batch 0 generator forward 1,270.963 ms, 4.73 s total +[ 2023-09-21 06:40:44 ] Completed train batch 0 generator loss: 1.542 75.762 ms, 4.81 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 06:40:45 ] Completed train batch 0 generator backward 1,491.183 ms, 6.30 s total +[ 2023-09-21 06:40:45 ] Completed train batch 0 discriminator loss 0.131 56.011 ms, 6.36 s total +[ 2023-09-21 06:40:45 ] Completed train batch 0 discriminator backward 75.967 ms, 6.43 s total +Epoch [26] Step [0/2], gen_loss: 0.541, disc_loss: 0.042 +[ 2023-09-21 06:40:45 ] Completed train batch 0 metrics update 3.219 ms, 6.44 s total +Saving checkpoint at epoch 26 train batch 0 +[ 2023-09-21 06:40:46 ] Completed saving temp checkpoint 461.288 ms, 6.90 s total +[ 2023-09-21 06:40:46 ] Completed replacing temp checkpoint with checkpoint 119.659 ms, 7.02 s total +[ 2023-09-21 06:40:46 ] Completed train batch 1 to device 12.138 ms, 7.03 s total +[ 2023-09-21 06:40:46 ] Completed train batch 1 generator forward 22.221 ms, 7.05 s total +[ 2023-09-21 06:40:46 ] Completed train batch 1 generator loss: 1.540 61.811 ms, 7.11 s total +[ 2023-09-21 06:40:47 ] Completed train batch 1 generator backward 1,181.699 ms, 8.30 s total +[ 2023-09-21 06:40:47 ] Completed train batch 1 discriminator loss 0.125 56.271 ms, 8.35 s total +[ 2023-09-21 06:40:47 ] Completed train batch 1 discriminator backward 73.134 ms, 8.42 s total +Epoch [26] Step [1/2], gen_loss: 0.535, disc_loss: 0.042 +[ 2023-09-21 06:40:47 ] Completed train batch 1 metrics update 5.067 ms, 8.43 s total +Saving checkpoint at epoch 26 train batch 1 +[ 2023-09-21 06:40:48 ] Completed saving temp checkpoint 504.210 ms, 8.93 s total +[ 2023-09-21 06:40:48 ] Completed replacing temp checkpoint with checkpoint 26.556 ms, 8.96 s total +Epoch [26] :: gen_loss: 0.538, disc_loss: 0.042 +[ 2023-09-21 06:40:48 ] Completed training generator for epoch 26 46.396 ms, 9.01 s total +[ 2023-09-21 06:40:48 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 26 from eval step 0 + +[ 2023-09-21 06:40:49 ] Completed eval batch 0 to device 1,086.993 ms, 1.09 s total +[ 2023-09-21 06:40:49 ] Completed eval batch 0 forward 28.685 ms, 1.12 s total +[ 2023-09-21 06:40:49 ] Completed eval batch 0 recons_loss 0.299 ms, 1.12 s total +[ 2023-09-21 06:40:49 ] Completed eval batch 0 metrics update 0.561 ms, 1.12 s total +Saving checkpoint at epoch 26 val batch 0 +[ 2023-09-21 06:40:49 ] Completed saving temp checkpoint 379.013 ms, 1.50 s total +[ 2023-09-21 06:40:50 ] Completed replacing temp checkpoint with checkpoint 28.409 ms, 1.52 s total +[ 2023-09-21 06:40:50 ] Completed eval batch 1 to device 383.723 ms, 1.91 s total +[ 2023-09-21 06:40:50 ] Completed eval batch 1 forward 21.168 ms, 1.93 s total +[ 2023-09-21 06:40:50 ] Completed eval batch 1 recons_loss 0.324 ms, 1.93 s total +[ 2023-09-21 06:40:50 ] Completed eval batch 1 metrics update 219.131 ms, 2.15 s total +Saving checkpoint at epoch 26 val batch 1 +Epoch 26 val loss: 0.1119 +[ 2023-09-21 06:40:51 ] Completed saving temp checkpoint 401.723 ms, 2.55 s total +[ 2023-09-21 06:40:51 ] Completed replacing temp checkpoint with checkpoint 34.178 ms, 2.58 s total +[ 2023-09-21 06:40:51 ] Completed evaluating generator for epoch 26 54.337 ms, 2.64 s total + + +EPOCH :: 27 + + +[ 2023-09-21 06:40:51 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 27 from training step 0 + +[ 2023-09-21 06:40:53 ] Completed train batch 0 to device 2,863.214 ms, 2.86 s total +[ 2023-09-21 06:40:54 ] Completed train batch 0 generator forward 43.911 ms, 2.91 s total +[ 2023-09-21 06:40:54 ] Completed train batch 0 generator loss: 1.629 66.258 ms, 2.97 s total +[ 2023-09-21 06:40:55 ] Completed train batch 0 generator backward 985.470 ms, 3.96 s total +[ 2023-09-21 06:40:55 ] Completed train batch 0 discriminator loss 0.127 57.036 ms, 4.02 s total +[ 2023-09-21 06:40:55 ] Completed train batch 0 discriminator backward 99.711 ms, 4.12 s total +Epoch [27] Step [0/2], gen_loss: 0.539, disc_loss: 0.042 +[ 2023-09-21 06:40:55 ] Completed train batch 0 metrics update 6.477 ms, 4.12 s total +Saving checkpoint at epoch 27 train batch 0 +[ 2023-09-21 06:40:55 ] Completed saving temp checkpoint 482.229 ms, 4.60 s total +[ 2023-09-21 06:40:55 ] Completed replacing temp checkpoint with checkpoint 31.013 ms, 4.64 s total +[ 2023-09-21 06:40:56 ] Completed train batch 1 to device 795.688 ms, 5.43 s total +[ 2023-09-21 06:40:56 ] Completed train batch 1 generator forward 38.278 ms, 5.47 s total +[ 2023-09-21 06:40:56 ] Completed train batch 1 generator loss: 1.583 61.993 ms, 5.53 s total +[ 2023-09-21 06:40:58 ] Completed train batch 1 generator backward 1,953.289 ms, 7.48 s total +[ 2023-09-21 06:40:58 ] Completed train batch 1 discriminator loss 0.127 56.421 ms, 7.54 s total +[ 2023-09-21 06:40:58 ] Completed train batch 1 discriminator backward 77.820 ms, 7.62 s total +Epoch [27] Step [1/2], gen_loss: 0.531, disc_loss: 0.042 +[ 2023-09-21 06:40:58 ] Completed train batch 1 metrics update 1.841 ms, 7.62 s total +Saving checkpoint at epoch 27 train batch 1 +[ 2023-09-21 06:40:59 ] Completed saving temp checkpoint 756.298 ms, 8.38 s total +[ 2023-09-21 06:40:59 ] Completed replacing temp checkpoint with checkpoint 31.456 ms, 8.41 s total +Epoch [27] :: gen_loss: 0.535, disc_loss: 0.042 +[ 2023-09-21 06:40:59 ] Completed training generator for epoch 27 65.165 ms, 8.47 s total +[ 2023-09-21 06:40:59 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 27 from eval step 0 + +[ 2023-09-21 06:41:00 ] Completed eval batch 0 to device 1,138.580 ms, 1.14 s total +[ 2023-09-21 06:41:00 ] Completed eval batch 0 forward 33.419 ms, 1.17 s total +[ 2023-09-21 06:41:00 ] Completed eval batch 0 recons_loss 0.376 ms, 1.17 s total +[ 2023-09-21 06:41:00 ] Completed eval batch 0 metrics update 0.603 ms, 1.17 s total +Saving checkpoint at epoch 27 val batch 0 +[ 2023-09-21 06:41:01 ] Completed saving temp checkpoint 727.666 ms, 1.90 s total +[ 2023-09-21 06:41:01 ] Completed replacing temp checkpoint with checkpoint 27.104 ms, 1.93 s total +[ 2023-09-21 06:41:01 ] Completed eval batch 1 to device 15.856 ms, 1.94 s total +[ 2023-09-21 06:41:01 ] Completed eval batch 1 forward 21.678 ms, 1.97 s total +[ 2023-09-21 06:41:01 ] Completed eval batch 1 recons_loss 0.342 ms, 1.97 s total +[ 2023-09-21 06:41:01 ] Completed eval batch 1 metrics update 0.314 ms, 1.97 s total +Saving checkpoint at epoch 27 val batch 1 +Epoch 27 val loss: 0.1120 +[ 2023-09-21 06:41:02 ] Completed saving temp checkpoint 662.863 ms, 2.63 s total +[ 2023-09-21 06:41:02 ] Completed replacing temp checkpoint with checkpoint 39.801 ms, 2.67 s total +[ 2023-09-21 06:41:02 ] Completed evaluating generator for epoch 27 63.333 ms, 2.73 s total + + +EPOCH :: 28 + + +[ 2023-09-21 06:41:02 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 28 from training step 0 + +[ 2023-09-21 06:41:05 ] Completed train batch 0 to device 3,030.454 ms, 3.03 s total +[ 2023-09-21 06:41:05 ] Completed train batch 0 generator forward 41.913 ms, 3.07 s total +[ 2023-09-21 06:41:05 ] Completed train batch 0 generator loss: 1.668 66.251 ms, 3.14 s total +[ 2023-09-21 06:41:05 ] Completed train batch 0 generator backward 417.154 ms, 3.56 s total +[ 2023-09-21 06:41:05 ] Completed train batch 0 discriminator loss 0.132 56.211 ms, 3.61 s total +[ 2023-09-21 06:41:06 ] Completed train batch 0 discriminator backward 108.202 ms, 3.72 s total +Epoch [28] Step [0/2], gen_loss: 0.535, disc_loss: 0.043 +[ 2023-09-21 06:41:06 ] Completed train batch 0 metrics update 22.793 ms, 3.74 s total +Saving checkpoint at epoch 28 train batch 0 +[ 2023-09-21 06:41:06 ] Completed saving temp checkpoint 579.262 ms, 4.32 s total +[ 2023-09-21 06:41:06 ] Completed replacing temp checkpoint with checkpoint 22.971 ms, 4.35 s total +[ 2023-09-21 06:41:08 ] Completed train batch 1 to device 1,433.779 ms, 5.78 s total +[ 2023-09-21 06:41:08 ] Completed train batch 1 generator forward 22.257 ms, 5.80 s total +[ 2023-09-21 06:41:08 ] Completed train batch 1 generator loss: 1.533 64.441 ms, 5.87 s total +[ 2023-09-21 06:41:09 ] Completed train batch 1 generator backward 1,220.952 ms, 7.09 s total +[ 2023-09-21 06:41:09 ] Completed train batch 1 discriminator loss 0.123 56.110 ms, 7.14 s total +[ 2023-09-21 06:41:09 ] Completed train batch 1 discriminator backward 90.878 ms, 7.23 s total +Epoch [28] Step [1/2], gen_loss: 0.529, disc_loss: 0.042 +[ 2023-09-21 06:41:09 ] Completed train batch 1 metrics update 4.099 ms, 7.24 s total +Saving checkpoint at epoch 28 train batch 1 +[ 2023-09-21 06:41:10 ] Completed saving temp checkpoint 598.540 ms, 7.84 s total +[ 2023-09-21 06:41:10 ] Completed replacing temp checkpoint with checkpoint 38.369 ms, 7.87 s total +Epoch [28] :: gen_loss: 0.532, disc_loss: 0.042 +[ 2023-09-21 06:41:10 ] Completed training generator for epoch 28 65.242 ms, 7.94 s total +[ 2023-09-21 06:41:10 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 28 from eval step 0 + +[ 2023-09-21 06:41:11 ] Completed eval batch 0 to device 1,118.283 ms, 1.12 s total +[ 2023-09-21 06:41:11 ] Completed eval batch 0 forward 24.888 ms, 1.14 s total +[ 2023-09-21 06:41:11 ] Completed eval batch 0 recons_loss 0.313 ms, 1.14 s total +[ 2023-09-21 06:41:11 ] Completed eval batch 0 metrics update 0.642 ms, 1.14 s total +Saving checkpoint at epoch 28 val batch 0 +[ 2023-09-21 06:41:11 ] Completed saving temp checkpoint 519.305 ms, 1.66 s total +[ 2023-09-21 06:41:11 ] Completed replacing temp checkpoint with checkpoint 33.808 ms, 1.70 s total +[ 2023-09-21 06:41:12 ] Completed eval batch 1 to device 214.526 ms, 1.91 s total +[ 2023-09-21 06:41:12 ] Completed eval batch 1 forward 39.705 ms, 1.95 s total +[ 2023-09-21 06:41:12 ] Completed eval batch 1 recons_loss 0.580 ms, 1.95 s total +[ 2023-09-21 06:41:12 ] Completed eval batch 1 metrics update 87.338 ms, 2.04 s total +Saving checkpoint at epoch 28 val batch 1 +Epoch 28 val loss: 0.1114 +[ 2023-09-21 06:41:12 ] Completed saving temp checkpoint 580.004 ms, 2.62 s total +[ 2023-09-21 06:41:12 ] Completed replacing temp checkpoint with checkpoint 41.935 ms, 2.66 s total +[ 2023-09-21 06:41:12 ] Completed evaluating generator for epoch 28 59.099 ms, 2.72 s total + + +EPOCH :: 29 + + +[ 2023-09-21 06:41:12 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 29 from training step 0 + +[ 2023-09-21 06:41:15 ] Completed train batch 0 to device 2,776.842 ms, 2.78 s total +[ 2023-09-21 06:41:15 ] Completed train batch 0 generator forward 26.394 ms, 2.80 s total +[ 2023-09-21 06:41:15 ] Completed train batch 0 generator loss: 1.624 65.819 ms, 2.87 s total +[ 2023-09-21 06:41:16 ] Completed train batch 0 generator backward 857.939 ms, 3.73 s total +[ 2023-09-21 06:41:16 ] Completed train batch 0 discriminator loss 0.127 56.752 ms, 3.78 s total +[ 2023-09-21 06:41:16 ] Completed train batch 0 discriminator backward 113.671 ms, 3.90 s total +Epoch [29] Step [0/2], gen_loss: 0.533, disc_loss: 0.042 +[ 2023-09-21 06:41:16 ] Completed train batch 0 metrics update 5.636 ms, 3.90 s total +Saving checkpoint at epoch 29 train batch 0 +[ 2023-09-21 06:41:17 ] Completed saving temp checkpoint 577.115 ms, 4.48 s total +[ 2023-09-21 06:41:17 ] Completed replacing temp checkpoint with checkpoint 26.013 ms, 4.51 s total +[ 2023-09-21 06:41:18 ] Completed train batch 1 to device 971.227 ms, 5.48 s total +[ 2023-09-21 06:41:18 ] Completed train batch 1 generator forward 40.971 ms, 5.52 s total +[ 2023-09-21 06:41:18 ] Completed train batch 1 generator loss: 1.530 62.983 ms, 5.58 s total +[ 2023-09-21 06:41:20 ] Completed train batch 1 generator backward 1,541.150 ms, 7.12 s total +[ 2023-09-21 06:41:20 ] Completed train batch 1 discriminator loss 0.122 56.195 ms, 7.18 s total +[ 2023-09-21 06:41:20 ] Completed train batch 1 discriminator backward 90.724 ms, 7.27 s total +Epoch [29] Step [1/2], gen_loss: 0.525, disc_loss: 0.041 +[ 2023-09-21 06:41:20 ] Completed train batch 1 metrics update 5.623 ms, 7.28 s total +Saving checkpoint at epoch 29 train batch 1 +[ 2023-09-21 06:41:20 ] Completed saving temp checkpoint 567.993 ms, 7.84 s total +[ 2023-09-21 06:41:20 ] Completed replacing temp checkpoint with checkpoint 25.062 ms, 7.87 s total +Epoch [29] :: gen_loss: 0.529, disc_loss: 0.042 +[ 2023-09-21 06:41:20 ] Completed training generator for epoch 29 68.624 ms, 7.94 s total +[ 2023-09-21 06:41:20 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 29 from eval step 0 + +[ 2023-09-21 06:41:21 ] Completed eval batch 0 to device 990.118 ms, 0.99 s total +[ 2023-09-21 06:41:21 ] Completed eval batch 0 forward 25.301 ms, 1.02 s total +[ 2023-09-21 06:41:21 ] Completed eval batch 0 recons_loss 0.301 ms, 1.02 s total +[ 2023-09-21 06:41:21 ] Completed eval batch 0 metrics update 0.601 ms, 1.02 s total +Saving checkpoint at epoch 29 val batch 0 +[ 2023-09-21 06:41:22 ] Completed saving temp checkpoint 393.356 ms, 1.41 s total +[ 2023-09-21 06:41:22 ] Completed replacing temp checkpoint with checkpoint 28.632 ms, 1.44 s total +[ 2023-09-21 06:41:22 ] Completed eval batch 1 to device 360.564 ms, 1.80 s total +[ 2023-09-21 06:41:22 ] Completed eval batch 1 forward 21.284 ms, 1.82 s total +[ 2023-09-21 06:41:22 ] Completed eval batch 1 recons_loss 0.331 ms, 1.82 s total +[ 2023-09-21 06:41:23 ] Completed eval batch 1 metrics update 314.671 ms, 2.14 s total +Saving checkpoint at epoch 29 val batch 1 +Epoch 29 val loss: 0.1018 +[ 2023-09-21 06:41:23 ] Completed saving temp checkpoint 462.972 ms, 2.60 s total +[ 2023-09-21 06:41:23 ] Completed replacing temp checkpoint with checkpoint 22.868 ms, 2.62 s total +[ 2023-09-21 06:41:23 ] Completed evaluating generator for epoch 29 60.910 ms, 2.68 s total + + +EPOCH :: 30 + + +[ 2023-09-21 06:41:23 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 30 from training step 0 + +[ 2023-09-21 06:41:26 ] Completed train batch 0 to device 2,772.230 ms, 2.77 s total +[ 2023-09-21 06:41:26 ] Completed train batch 0 generator forward 25.450 ms, 2.80 s total +[ 2023-09-21 06:41:26 ] Completed train batch 0 generator loss: 1.653 64.730 ms, 2.86 s total +[ 2023-09-21 06:41:27 ] Completed train batch 0 generator backward 1,041.165 ms, 3.90 s total +[ 2023-09-21 06:41:27 ] Completed train batch 0 discriminator loss 0.124 56.405 ms, 3.96 s total +[ 2023-09-21 06:41:27 ] Completed train batch 0 discriminator backward 107.133 ms, 4.07 s total +Epoch [30] Step [0/2], gen_loss: 0.531, disc_loss: 0.041 +[ 2023-09-21 06:41:27 ] Completed train batch 0 metrics update 8.183 ms, 4.08 s total +Saving checkpoint at epoch 30 train batch 0 +[ 2023-09-21 06:41:28 ] Completed saving temp checkpoint 429.459 ms, 4.50 s total +[ 2023-09-21 06:41:28 ] Completed replacing temp checkpoint with checkpoint 32.292 ms, 4.54 s total +[ 2023-09-21 06:41:28 ] Completed train batch 1 to device 795.923 ms, 5.33 s total +[ 2023-09-21 06:41:28 ] Completed train batch 1 generator forward 40.771 ms, 5.37 s total +[ 2023-09-21 06:41:29 ] Completed train batch 1 generator loss: 1.475 61.856 ms, 5.44 s total +[ 2023-09-21 06:41:31 ] Completed train batch 1 generator backward 2,009.270 ms, 7.44 s total +[ 2023-09-21 06:41:31 ] Completed train batch 1 discriminator loss 0.129 55.948 ms, 7.50 s total +[ 2023-09-21 06:41:31 ] Completed train batch 1 discriminator backward 78.903 ms, 7.58 s total +Epoch [30] Step [1/2], gen_loss: 0.523, disc_loss: 0.041 +[ 2023-09-21 06:41:31 ] Completed train batch 1 metrics update 0.842 ms, 7.58 s total +Saving checkpoint at epoch 30 train batch 1 +[ 2023-09-21 06:41:31 ] Completed saving temp checkpoint 438.522 ms, 8.02 s total +[ 2023-09-21 06:41:31 ] Completed replacing temp checkpoint with checkpoint 29.663 ms, 8.05 s total +Epoch [30] :: gen_loss: 0.527, disc_loss: 0.041 +[ 2023-09-21 06:41:31 ] Completed training generator for epoch 30 56.105 ms, 8.10 s total +[ 2023-09-21 06:41:31 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 30 from eval step 0 + +[ 2023-09-21 06:41:32 ] Completed eval batch 0 to device 1,032.118 ms, 1.03 s total +[ 2023-09-21 06:41:32 ] Completed eval batch 0 forward 23.882 ms, 1.06 s total +[ 2023-09-21 06:41:32 ] Completed eval batch 0 recons_loss 0.236 ms, 1.06 s total +[ 2023-09-21 06:41:32 ] Completed eval batch 0 metrics update 22.668 ms, 1.08 s total +Saving checkpoint at epoch 30 val batch 0 +[ 2023-09-21 06:41:33 ] Completed saving temp checkpoint 405.074 ms, 1.48 s total +[ 2023-09-21 06:41:33 ] Completed replacing temp checkpoint with checkpoint 24.651 ms, 1.51 s total +[ 2023-09-21 06:41:33 ] Completed eval batch 1 to device 337.995 ms, 1.85 s total +[ 2023-09-21 06:41:33 ] Completed eval batch 1 forward 38.956 ms, 1.89 s total +[ 2023-09-21 06:41:33 ] Completed eval batch 1 recons_loss 0.348 ms, 1.89 s total +[ 2023-09-21 06:41:33 ] Completed eval batch 1 metrics update 362.205 ms, 2.25 s total +Saving checkpoint at epoch 30 val batch 1 +Epoch 30 val loss: 0.0929 +[ 2023-09-21 06:41:34 ] Completed saving temp checkpoint 440.899 ms, 2.69 s total +[ 2023-09-21 06:41:34 ] Completed replacing temp checkpoint with checkpoint 23.979 ms, 2.71 s total +[ 2023-09-21 06:41:34 ] Completed evaluating generator for epoch 30 73.677 ms, 2.79 s total + + +EPOCH :: 31 + + +[ 2023-09-21 06:41:34 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 31 from training step 0 + +[ 2023-09-21 06:41:37 ] Completed train batch 0 to device 2,926.802 ms, 2.93 s total +[ 2023-09-21 06:41:37 ] Completed train batch 0 generator forward 27.311 ms, 2.95 s total +[ 2023-09-21 06:41:37 ] Completed train batch 0 generator loss: 1.543 65.840 ms, 3.02 s total +[ 2023-09-21 06:41:38 ] Completed train batch 0 generator backward 848.863 ms, 3.87 s total +[ 2023-09-21 06:41:38 ] Completed train batch 0 discriminator loss 0.121 56.520 ms, 3.93 s total +[ 2023-09-21 06:41:38 ] Completed train batch 0 discriminator backward 112.160 ms, 4.04 s total +Epoch [31] Step [0/2], gen_loss: 0.523, disc_loss: 0.043 +[ 2023-09-21 06:41:38 ] Completed train batch 0 metrics update 7.135 ms, 4.04 s total +Saving checkpoint at epoch 31 train batch 0 +[ 2023-09-21 06:41:38 ] Completed saving temp checkpoint 428.031 ms, 4.47 s total +[ 2023-09-21 06:41:38 ] Completed replacing temp checkpoint with checkpoint 24.370 ms, 4.50 s total +[ 2023-09-21 06:41:40 ] Completed train batch 1 to device 1,150.628 ms, 5.65 s total +[ 2023-09-21 06:41:40 ] Completed train batch 1 generator forward 27.753 ms, 5.68 s total +[ 2023-09-21 06:41:40 ] Completed train batch 1 generator loss: 1.469 62.045 ms, 5.74 s total +[ 2023-09-21 06:41:41 ] Completed train batch 1 generator backward 1,377.699 ms, 7.12 s total +[ 2023-09-21 06:41:41 ] Completed train batch 1 discriminator loss 0.171 57.669 ms, 7.17 s total +[ 2023-09-21 06:41:41 ] Completed train batch 1 discriminator backward 65.387 ms, 7.24 s total +Epoch [31] Step [1/2], gen_loss: 0.506, disc_loss: 0.052 +[ 2023-09-21 06:41:41 ] Completed train batch 1 metrics update 2.217 ms, 7.24 s total +Saving checkpoint at epoch 31 train batch 1 +[ 2023-09-21 06:41:42 ] Completed saving temp checkpoint 440.861 ms, 7.68 s total +[ 2023-09-21 06:41:42 ] Completed replacing temp checkpoint with checkpoint 29.608 ms, 7.71 s total +Epoch [31] :: gen_loss: 0.515, disc_loss: 0.047 +[ 2023-09-21 06:41:42 ] Completed training generator for epoch 31 61.086 ms, 7.77 s total +[ 2023-09-21 06:41:42 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 31 from eval step 0 + +[ 2023-09-21 06:41:43 ] Completed eval batch 0 to device 1,112.975 ms, 1.11 s total +[ 2023-09-21 06:41:43 ] Completed eval batch 0 forward 43.529 ms, 1.16 s total +[ 2023-09-21 06:41:43 ] Completed eval batch 0 recons_loss 0.327 ms, 1.16 s total +[ 2023-09-21 06:41:43 ] Completed eval batch 0 metrics update 0.628 ms, 1.16 s total +Saving checkpoint at epoch 31 val batch 0 +[ 2023-09-21 06:41:43 ] Completed saving temp checkpoint 400.309 ms, 1.56 s total +[ 2023-09-21 06:41:43 ] Completed replacing temp checkpoint with checkpoint 33.866 ms, 1.59 s total +[ 2023-09-21 06:41:44 ] Completed eval batch 1 to device 390.629 ms, 1.98 s total +[ 2023-09-21 06:41:44 ] Completed eval batch 1 forward 39.579 ms, 2.02 s total +[ 2023-09-21 06:41:44 ] Completed eval batch 1 recons_loss 0.551 ms, 2.02 s total +[ 2023-09-21 06:41:44 ] Completed eval batch 1 metrics update 147.382 ms, 2.17 s total +Saving checkpoint at epoch 31 val batch 1 +Epoch 31 val loss: 0.0812 +[ 2023-09-21 06:41:44 ] Completed saving temp checkpoint 424.985 ms, 2.59 s total +[ 2023-09-21 06:41:44 ] Completed replacing temp checkpoint with checkpoint 28.619 ms, 2.62 s total +[ 2023-09-21 06:41:44 ] Completed evaluating generator for epoch 31 60.088 ms, 2.68 s total + + +EPOCH :: 32 + + +[ 2023-09-21 06:41:44 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 32 from training step 0 + +[ 2023-09-21 06:41:48 ] Completed train batch 0 to device 3,047.914 ms, 3.05 s total +[ 2023-09-21 06:41:48 ] Completed train batch 0 generator forward 26.374 ms, 3.07 s total +[ 2023-09-21 06:41:48 ] Completed train batch 0 generator loss: 1.476 65.333 ms, 3.14 s total +[ 2023-09-21 06:41:48 ] Completed train batch 0 generator backward 697.477 ms, 3.84 s total +[ 2023-09-21 06:41:48 ] Completed train batch 0 discriminator loss 0.183 56.656 ms, 3.89 s total +[ 2023-09-21 06:41:48 ] Completed train batch 0 discriminator backward 88.961 ms, 3.98 s total +Epoch [32] Step [0/2], gen_loss: 0.500, disc_loss: 0.067 +[ 2023-09-21 06:41:48 ] Completed train batch 0 metrics update 5.250 ms, 3.99 s total +Saving checkpoint at epoch 32 train batch 0 +[ 2023-09-21 06:41:49 ] Completed saving temp checkpoint 419.650 ms, 4.41 s total +[ 2023-09-21 06:41:49 ] Completed replacing temp checkpoint with checkpoint 26.724 ms, 4.43 s total +[ 2023-09-21 06:41:50 ] Completed train batch 1 to device 1,309.196 ms, 5.74 s total +[ 2023-09-21 06:41:50 ] Completed train batch 1 generator forward 22.098 ms, 5.77 s total +[ 2023-09-21 06:41:50 ] Completed train batch 1 generator loss: 1.451 63.639 ms, 5.83 s total +[ 2023-09-21 06:41:51 ] Completed train batch 1 generator backward 1,161.742 ms, 6.99 s total +[ 2023-09-21 06:41:52 ] Completed train batch 1 discriminator loss 0.206 56.289 ms, 7.05 s total +[ 2023-09-21 06:41:52 ] Completed train batch 1 discriminator backward 71.611 ms, 7.12 s total +Epoch [32] Step [1/2], gen_loss: 0.490, disc_loss: 0.072 +[ 2023-09-21 06:41:52 ] Completed train batch 1 metrics update 5.578 ms, 7.12 s total +Saving checkpoint at epoch 32 train batch 1 +[ 2023-09-21 06:41:52 ] Completed saving temp checkpoint 474.993 ms, 7.60 s total +[ 2023-09-21 06:41:52 ] Completed replacing temp checkpoint with checkpoint 25.553 ms, 7.63 s total +Epoch [32] :: gen_loss: 0.495, disc_loss: 0.069 +[ 2023-09-21 06:41:52 ] Completed training generator for epoch 32 68.731 ms, 7.69 s total +[ 2023-09-21 06:41:52 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 32 from eval step 0 + +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 06:53:07 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 06:53:07 ] Completed importing Timer 0.023 ms, 0.00 s total +[ 2023-09-21 06:53:12 ] Completed importing everything else 4,861.746 ms, 4.86 s total +| distributed init (rank 3): env:// +| distributed init (rank 2): env:// +| distributed init (rank 5): env:// +| distributed init (rank 0): env:// +| distributed init (rank 4): env:// +| distributed init (rank 1): env:// +[ 2023-09-21 06:53:20 ] Completed preliminaries 7,314.090 ms, 12.18 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 06:53:20 ] Completed build datasets 11.715 ms, 12.19 s total +[ 2023-09-21 06:53:20 ] Completed build samplers 0.047 ms, 12.19 s total +[ 2023-09-21 06:53:20 ] Completed build dataloaders 0.129 ms, 12.19 s total +[ 2023-09-21 06:53:20 ] Completed generator to device 95.272 ms, 12.28 s total +[ 2023-09-21 06:53:20 ] Completed discriminator to device 7.008 ms, 12.29 s total +[ 2023-09-21 06:53:20 ] Completed loss functions 373.022 ms, 12.66 s total +[ 2023-09-21 06:53:20 ] Completed models prepped for distribution 102.576 ms, 12.77 s total +[ 2023-09-21 06:53:20 ] Completed optimizers 0.671 ms, 12.77 s total +[ 2023-09-21 06:53:20 ] Completed grad scalers 0.029 ms, 12.77 s total +[ 2023-09-21 06:53:21 ] Completed checkpoint retrieval 493.675 ms, 13.26 s total + + +EPOCH :: 32 + + +[ 2023-09-21 06:53:21 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 32 from training step 2 + +Epoch [32] :: gen_loss: 0.495, disc_loss: 0.069 +[ 2023-09-21 06:53:21 ] Completed training generator for epoch 32 114.119 ms, 0.11 s total +[ 2023-09-21 06:53:21 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 32 from eval step 0 + +[ 2023-09-21 06:53:22 ] Completed eval batch 0 to device 1,543.577 ms, 1.54 s total +[ 2023-09-21 06:53:24 ] Completed eval batch 0 forward 1,545.339 ms, 3.09 s total +[ 2023-09-21 06:53:24 ] Completed eval batch 0 recons_loss 0.280 ms, 3.09 s total +[ 2023-09-21 06:53:24 ] Completed eval batch 0 metrics update 597.080 ms, 3.69 s total +Saving checkpoint at epoch 32 val batch 0 +[ 2023-09-21 06:53:25 ] Completed saving temp checkpoint 341.601 ms, 4.03 s total +[ 2023-09-21 06:53:25 ] Completed replacing temp checkpoint with checkpoint 124.077 ms, 4.15 s total +[ 2023-09-21 06:53:25 ] Completed eval batch 1 to device 16.725 ms, 4.17 s total +[ 2023-09-21 06:53:25 ] Completed eval batch 1 forward 21.114 ms, 4.19 s total +[ 2023-09-21 06:53:25 ] Completed eval batch 1 recons_loss 0.334 ms, 4.19 s total +[ 2023-09-21 06:53:25 ] Completed eval batch 1 metrics update 0.295 ms, 4.19 s total +Saving checkpoint at epoch 32 val batch 1 +Epoch 32 val loss: 0.0765 +[ 2023-09-21 06:53:25 ] Completed saving temp checkpoint 388.864 ms, 4.58 s total +[ 2023-09-21 06:53:25 ] Completed replacing temp checkpoint with checkpoint 21.574 ms, 4.60 s total +[ 2023-09-21 06:53:25 ] Completed evaluating generator for epoch 32 36.317 ms, 4.64 s total + + +EPOCH :: 33 + + +[ 2023-09-21 06:53:25 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 33 from training step 0 + +[ 2023-09-21 06:53:28 ] Completed train batch 0 to device 3,030.864 ms, 3.03 s total +[ 2023-09-21 06:53:28 ] Completed train batch 0 generator forward 36.295 ms, 3.07 s total +[ 2023-09-21 06:53:28 ] Completed train batch 0 generator loss: 1.482 76.236 ms, 3.14 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 06:53:29 ] Completed train batch 0 generator backward 846.601 ms, 3.99 s total +[ 2023-09-21 06:53:29 ] Completed train batch 0 discriminator loss 0.214 57.214 ms, 4.05 s total +[ 2023-09-21 06:53:30 ] Completed train batch 0 discriminator backward 119.646 ms, 4.17 s total +Epoch [33] Step [0/2], gen_loss: 0.494, disc_loss: 0.072 +[ 2023-09-21 06:53:30 ] Completed train batch 0 metrics update 6.875 ms, 4.17 s total +Saving checkpoint at epoch 33 train batch 0 +[ 2023-09-21 06:53:30 ] Completed saving temp checkpoint 385.520 ms, 4.56 s total +[ 2023-09-21 06:53:30 ] Completed replacing temp checkpoint with checkpoint 22.136 ms, 4.58 s total +[ 2023-09-21 06:53:32 ] Completed train batch 1 to device 1,722.468 ms, 6.30 s total +[ 2023-09-21 06:53:32 ] Completed train batch 1 generator forward 25.634 ms, 6.33 s total +[ 2023-09-21 06:53:32 ] Completed train batch 1 generator loss: 1.372 63.081 ms, 6.39 s total +[ 2023-09-21 06:53:33 ] Completed train batch 1 generator backward 840.321 ms, 7.23 s total +[ 2023-09-21 06:53:33 ] Completed train batch 1 discriminator loss 0.203 55.957 ms, 7.29 s total +[ 2023-09-21 06:53:33 ] Completed train batch 1 discriminator backward 93.029 ms, 7.38 s total +Epoch [33] Step [1/2], gen_loss: 0.491, disc_loss: 0.069 +[ 2023-09-21 06:53:33 ] Completed train batch 1 metrics update 4.242 ms, 7.39 s total +Saving checkpoint at epoch 33 train batch 1 +[ 2023-09-21 06:53:33 ] Completed saving temp checkpoint 403.596 ms, 7.79 s total +[ 2023-09-21 06:53:33 ] Completed replacing temp checkpoint with checkpoint 19.731 ms, 7.81 s total +Epoch [33] :: gen_loss: 0.493, disc_loss: 0.071 +[ 2023-09-21 06:53:33 ] Completed training generator for epoch 33 48.288 ms, 7.86 s total +[ 2023-09-21 06:53:33 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 33 from eval step 0 + +[ 2023-09-21 06:53:34 ] Completed eval batch 0 to device 1,156.707 ms, 1.16 s total +[ 2023-09-21 06:53:34 ] Completed eval batch 0 forward 24.287 ms, 1.18 s total +[ 2023-09-21 06:53:34 ] Completed eval batch 0 recons_loss 0.273 ms, 1.18 s total +[ 2023-09-21 06:53:34 ] Completed eval batch 0 metrics update 0.562 ms, 1.18 s total +Saving checkpoint at epoch 33 val batch 0 +[ 2023-09-21 06:53:35 ] Completed saving temp checkpoint 491.662 ms, 1.67 s total +[ 2023-09-21 06:53:35 ] Completed replacing temp checkpoint with checkpoint 38.420 ms, 1.71 s total +[ 2023-09-21 06:53:35 ] Completed eval batch 1 to device 316.236 ms, 2.03 s total +[ 2023-09-21 06:53:35 ] Completed eval batch 1 forward 22.106 ms, 2.05 s total +[ 2023-09-21 06:53:35 ] Completed eval batch 1 recons_loss 0.408 ms, 2.05 s total +[ 2023-09-21 06:53:36 ] Completed eval batch 1 metrics update 256.993 ms, 2.31 s total +Saving checkpoint at epoch 33 val batch 1 +Epoch 33 val loss: 0.0783 +[ 2023-09-21 06:53:36 ] Completed saving temp checkpoint 687.921 ms, 3.00 s total +[ 2023-09-21 06:53:36 ] Completed replacing temp checkpoint with checkpoint 35.277 ms, 3.03 s total +[ 2023-09-21 06:53:36 ] Completed evaluating generator for epoch 33 51.021 ms, 3.08 s total + + +EPOCH :: 34 + + +[ 2023-09-21 06:53:36 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 34 from training step 0 + +[ 2023-09-21 06:53:39 ] Completed train batch 0 to device 2,881.583 ms, 2.88 s total +[ 2023-09-21 06:53:39 ] Completed train batch 0 generator forward 24.707 ms, 2.91 s total +[ 2023-09-21 06:53:39 ] Completed train batch 0 generator loss: 1.541 65.098 ms, 2.97 s total +[ 2023-09-21 06:53:40 ] Completed train batch 0 generator backward 727.434 ms, 3.70 s total +[ 2023-09-21 06:53:40 ] Completed train batch 0 discriminator loss 0.182 57.259 ms, 3.76 s total +[ 2023-09-21 06:53:40 ] Completed train batch 0 discriminator backward 102.944 ms, 3.86 s total +Epoch [34] Step [0/2], gen_loss: 0.505, disc_loss: 0.065 +[ 2023-09-21 06:53:40 ] Completed train batch 0 metrics update 9.958 ms, 3.87 s total +Saving checkpoint at epoch 34 train batch 0 +[ 2023-09-21 06:53:41 ] Completed saving temp checkpoint 419.953 ms, 4.29 s total +[ 2023-09-21 06:53:41 ] Completed replacing temp checkpoint with checkpoint 20.192 ms, 4.31 s total +[ 2023-09-21 06:53:42 ] Completed train batch 1 to device 1,170.004 ms, 5.48 s total +[ 2023-09-21 06:53:42 ] Completed train batch 1 generator forward 40.896 ms, 5.52 s total +[ 2023-09-21 06:53:42 ] Completed train batch 1 generator loss: 1.550 61.235 ms, 5.58 s total +[ 2023-09-21 06:53:44 ] Completed train batch 1 generator backward 1,842.580 ms, 7.42 s total +[ 2023-09-21 06:53:44 ] Completed train batch 1 discriminator loss 0.186 55.682 ms, 7.48 s total +[ 2023-09-21 06:53:44 ] Completed train batch 1 discriminator backward 82.165 ms, 7.56 s total +Epoch [34] Step [1/2], gen_loss: 0.501, disc_loss: 0.062 +[ 2023-09-21 06:53:44 ] Completed train batch 1 metrics update 1.495 ms, 7.56 s total +Saving checkpoint at epoch 34 train batch 1 +[ 2023-09-21 06:53:44 ] Completed saving temp checkpoint 414.400 ms, 7.98 s total +[ 2023-09-21 06:53:44 ] Completed replacing temp checkpoint with checkpoint 19.745 ms, 8.00 s total +Epoch [34] :: gen_loss: 0.503, disc_loss: 0.063 +[ 2023-09-21 06:53:44 ] Completed training generator for epoch 34 54.274 ms, 8.05 s total +[ 2023-09-21 06:53:44 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 34 from eval step 0 + +[ 2023-09-21 06:53:45 ] Completed eval batch 0 to device 1,100.762 ms, 1.10 s total +[ 2023-09-21 06:53:45 ] Completed eval batch 0 forward 43.813 ms, 1.14 s total +[ 2023-09-21 06:53:45 ] Completed eval batch 0 recons_loss 0.360 ms, 1.14 s total +[ 2023-09-21 06:53:45 ] Completed eval batch 0 metrics update 0.655 ms, 1.15 s total +Saving checkpoint at epoch 34 val batch 0 +[ 2023-09-21 06:53:46 ] Completed saving temp checkpoint 418.936 ms, 1.56 s total +[ 2023-09-21 06:53:46 ] Completed replacing temp checkpoint with checkpoint 29.895 ms, 1.59 s total +[ 2023-09-21 06:53:46 ] Completed eval batch 1 to device 339.512 ms, 1.93 s total +[ 2023-09-21 06:53:46 ] Completed eval batch 1 forward 21.337 ms, 1.96 s total +[ 2023-09-21 06:53:46 ] Completed eval batch 1 recons_loss 0.335 ms, 1.96 s total +[ 2023-09-21 06:53:47 ] Completed eval batch 1 metrics update 332.383 ms, 2.29 s total +Saving checkpoint at epoch 34 val batch 1 +Epoch 34 val loss: 0.0768 +[ 2023-09-21 06:53:47 ] Completed saving temp checkpoint 415.019 ms, 2.70 s total +[ 2023-09-21 06:53:47 ] Completed replacing temp checkpoint with checkpoint 24.229 ms, 2.73 s total +[ 2023-09-21 06:53:47 ] Completed evaluating generator for epoch 34 52.691 ms, 2.78 s total + + +EPOCH :: 35 + + +[ 2023-09-21 06:53:47 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 35 from training step 0 + +[ 2023-09-21 06:53:50 ] Completed train batch 0 to device 3,047.354 ms, 3.05 s total +[ 2023-09-21 06:53:50 ] Completed train batch 0 generator forward 30.071 ms, 3.08 s total +[ 2023-09-21 06:53:50 ] Completed train batch 0 generator loss: 1.495 88.676 ms, 3.17 s total +[ 2023-09-21 06:53:51 ] Completed train batch 0 generator backward 678.617 ms, 3.84 s total +[ 2023-09-21 06:53:51 ] Completed train batch 0 discriminator loss 0.176 59.229 ms, 3.90 s total +[ 2023-09-21 06:53:51 ] Completed train batch 0 discriminator backward 99.949 ms, 4.00 s total +Epoch [35] Step [0/2], gen_loss: 0.506, disc_loss: 0.058 +[ 2023-09-21 06:53:51 ] Completed train batch 0 metrics update 4.300 ms, 4.01 s total +Saving checkpoint at epoch 35 train batch 0 +[ 2023-09-21 06:53:52 ] Completed saving temp checkpoint 431.631 ms, 4.44 s total +[ 2023-09-21 06:53:52 ] Completed replacing temp checkpoint with checkpoint 30.954 ms, 4.47 s total +[ 2023-09-21 06:53:53 ] Completed train batch 1 to device 1,260.500 ms, 5.73 s total +[ 2023-09-21 06:53:53 ] Completed train batch 1 generator forward 41.151 ms, 5.77 s total +[ 2023-09-21 06:53:53 ] Completed train batch 1 generator loss: 1.567 63.773 ms, 5.84 s total +[ 2023-09-21 06:53:54 ] Completed train batch 1 generator backward 1,346.190 ms, 7.18 s total +[ 2023-09-21 06:53:54 ] Completed train batch 1 discriminator loss 0.163 56.183 ms, 7.24 s total +[ 2023-09-21 06:53:54 ] Completed train batch 1 discriminator backward 74.359 ms, 7.31 s total +Epoch [35] Step [1/2], gen_loss: 0.504, disc_loss: 0.055 +[ 2023-09-21 06:53:54 ] Completed train batch 1 metrics update 0.894 ms, 7.31 s total +Saving checkpoint at epoch 35 train batch 1 +[ 2023-09-21 06:53:55 ] Completed saving temp checkpoint 405.449 ms, 7.72 s total +[ 2023-09-21 06:53:55 ] Completed replacing temp checkpoint with checkpoint 22.270 ms, 7.74 s total +Epoch [35] :: gen_loss: 0.505, disc_loss: 0.056 +[ 2023-09-21 06:53:55 ] Completed training generator for epoch 35 50.578 ms, 7.79 s total +[ 2023-09-21 06:53:55 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 35 from eval step 0 + +[ 2023-09-21 06:53:56 ] Completed eval batch 0 to device 1,051.696 ms, 1.05 s total +[ 2023-09-21 06:53:56 ] Completed eval batch 0 forward 24.921 ms, 1.08 s total +[ 2023-09-21 06:53:56 ] Completed eval batch 0 recons_loss 0.290 ms, 1.08 s total +[ 2023-09-21 06:53:56 ] Completed eval batch 0 metrics update 30.204 ms, 1.11 s total +Saving checkpoint at epoch 35 val batch 0 +[ 2023-09-21 06:53:56 ] Completed saving temp checkpoint 388.340 ms, 1.50 s total +[ 2023-09-21 06:53:56 ] Completed replacing temp checkpoint with checkpoint 34.696 ms, 1.53 s total +[ 2023-09-21 06:53:57 ] Completed eval batch 1 to device 405.081 ms, 1.94 s total +[ 2023-09-21 06:53:57 ] Completed eval batch 1 forward 39.814 ms, 1.98 s total +[ 2023-09-21 06:53:57 ] Completed eval batch 1 recons_loss 0.570 ms, 1.98 s total +[ 2023-09-21 06:53:57 ] Completed eval batch 1 metrics update 344.017 ms, 2.32 s total +Saving checkpoint at epoch 35 val batch 1 +Epoch 35 val loss: 0.0766 +[ 2023-09-21 06:53:58 ] Completed saving temp checkpoint 421.510 ms, 2.74 s total +[ 2023-09-21 06:53:58 ] Completed replacing temp checkpoint with checkpoint 15.814 ms, 2.76 s total +[ 2023-09-21 06:53:58 ] Completed evaluating generator for epoch 35 45.224 ms, 2.80 s total + + +EPOCH :: 36 + + +[ 2023-09-21 06:53:58 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 36 from training step 0 + +[ 2023-09-21 06:54:01 ] Completed train batch 0 to device 2,976.491 ms, 2.98 s total +[ 2023-09-21 06:54:01 ] Completed train batch 0 generator forward 25.112 ms, 3.00 s total +[ 2023-09-21 06:54:01 ] Completed train batch 0 generator loss: 1.483 64.761 ms, 3.07 s total +[ 2023-09-21 06:54:02 ] Completed train batch 0 generator backward 750.223 ms, 3.82 s total +[ 2023-09-21 06:54:02 ] Completed train batch 0 discriminator loss 0.163 56.859 ms, 3.87 s total +[ 2023-09-21 06:54:02 ] Completed train batch 0 discriminator backward 86.298 ms, 3.96 s total +Epoch [36] Step [0/2], gen_loss: 0.506, disc_loss: 0.053 +[ 2023-09-21 06:54:02 ] Completed train batch 0 metrics update 5.351 ms, 3.97 s total +Saving checkpoint at epoch 36 train batch 0 +[ 2023-09-21 06:54:02 ] Completed saving temp checkpoint 400.978 ms, 4.37 s total +[ 2023-09-21 06:54:02 ] Completed replacing temp checkpoint with checkpoint 24.168 ms, 4.39 s total +[ 2023-09-21 06:54:03 ] Completed train batch 1 to device 1,161.093 ms, 5.55 s total +[ 2023-09-21 06:54:03 ] Completed train batch 1 generator forward 40.788 ms, 5.59 s total +[ 2023-09-21 06:54:03 ] Completed train batch 1 generator loss: 1.513 61.635 ms, 5.65 s total +[ 2023-09-21 06:54:05 ] Completed train batch 1 generator backward 1,681.650 ms, 7.34 s total +[ 2023-09-21 06:54:05 ] Completed train batch 1 discriminator loss 0.154 55.760 ms, 7.39 s total +[ 2023-09-21 06:54:05 ] Completed train batch 1 discriminator backward 63.323 ms, 7.45 s total +Epoch [36] Step [1/2], gen_loss: 0.503, disc_loss: 0.051 +[ 2023-09-21 06:54:05 ] Completed train batch 1 metrics update 2.046 ms, 7.46 s total +Saving checkpoint at epoch 36 train batch 1 +[ 2023-09-21 06:54:06 ] Completed saving temp checkpoint 457.855 ms, 7.91 s total +[ 2023-09-21 06:54:06 ] Completed replacing temp checkpoint with checkpoint 25.195 ms, 7.94 s total +Epoch [36] :: gen_loss: 0.504, disc_loss: 0.052 +[ 2023-09-21 06:54:06 ] Completed training generator for epoch 36 61.489 ms, 8.00 s total +[ 2023-09-21 06:54:06 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 36 from eval step 0 + +[ 2023-09-21 06:54:07 ] Completed eval batch 0 to device 1,086.941 ms, 1.09 s total +[ 2023-09-21 06:54:07 ] Completed eval batch 0 forward 29.769 ms, 1.12 s total +[ 2023-09-21 06:54:07 ] Completed eval batch 0 recons_loss 0.316 ms, 1.12 s total +[ 2023-09-21 06:54:07 ] Completed eval batch 0 metrics update 0.595 ms, 1.12 s total +Saving checkpoint at epoch 36 val batch 0 +[ 2023-09-21 06:54:07 ] Completed saving temp checkpoint 423.341 ms, 1.54 s total +[ 2023-09-21 06:54:07 ] Completed replacing temp checkpoint with checkpoint 18.295 ms, 1.56 s total +[ 2023-09-21 06:54:08 ] Completed eval batch 1 to device 397.011 ms, 1.96 s total +[ 2023-09-21 06:54:08 ] Completed eval batch 1 forward 21.588 ms, 1.98 s total +[ 2023-09-21 06:54:08 ] Completed eval batch 1 recons_loss 0.345 ms, 1.98 s total +[ 2023-09-21 06:54:08 ] Completed eval batch 1 metrics update 206.117 ms, 2.18 s total +Saving checkpoint at epoch 36 val batch 1 +Epoch 36 val loss: 0.0719 +[ 2023-09-21 06:54:08 ] Completed saving temp checkpoint 426.792 ms, 2.61 s total +[ 2023-09-21 06:54:08 ] Completed replacing temp checkpoint with checkpoint 29.135 ms, 2.64 s total +[ 2023-09-21 06:54:08 ] Completed evaluating generator for epoch 36 51.482 ms, 2.69 s total + + +EPOCH :: 37 + + +[ 2023-09-21 06:54:08 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 37 from training step 0 + +[ 2023-09-21 06:54:11 ] Completed train batch 0 to device 3,043.030 ms, 3.04 s total +[ 2023-09-21 06:54:11 ] Completed train batch 0 generator forward 32.783 ms, 3.08 s total +[ 2023-09-21 06:54:12 ] Completed train batch 0 generator loss: 1.587 69.032 ms, 3.14 s total +[ 2023-09-21 06:54:12 ] Completed train batch 0 generator backward 635.842 ms, 3.78 s total +[ 2023-09-21 06:54:12 ] Completed train batch 0 discriminator loss 0.142 78.736 ms, 3.86 s total +[ 2023-09-21 06:54:12 ] Completed train batch 0 discriminator backward 76.232 ms, 3.94 s total +Epoch [37] Step [0/2], gen_loss: 0.511, disc_loss: 0.052 +[ 2023-09-21 06:54:12 ] Completed train batch 0 metrics update 24.142 ms, 3.96 s total +Saving checkpoint at epoch 37 train batch 0 +[ 2023-09-21 06:54:13 ] Completed saving temp checkpoint 440.071 ms, 4.40 s total +[ 2023-09-21 06:54:13 ] Completed replacing temp checkpoint with checkpoint 26.195 ms, 4.43 s total +[ 2023-09-21 06:54:14 ] Completed train batch 1 to device 1,338.557 ms, 5.76 s total +[ 2023-09-21 06:54:14 ] Completed train batch 1 generator forward 27.520 ms, 5.79 s total +[ 2023-09-21 06:54:14 ] Completed train batch 1 generator loss: 1.504 63.273 ms, 5.86 s total +[ 2023-09-21 06:54:15 ] Completed train batch 1 generator backward 1,076.797 ms, 6.93 s total +[ 2023-09-21 06:54:15 ] Completed train batch 1 discriminator loss 0.159 55.952 ms, 6.99 s total +[ 2023-09-21 06:54:15 ] Completed train batch 1 discriminator backward 74.880 ms, 7.06 s total +Epoch [37] Step [1/2], gen_loss: 0.504, disc_loss: 0.052 +[ 2023-09-21 06:54:15 ] Completed train batch 1 metrics update 1.965 ms, 7.07 s total +Saving checkpoint at epoch 37 train batch 1 +[ 2023-09-21 06:54:16 ] Completed saving temp checkpoint 473.460 ms, 7.54 s total +[ 2023-09-21 06:54:16 ] Completed replacing temp checkpoint with checkpoint 29.862 ms, 7.57 s total +Epoch [37] :: gen_loss: 0.508, disc_loss: 0.052 +[ 2023-09-21 06:54:16 ] Completed training generator for epoch 37 50.631 ms, 7.62 s total +[ 2023-09-21 06:54:16 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 37 from eval step 0 + +[ 2023-09-21 06:54:17 ] Completed eval batch 0 to device 1,083.109 ms, 1.08 s total +[ 2023-09-21 06:54:17 ] Completed eval batch 0 forward 35.010 ms, 1.12 s total +[ 2023-09-21 06:54:17 ] Completed eval batch 0 recons_loss 0.304 ms, 1.12 s total +[ 2023-09-21 06:54:17 ] Completed eval batch 0 metrics update 0.564 ms, 1.12 s total +Saving checkpoint at epoch 37 val batch 0 +[ 2023-09-21 06:54:18 ] Completed saving temp checkpoint 626.016 ms, 1.75 s total +[ 2023-09-21 06:54:18 ] Completed replacing temp checkpoint with checkpoint 33.807 ms, 1.78 s total +[ 2023-09-21 06:54:18 ] Completed eval batch 1 to device 192.243 ms, 1.97 s total +[ 2023-09-21 06:54:18 ] Completed eval batch 1 forward 21.367 ms, 1.99 s total +[ 2023-09-21 06:54:18 ] Completed eval batch 1 recons_loss 0.344 ms, 1.99 s total +[ 2023-09-21 06:54:18 ] Completed eval batch 1 metrics update 286.619 ms, 2.28 s total +Saving checkpoint at epoch 37 val batch 1 +Epoch 37 val loss: 0.0655 +[ 2023-09-21 06:54:19 ] Completed saving temp checkpoint 722.171 ms, 3.00 s total +[ 2023-09-21 06:54:19 ] Completed replacing temp checkpoint with checkpoint 51.861 ms, 3.05 s total +[ 2023-09-21 06:54:19 ] Completed evaluating generator for epoch 37 58.514 ms, 3.11 s total + + +EPOCH :: 38 + + +[ 2023-09-21 06:54:19 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 38 from training step 0 + +[ 2023-09-21 06:54:22 ] Completed train batch 0 to device 3,225.227 ms, 3.23 s total +[ 2023-09-21 06:54:22 ] Completed train batch 0 generator forward 26.361 ms, 3.25 s total +[ 2023-09-21 06:54:22 ] Completed train batch 0 generator loss: 1.511 65.745 ms, 3.32 s total +[ 2023-09-21 06:54:23 ] Completed train batch 0 generator backward 250.845 ms, 3.57 s total +[ 2023-09-21 06:54:23 ] Completed train batch 0 discriminator loss 0.155 56.377 ms, 3.62 s total +[ 2023-09-21 06:54:23 ] Completed train batch 0 discriminator backward 100.811 ms, 3.73 s total +Epoch [38] Step [0/2], gen_loss: 0.502, disc_loss: 0.052 +[ 2023-09-21 06:54:23 ] Completed train batch 0 metrics update 6.473 ms, 3.73 s total +Saving checkpoint at epoch 38 train batch 0 +[ 2023-09-21 06:54:23 ] Completed saving temp checkpoint 424.719 ms, 4.16 s total +[ 2023-09-21 06:54:23 ] Completed replacing temp checkpoint with checkpoint 18.065 ms, 4.17 s total +[ 2023-09-21 06:54:25 ] Completed train batch 1 to device 1,994.726 ms, 6.17 s total +[ 2023-09-21 06:54:25 ] Completed train batch 1 generator forward 22.193 ms, 6.19 s total +[ 2023-09-21 06:54:25 ] Completed train batch 1 generator loss: 1.479 63.478 ms, 6.26 s total +[ 2023-09-21 06:54:26 ] Completed train batch 1 generator backward 624.587 ms, 6.88 s total +[ 2023-09-21 06:54:26 ] Completed train batch 1 discriminator loss 0.147 56.019 ms, 6.94 s total +[ 2023-09-21 06:54:26 ] Completed train batch 1 discriminator backward 72.333 ms, 7.01 s total +Epoch [38] Step [1/2], gen_loss: 0.491, disc_loss: 0.049 +[ 2023-09-21 06:54:26 ] Completed train batch 1 metrics update 1.233 ms, 7.01 s total +Saving checkpoint at epoch 38 train batch 1 +[ 2023-09-21 06:54:27 ] Completed saving temp checkpoint 437.039 ms, 7.45 s total +[ 2023-09-21 06:54:27 ] Completed replacing temp checkpoint with checkpoint 29.549 ms, 7.48 s total +Epoch [38] :: gen_loss: 0.497, disc_loss: 0.050 +[ 2023-09-21 06:54:27 ] Completed training generator for epoch 38 48.774 ms, 7.52 s total +[ 2023-09-21 06:54:27 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 38 from eval step 0 + +[ 2023-09-21 06:54:28 ] Completed eval batch 0 to device 1,148.208 ms, 1.15 s total +[ 2023-09-21 06:54:28 ] Completed eval batch 0 forward 40.780 ms, 1.19 s total +[ 2023-09-21 06:54:28 ] Completed eval batch 0 recons_loss 0.337 ms, 1.19 s total +[ 2023-09-21 06:54:28 ] Completed eval batch 0 metrics update 0.568 ms, 1.19 s total +Saving checkpoint at epoch 38 val batch 0 +[ 2023-09-21 06:54:28 ] Completed saving temp checkpoint 425.403 ms, 1.62 s total +[ 2023-09-21 06:54:28 ] Completed replacing temp checkpoint with checkpoint 22.126 ms, 1.64 s total +[ 2023-09-21 06:54:29 ] Completed eval batch 1 to device 394.232 ms, 2.03 s total +[ 2023-09-21 06:54:29 ] Completed eval batch 1 forward 25.624 ms, 2.06 s total +[ 2023-09-21 06:54:29 ] Completed eval batch 1 recons_loss 0.371 ms, 2.06 s total +[ 2023-09-21 06:54:29 ] Completed eval batch 1 metrics update 260.318 ms, 2.32 s total +Saving checkpoint at epoch 38 val batch 1 +Epoch 38 val loss: 0.0640 +[ 2023-09-21 06:54:29 ] Completed saving temp checkpoint 455.059 ms, 2.77 s total +[ 2023-09-21 06:54:29 ] Completed replacing temp checkpoint with checkpoint 26.870 ms, 2.80 s total +[ 2023-09-21 06:54:30 ] Completed evaluating generator for epoch 38 49.693 ms, 2.85 s total + + +EPOCH :: 39 + + +[ 2023-09-21 06:54:30 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 39 from training step 0 + +[ 2023-09-21 06:54:33 ] Completed train batch 0 to device 3,026.482 ms, 3.03 s total +[ 2023-09-21 06:54:33 ] Completed train batch 0 generator forward 30.629 ms, 3.06 s total +[ 2023-09-21 06:54:33 ] Completed train batch 0 generator loss: 1.482 88.301 ms, 3.15 s total +[ 2023-09-21 06:54:33 ] Completed train batch 0 generator backward 643.514 ms, 3.79 s total +[ 2023-09-21 06:54:33 ] Completed train batch 0 discriminator loss 0.150 56.823 ms, 3.85 s total +[ 2023-09-21 06:54:33 ] Completed train batch 0 discriminator backward 125.582 ms, 3.97 s total +Epoch [39] Step [0/2], gen_loss: 0.497, disc_loss: 0.051 +[ 2023-09-21 06:54:33 ] Completed train batch 0 metrics update 12.338 ms, 3.98 s total +Saving checkpoint at epoch 39 train batch 0 +[ 2023-09-21 06:54:34 ] Completed saving temp checkpoint 689.632 ms, 4.67 s total +[ 2023-09-21 06:54:34 ] Completed replacing temp checkpoint with checkpoint 27.091 ms, 4.70 s total +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 07:05:23 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 07:05:23 ] Completed importing Timer 0.023 ms, 0.00 s total +[ 2023-09-21 07:05:28 ] Completed importing everything else 4,644.737 ms, 4.64 s total +| distributed init (rank 5): env:// +| distributed init (rank 3): env:// +| distributed init (rank 1): env:// +| distributed init (rank 0): env:// +| distributed init (rank 2): env:// +| distributed init (rank 4): env:// +[ 2023-09-21 07:05:36 ] Completed preliminaries 7,988.197 ms, 12.63 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 07:05:36 ] Completed build datasets 11.948 ms, 12.64 s total +[ 2023-09-21 07:05:36 ] Completed build samplers 0.047 ms, 12.64 s total +[ 2023-09-21 07:05:36 ] Completed build dataloaders 0.136 ms, 12.65 s total +[ 2023-09-21 07:05:36 ] Completed generator to device 88.167 ms, 12.73 s total +[ 2023-09-21 07:05:36 ] Completed discriminator to device 7.019 ms, 12.74 s total +[ 2023-09-21 07:05:37 ] Completed loss functions 390.725 ms, 13.13 s total +[ 2023-09-21 07:05:37 ] Completed models prepped for distribution 65.639 ms, 13.20 s total +[ 2023-09-21 07:05:37 ] Completed optimizers 0.690 ms, 13.20 s total +[ 2023-09-21 07:05:37 ] Completed grad scalers 0.032 ms, 13.20 s total +[ 2023-09-21 07:05:37 ] Completed checkpoint retrieval 443.243 ms, 13.64 s total + + +EPOCH :: 39 + + +[ 2023-09-21 07:05:37 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 39 from training step 1 + +[ 2023-09-21 07:05:40 ] Completed train batch 1 to device 3,022.966 ms, 3.02 s total +[ 2023-09-21 07:05:41 ] Completed train batch 1 generator forward 1,066.931 ms, 4.09 s total +[ 2023-09-21 07:05:41 ] Completed train batch 1 generator loss: 1.563 72.880 ms, 4.16 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 07:05:43 ] Completed train batch 1 generator backward 1,574.335 ms, 5.74 s total +[ 2023-09-21 07:05:43 ] Completed train batch 1 discriminator loss 0.146 55.807 ms, 5.79 s total +[ 2023-09-21 07:05:43 ] Completed train batch 1 discriminator backward 91.901 ms, 5.88 s total +Epoch [39] Step [1/2], gen_loss: 0.503, disc_loss: 0.049 +[ 2023-09-21 07:05:43 ] Completed train batch 1 metrics update 1.248 ms, 5.89 s total +Saving checkpoint at epoch 39 train batch 1 +[ 2023-09-21 07:05:43 ] Completed saving temp checkpoint 417.989 ms, 6.30 s total +[ 2023-09-21 07:05:44 ] Completed replacing temp checkpoint with checkpoint 113.759 ms, 6.42 s total +Epoch [39] :: gen_loss: 0.500, disc_loss: 0.050 +[ 2023-09-21 07:05:44 ] Completed training generator for epoch 39 39.509 ms, 6.46 s total +[ 2023-09-21 07:05:44 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 39 from eval step 0 + +[ 2023-09-21 07:05:45 ] Completed eval batch 0 to device 1,159.857 ms, 1.16 s total +[ 2023-09-21 07:05:45 ] Completed eval batch 0 forward 40.262 ms, 1.20 s total +[ 2023-09-21 07:05:45 ] Completed eval batch 0 recons_loss 0.366 ms, 1.20 s total +[ 2023-09-21 07:05:45 ] Completed eval batch 0 metrics update 0.676 ms, 1.20 s total +Saving checkpoint at epoch 39 val batch 0 +[ 2023-09-21 07:05:45 ] Completed saving temp checkpoint 396.578 ms, 1.60 s total +[ 2023-09-21 07:05:45 ] Completed replacing temp checkpoint with checkpoint 24.507 ms, 1.62 s total +[ 2023-09-21 07:05:46 ] Completed eval batch 1 to device 363.866 ms, 1.99 s total +[ 2023-09-21 07:05:46 ] Completed eval batch 1 forward 25.247 ms, 2.01 s total +[ 2023-09-21 07:05:46 ] Completed eval batch 1 recons_loss 0.377 ms, 2.01 s total +[ 2023-09-21 07:05:46 ] Completed eval batch 1 metrics update 155.394 ms, 2.17 s total +Saving checkpoint at epoch 39 val batch 1 +Epoch 39 val loss: 0.0645 +[ 2023-09-21 07:05:46 ] Completed saving temp checkpoint 433.957 ms, 2.60 s total +[ 2023-09-21 07:05:46 ] Completed replacing temp checkpoint with checkpoint 19.637 ms, 2.62 s total +[ 2023-09-21 07:05:46 ] Completed evaluating generator for epoch 39 60.775 ms, 2.68 s total + + +EPOCH :: 40 + + +[ 2023-09-21 07:05:46 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 40 from training step 0 + +[ 2023-09-21 07:05:49 ] Completed train batch 0 to device 3,154.577 ms, 3.15 s total +[ 2023-09-21 07:05:49 ] Completed train batch 0 generator forward 27.481 ms, 3.18 s total +[ 2023-09-21 07:05:50 ] Completed train batch 0 generator loss: 1.498 70.924 ms, 3.25 s total +[ 2023-09-21 07:05:50 ] Completed train batch 0 generator backward 510.772 ms, 3.76 s total +[ 2023-09-21 07:05:50 ] Completed train batch 0 discriminator loss 0.140 57.187 ms, 3.82 s total +[ 2023-09-21 07:05:50 ] Completed train batch 0 discriminator backward 107.492 ms, 3.93 s total +Epoch [40] Step [0/2], gen_loss: 0.499, disc_loss: 0.049 +[ 2023-09-21 07:05:50 ] Completed train batch 0 metrics update 5.561 ms, 3.93 s total +Saving checkpoint at epoch 40 train batch 0 +[ 2023-09-21 07:05:51 ] Completed saving temp checkpoint 453.000 ms, 4.39 s total +[ 2023-09-21 07:05:51 ] Completed replacing temp checkpoint with checkpoint 23.541 ms, 4.41 s total +[ 2023-09-21 07:05:52 ] Completed train batch 1 to device 1,805.453 ms, 6.22 s total +[ 2023-09-21 07:05:52 ] Completed train batch 1 generator forward 22.757 ms, 6.24 s total +[ 2023-09-21 07:05:53 ] Completed train batch 1 generator loss: 1.468 63.663 ms, 6.30 s total +[ 2023-09-21 07:05:53 ] Completed train batch 1 generator backward 831.206 ms, 7.13 s total +[ 2023-09-21 07:05:53 ] Completed train batch 1 discriminator loss 0.148 56.479 ms, 7.19 s total +[ 2023-09-21 07:05:54 ] Completed train batch 1 discriminator backward 76.448 ms, 7.27 s total +Epoch [40] Step [1/2], gen_loss: 0.488, disc_loss: 0.049 +[ 2023-09-21 07:05:54 ] Completed train batch 1 metrics update 0.989 ms, 7.27 s total +Saving checkpoint at epoch 40 train batch 1 +[ 2023-09-21 07:05:54 ] Completed saving temp checkpoint 530.147 ms, 7.80 s total +[ 2023-09-21 07:05:54 ] Completed replacing temp checkpoint with checkpoint 29.833 ms, 7.83 s total +Epoch [40] :: gen_loss: 0.493, disc_loss: 0.049 +[ 2023-09-21 07:05:54 ] Completed training generator for epoch 40 77.305 ms, 7.90 s total +[ 2023-09-21 07:05:54 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 40 from eval step 0 + +[ 2023-09-21 07:05:55 ] Completed eval batch 0 to device 1,141.667 ms, 1.14 s total +[ 2023-09-21 07:05:55 ] Completed eval batch 0 forward 28.488 ms, 1.17 s total +[ 2023-09-21 07:05:55 ] Completed eval batch 0 recons_loss 0.371 ms, 1.17 s total +[ 2023-09-21 07:05:55 ] Completed eval batch 0 metrics update 0.644 ms, 1.17 s total +Saving checkpoint at epoch 40 val batch 0 +[ 2023-09-21 07:05:56 ] Completed saving temp checkpoint 657.932 ms, 1.83 s total +[ 2023-09-21 07:05:56 ] Completed replacing temp checkpoint with checkpoint 38.127 ms, 1.87 s total +[ 2023-09-21 07:05:56 ] Completed eval batch 1 to device 175.778 ms, 2.04 s total +[ 2023-09-21 07:05:56 ] Completed eval batch 1 forward 24.024 ms, 2.07 s total +[ 2023-09-21 07:05:56 ] Completed eval batch 1 recons_loss 0.348 ms, 2.07 s total +[ 2023-09-21 07:05:56 ] Completed eval batch 1 metrics update 79.519 ms, 2.15 s total +Saving checkpoint at epoch 40 val batch 1 +Epoch 40 val loss: 0.0600 +[ 2023-09-21 07:05:57 ] Completed saving temp checkpoint 671.251 ms, 2.82 s total +[ 2023-09-21 07:05:57 ] Completed replacing temp checkpoint with checkpoint 41.162 ms, 2.86 s total +[ 2023-09-21 07:05:57 ] Completed evaluating generator for epoch 40 54.755 ms, 2.91 s total + + +EPOCH :: 41 + + +[ 2023-09-21 07:05:57 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 41 from training step 0 + +[ 2023-09-21 07:06:00 ] Completed train batch 0 to device 3,070.886 ms, 3.07 s total +[ 2023-09-21 07:06:00 ] Completed train batch 0 generator forward 25.109 ms, 3.10 s total +[ 2023-09-21 07:06:00 ] Completed train batch 0 generator loss: 1.535 66.550 ms, 3.16 s total +[ 2023-09-21 07:06:01 ] Completed train batch 0 generator backward 388.998 ms, 3.55 s total +[ 2023-09-21 07:06:01 ] Completed train batch 0 discriminator loss 0.145 80.970 ms, 3.63 s total +[ 2023-09-21 07:06:01 ] Completed train batch 0 discriminator backward 79.344 ms, 3.71 s total +Epoch [41] Step [0/2], gen_loss: 0.498, disc_loss: 0.048 +[ 2023-09-21 07:06:01 ] Completed train batch 0 metrics update 1.655 ms, 3.71 s total +Saving checkpoint at epoch 41 train batch 0 +[ 2023-09-21 07:06:01 ] Completed saving temp checkpoint 438.953 ms, 4.15 s total +[ 2023-09-21 07:06:01 ] Completed replacing temp checkpoint with checkpoint 19.534 ms, 4.17 s total +[ 2023-09-21 07:06:03 ] Completed train batch 1 to device 1,715.813 ms, 5.89 s total +[ 2023-09-21 07:06:03 ] Completed train batch 1 generator forward 22.355 ms, 5.91 s total +[ 2023-09-21 07:06:03 ] Completed train batch 1 generator loss: 1.472 63.750 ms, 5.97 s total +[ 2023-09-21 07:06:04 ] Completed train batch 1 generator backward 1,066.320 ms, 7.04 s total +[ 2023-09-21 07:06:04 ] Completed train batch 1 discriminator loss 0.143 69.552 ms, 7.11 s total +[ 2023-09-21 07:06:04 ] Completed train batch 1 discriminator backward 59.250 ms, 7.17 s total +Epoch [41] Step [1/2], gen_loss: 0.489, disc_loss: 0.047 +[ 2023-09-21 07:06:04 ] Completed train batch 1 metrics update 1.932 ms, 7.17 s total +Saving checkpoint at epoch 41 train batch 1 +[ 2023-09-21 07:06:05 ] Completed saving temp checkpoint 472.203 ms, 7.64 s total +[ 2023-09-21 07:06:05 ] Completed replacing temp checkpoint with checkpoint 23.048 ms, 7.67 s total +Epoch [41] :: gen_loss: 0.494, disc_loss: 0.048 +[ 2023-09-21 07:06:05 ] Completed training generator for epoch 41 59.584 ms, 7.73 s total +[ 2023-09-21 07:06:05 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 41 from eval step 0 + +[ 2023-09-21 07:06:06 ] Completed eval batch 0 to device 1,130.800 ms, 1.13 s total +[ 2023-09-21 07:06:06 ] Completed eval batch 0 forward 30.964 ms, 1.16 s total +[ 2023-09-21 07:06:06 ] Completed eval batch 0 recons_loss 0.520 ms, 1.16 s total +[ 2023-09-21 07:06:06 ] Completed eval batch 0 metrics update 1.146 ms, 1.16 s total +Saving checkpoint at epoch 41 val batch 0 +[ 2023-09-21 07:06:06 ] Completed saving temp checkpoint 480.171 ms, 1.64 s total +[ 2023-09-21 07:06:06 ] Completed replacing temp checkpoint with checkpoint 36.711 ms, 1.68 s total +[ 2023-09-21 07:06:07 ] Completed eval batch 1 to device 359.848 ms, 2.04 s total +[ 2023-09-21 07:06:07 ] Completed eval batch 1 forward 40.314 ms, 2.08 s total +[ 2023-09-21 07:06:07 ] Completed eval batch 1 recons_loss 0.593 ms, 2.08 s total +[ 2023-09-21 07:06:07 ] Completed eval batch 1 metrics update 124.310 ms, 2.21 s total +Saving checkpoint at epoch 41 val batch 1 +Epoch 41 val loss: 0.0598 +[ 2023-09-21 07:06:07 ] Completed saving temp checkpoint 475.137 ms, 2.68 s total +[ 2023-09-21 07:06:08 ] Completed replacing temp checkpoint with checkpoint 28.390 ms, 2.71 s total +[ 2023-09-21 07:06:08 ] Completed evaluating generator for epoch 41 54.527 ms, 2.76 s total + + +EPOCH :: 42 + + +[ 2023-09-21 07:06:08 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 42 from training step 0 + +[ 2023-09-21 07:06:11 ] Completed train batch 0 to device 3,121.244 ms, 3.12 s total +[ 2023-09-21 07:06:11 ] Completed train batch 0 generator forward 27.734 ms, 3.15 s total +[ 2023-09-21 07:06:11 ] Completed train batch 0 generator loss: 1.512 68.441 ms, 3.22 s total +[ 2023-09-21 07:06:11 ] Completed train batch 0 generator backward 521.588 ms, 3.74 s total +[ 2023-09-21 07:06:11 ] Completed train batch 0 discriminator loss 0.130 82.214 ms, 3.82 s total +[ 2023-09-21 07:06:11 ] Completed train batch 0 discriminator backward 82.447 ms, 3.90 s total +Epoch [42] Step [0/2], gen_loss: 0.510, disc_loss: 0.043 +[ 2023-09-21 07:06:11 ] Completed train batch 0 metrics update 0.893 ms, 3.90 s total +Saving checkpoint at epoch 42 train batch 0 +[ 2023-09-21 07:06:12 ] Completed saving temp checkpoint 474.831 ms, 4.38 s total +[ 2023-09-21 07:06:12 ] Completed replacing temp checkpoint with checkpoint 26.178 ms, 4.41 s total +[ 2023-09-21 07:06:14 ] Completed train batch 1 to device 1,555.094 ms, 5.96 s total +[ 2023-09-21 07:06:14 ] Completed train batch 1 generator forward 26.635 ms, 5.99 s total +[ 2023-09-21 07:06:14 ] Completed train batch 1 generator loss: 1.503 63.590 ms, 6.05 s total +[ 2023-09-21 07:06:15 ] Completed train batch 1 generator backward 1,271.978 ms, 7.32 s total +[ 2023-09-21 07:06:15 ] Completed train batch 1 discriminator loss 0.137 56.366 ms, 7.38 s total +[ 2023-09-21 07:06:15 ] Completed train batch 1 discriminator backward 75.974 ms, 7.46 s total +Epoch [42] Step [1/2], gen_loss: 0.505, disc_loss: 0.046 +[ 2023-09-21 07:06:15 ] Completed train batch 1 metrics update 3.972 ms, 7.46 s total +Saving checkpoint at epoch 42 train batch 1 +[ 2023-09-21 07:06:15 ] Completed saving temp checkpoint 403.848 ms, 7.86 s total +[ 2023-09-21 07:06:15 ] Completed replacing temp checkpoint with checkpoint 25.428 ms, 7.89 s total +Epoch [42] :: gen_loss: 0.507, disc_loss: 0.045 +[ 2023-09-21 07:06:16 ] Completed training generator for epoch 42 61.049 ms, 7.95 s total +[ 2023-09-21 07:06:16 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 42 from eval step 0 + +[ 2023-09-21 07:06:17 ] Completed eval batch 0 to device 1,113.964 ms, 1.11 s total +[ 2023-09-21 07:06:17 ] Completed eval batch 0 forward 34.359 ms, 1.15 s total +[ 2023-09-21 07:06:17 ] Completed eval batch 0 recons_loss 0.364 ms, 1.15 s total +[ 2023-09-21 07:06:17 ] Completed eval batch 0 metrics update 0.618 ms, 1.15 s total +Saving checkpoint at epoch 42 val batch 0 +[ 2023-09-21 07:06:17 ] Completed saving temp checkpoint 365.322 ms, 1.51 s total +[ 2023-09-21 07:06:17 ] Completed replacing temp checkpoint with checkpoint 25.831 ms, 1.54 s total +[ 2023-09-21 07:06:18 ] Completed eval batch 1 to device 507.185 ms, 2.05 s total +[ 2023-09-21 07:06:18 ] Completed eval batch 1 forward 21.934 ms, 2.07 s total +[ 2023-09-21 07:06:18 ] Completed eval batch 1 recons_loss 0.379 ms, 2.07 s total +[ 2023-09-21 07:06:18 ] Completed eval batch 1 metrics update 205.030 ms, 2.27 s total +Saving checkpoint at epoch 42 val batch 1 +Epoch 42 val loss: 0.0852 +[ 2023-09-21 07:06:18 ] Completed saving temp checkpoint 461.410 ms, 2.74 s total +[ 2023-09-21 07:06:18 ] Completed replacing temp checkpoint with checkpoint 20.019 ms, 2.76 s total +[ 2023-09-21 07:06:18 ] Completed evaluating generator for epoch 42 51.541 ms, 2.81 s total + + +EPOCH :: 43 + + +[ 2023-09-21 07:06:18 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 43 from training step 0 + +[ 2023-09-21 07:06:21 ] Completed train batch 0 to device 3,014.369 ms, 3.01 s total +[ 2023-09-21 07:06:21 ] Completed train batch 0 generator forward 30.335 ms, 3.04 s total +[ 2023-09-21 07:06:21 ] Completed train batch 0 generator loss: 1.506 66.371 ms, 3.11 s total +[ 2023-09-21 07:06:22 ] Completed train batch 0 generator backward 654.990 ms, 3.77 s total +[ 2023-09-21 07:06:22 ] Completed train batch 0 discriminator loss 0.133 57.384 ms, 3.82 s total +[ 2023-09-21 07:06:22 ] Completed train batch 0 discriminator backward 106.099 ms, 3.93 s total +Epoch [43] Step [0/2], gen_loss: 0.495, disc_loss: 0.045 +[ 2023-09-21 07:06:22 ] Completed train batch 0 metrics update 5.368 ms, 3.93 s total +Saving checkpoint at epoch 43 train batch 0 +[ 2023-09-21 07:06:23 ] Completed saving temp checkpoint 448.580 ms, 4.38 s total +[ 2023-09-21 07:06:23 ] Completed replacing temp checkpoint with checkpoint 25.432 ms, 4.41 s total +[ 2023-09-21 07:06:24 ] Completed train batch 1 to device 1,339.399 ms, 5.75 s total +[ 2023-09-21 07:06:24 ] Completed train batch 1 generator forward 40.666 ms, 5.79 s total +[ 2023-09-21 07:06:24 ] Completed train batch 1 generator loss: 1.522 63.405 ms, 5.85 s total +[ 2023-09-21 07:06:25 ] Completed train batch 1 generator backward 1,184.482 ms, 7.04 s total +[ 2023-09-21 07:06:25 ] Completed train batch 1 discriminator loss 0.127 56.162 ms, 7.09 s total +[ 2023-09-21 07:06:25 ] Completed train batch 1 discriminator backward 74.925 ms, 7.17 s total +Epoch [43] Step [1/2], gen_loss: 0.513, disc_loss: 0.043 +[ 2023-09-21 07:06:25 ] Completed train batch 1 metrics update 1.218 ms, 7.17 s total +Saving checkpoint at epoch 43 train batch 1 +[ 2023-09-21 07:06:26 ] Completed saving temp checkpoint 459.538 ms, 7.63 s total +[ 2023-09-21 07:06:26 ] Completed replacing temp checkpoint with checkpoint 28.861 ms, 7.66 s total +Epoch [43] :: gen_loss: 0.504, disc_loss: 0.044 +[ 2023-09-21 07:06:26 ] Completed training generator for epoch 43 57.037 ms, 7.71 s total +[ 2023-09-21 07:06:26 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 43 from eval step 0 + +[ 2023-09-21 07:06:27 ] Completed eval batch 0 to device 1,030.465 ms, 1.03 s total +[ 2023-09-21 07:06:27 ] Completed eval batch 0 forward 30.254 ms, 1.06 s total +[ 2023-09-21 07:06:27 ] Completed eval batch 0 recons_loss 0.281 ms, 1.06 s total +[ 2023-09-21 07:06:27 ] Completed eval batch 0 metrics update 0.583 ms, 1.06 s total +Saving checkpoint at epoch 43 val batch 0 +[ 2023-09-21 07:06:27 ] Completed saving temp checkpoint 389.758 ms, 1.45 s total +[ 2023-09-21 07:06:28 ] Completed replacing temp checkpoint with checkpoint 30.970 ms, 1.48 s total +[ 2023-09-21 07:06:28 ] Completed eval batch 1 to device 530.329 ms, 2.01 s total +[ 2023-09-21 07:06:28 ] Completed eval batch 1 forward 39.214 ms, 2.05 s total +[ 2023-09-21 07:06:28 ] Completed eval batch 1 recons_loss 0.349 ms, 2.05 s total +[ 2023-09-21 07:06:28 ] Completed eval batch 1 metrics update 149.366 ms, 2.20 s total +Saving checkpoint at epoch 43 val batch 1 +Epoch 43 val loss: 0.0800 +[ 2023-09-21 07:06:29 ] Completed saving temp checkpoint 406.484 ms, 2.61 s total +[ 2023-09-21 07:06:29 ] Completed replacing temp checkpoint with checkpoint 21.200 ms, 2.63 s total +[ 2023-09-21 07:06:29 ] Completed evaluating generator for epoch 43 53.386 ms, 2.68 s total + + +EPOCH :: 44 + + +[ 2023-09-21 07:06:29 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 44 from training step 0 + +[ 2023-09-21 07:06:32 ] Completed train batch 0 to device 3,202.604 ms, 3.20 s total +[ 2023-09-21 07:06:32 ] Completed train batch 0 generator forward 41.640 ms, 3.24 s total +[ 2023-09-21 07:06:32 ] Completed train batch 0 generator loss: 1.566 80.623 ms, 3.32 s total +[ 2023-09-21 07:06:33 ] Completed train batch 0 generator backward 525.749 ms, 3.85 s total +[ 2023-09-21 07:06:33 ] Completed train batch 0 discriminator loss 0.123 59.276 ms, 3.91 s total +[ 2023-09-21 07:06:33 ] Completed train batch 0 discriminator backward 102.375 ms, 4.01 s total +Epoch [44] Step [0/2], gen_loss: 0.521, disc_loss: 0.041 +[ 2023-09-21 07:06:33 ] Completed train batch 0 metrics update 11.493 ms, 4.02 s total +Saving checkpoint at epoch 44 train batch 0 +[ 2023-09-21 07:06:33 ] Completed saving temp checkpoint 347.530 ms, 4.37 s total +[ 2023-09-21 07:06:33 ] Completed replacing temp checkpoint with checkpoint 22.018 ms, 4.39 s total +[ 2023-09-21 07:06:35 ] Completed train batch 1 to device 1,676.775 ms, 6.07 s total +[ 2023-09-21 07:06:35 ] Completed train batch 1 generator forward 26.091 ms, 6.10 s total +[ 2023-09-21 07:06:35 ] Completed train batch 1 generator loss: 1.524 67.508 ms, 6.16 s total +[ 2023-09-21 07:06:36 ] Completed train batch 1 generator backward 955.951 ms, 7.12 s total +[ 2023-09-21 07:06:36 ] Completed train batch 1 discriminator loss 0.129 57.411 ms, 7.18 s total +[ 2023-09-21 07:06:36 ] Completed train batch 1 discriminator backward 66.774 ms, 7.24 s total +Epoch [44] Step [1/2], gen_loss: 0.511, disc_loss: 0.044 +[ 2023-09-21 07:06:36 ] Completed train batch 1 metrics update 2.074 ms, 7.25 s total +Saving checkpoint at epoch 44 train batch 1 +[ 2023-09-21 07:06:36 ] Completed saving temp checkpoint 434.741 ms, 7.68 s total +[ 2023-09-21 07:06:36 ] Completed replacing temp checkpoint with checkpoint 30.576 ms, 7.71 s total +Epoch [44] :: gen_loss: 0.516, disc_loss: 0.043 +[ 2023-09-21 07:06:36 ] Completed training generator for epoch 44 60.137 ms, 7.77 s total +[ 2023-09-21 07:06:36 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 44 from eval step 0 + +[ 2023-09-21 07:06:38 ] Completed eval batch 0 to device 1,062.463 ms, 1.06 s total +[ 2023-09-21 07:06:38 ] Completed eval batch 0 forward 29.764 ms, 1.09 s total +[ 2023-09-21 07:06:38 ] Completed eval batch 0 recons_loss 0.274 ms, 1.09 s total +[ 2023-09-21 07:06:38 ] Completed eval batch 0 metrics update 0.594 ms, 1.09 s total +Saving checkpoint at epoch 44 val batch 0 +[ 2023-09-21 07:06:38 ] Completed saving temp checkpoint 475.919 ms, 1.57 s total +[ 2023-09-21 07:06:38 ] Completed replacing temp checkpoint with checkpoint 33.217 ms, 1.60 s total +[ 2023-09-21 07:06:38 ] Completed eval batch 1 to device 384.403 ms, 1.99 s total +[ 2023-09-21 07:06:39 ] Completed eval batch 1 forward 36.944 ms, 2.02 s total +[ 2023-09-21 07:06:39 ] Completed eval batch 1 recons_loss 0.389 ms, 2.02 s total +[ 2023-09-21 07:06:39 ] Completed eval batch 1 metrics update 192.669 ms, 2.22 s total +Saving checkpoint at epoch 44 val batch 1 +Epoch 44 val loss: 0.0733 +[ 2023-09-21 07:06:39 ] Completed saving temp checkpoint 594.753 ms, 2.81 s total +[ 2023-09-21 07:06:39 ] Completed replacing temp checkpoint with checkpoint 23.059 ms, 2.83 s total +[ 2023-09-21 07:06:39 ] Completed evaluating generator for epoch 44 63.419 ms, 2.90 s total + + +EPOCH :: 45 + + +[ 2023-09-21 07:06:39 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 45 from training step 0 + +[ 2023-09-21 07:06:42 ] Completed train batch 0 to device 3,088.746 ms, 3.09 s total +[ 2023-09-21 07:06:43 ] Completed train batch 0 generator forward 29.960 ms, 3.12 s total +[ 2023-09-21 07:06:43 ] Completed train batch 0 generator loss: 1.503 66.797 ms, 3.19 s total +[ 2023-09-21 07:06:43 ] Completed train batch 0 generator backward 342.728 ms, 3.53 s total +[ 2023-09-21 07:06:43 ] Completed train batch 0 discriminator loss 0.136 58.023 ms, 3.59 s total +[ 2023-09-21 07:06:43 ] Completed train batch 0 discriminator backward 106.709 ms, 3.69 s total +Epoch [45] Step [0/2], gen_loss: 0.506, disc_loss: 0.045 +[ 2023-09-21 07:06:43 ] Completed train batch 0 metrics update 8.739 ms, 3.70 s total +Saving checkpoint at epoch 45 train batch 0 +[ 2023-09-21 07:06:44 ] Completed saving temp checkpoint 420.765 ms, 4.12 s total +[ 2023-09-21 07:06:44 ] Completed replacing temp checkpoint with checkpoint 28.805 ms, 4.15 s total +[ 2023-09-21 07:06:46 ] Completed train batch 1 to device 2,115.938 ms, 6.27 s total +[ 2023-09-21 07:06:46 ] Completed train batch 1 generator forward 22.448 ms, 6.29 s total +[ 2023-09-21 07:06:46 ] Completed train batch 1 generator loss: 1.490 63.583 ms, 6.35 s total +[ 2023-09-21 07:06:46 ] Completed train batch 1 generator backward 608.564 ms, 6.96 s total +[ 2023-09-21 07:06:46 ] Completed train batch 1 discriminator loss 0.138 56.415 ms, 7.02 s total +[ 2023-09-21 07:06:46 ] Completed train batch 1 discriminator backward 74.148 ms, 7.09 s total +Epoch [45] Step [1/2], gen_loss: 0.498, disc_loss: 0.045 +[ 2023-09-21 07:06:46 ] Completed train batch 1 metrics update 4.994 ms, 7.10 s total +Saving checkpoint at epoch 45 train batch 1 +[ 2023-09-21 07:06:47 ] Completed saving temp checkpoint 430.568 ms, 7.53 s total +[ 2023-09-21 07:06:47 ] Completed replacing temp checkpoint with checkpoint 27.656 ms, 7.56 s total +Epoch [45] :: gen_loss: 0.502, disc_loss: 0.045 +[ 2023-09-21 07:06:47 ] Completed training generator for epoch 45 60.273 ms, 7.62 s total +[ 2023-09-21 07:06:47 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 45 from eval step 0 + +[ 2023-09-21 07:06:48 ] Completed eval batch 0 to device 1,134.429 ms, 1.13 s total +[ 2023-09-21 07:06:48 ] Completed eval batch 0 forward 26.363 ms, 1.16 s total +[ 2023-09-21 07:06:48 ] Completed eval batch 0 recons_loss 0.365 ms, 1.16 s total +[ 2023-09-21 07:06:48 ] Completed eval batch 0 metrics update 0.621 ms, 1.16 s total +Saving checkpoint at epoch 45 val batch 0 +[ 2023-09-21 07:06:49 ] Completed saving temp checkpoint 377.951 ms, 1.54 s total +[ 2023-09-21 07:06:49 ] Completed replacing temp checkpoint with checkpoint 24.105 ms, 1.56 s total +[ 2023-09-21 07:06:49 ] Completed eval batch 1 to device 523.387 ms, 2.09 s total +[ 2023-09-21 07:06:49 ] Completed eval batch 1 forward 39.404 ms, 2.13 s total +[ 2023-09-21 07:06:49 ] Completed eval batch 1 recons_loss 0.343 ms, 2.13 s total +[ 2023-09-21 07:06:49 ] Completed eval batch 1 metrics update 56.165 ms, 2.18 s total +Saving checkpoint at epoch 45 val batch 1 +Epoch 45 val loss: 0.0631 +[ 2023-09-21 07:06:50 ] Completed saving temp checkpoint 604.707 ms, 2.79 s total +[ 2023-09-21 07:06:50 ] Completed replacing temp checkpoint with checkpoint 26.340 ms, 2.81 s total +[ 2023-09-21 07:06:50 ] Completed evaluating generator for epoch 45 67.911 ms, 2.88 s total + + +EPOCH :: 46 + + +[ 2023-09-21 07:06:50 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 46 from training step 0 + +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 07:17:39 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 07:17:39 ] Completed importing Timer 0.025 ms, 0.00 s total +[ 2023-09-21 07:17:44 ] Completed importing everything else 4,968.759 ms, 4.97 s total +| distributed init (rank 1): env:// +| distributed init (rank 4): env:// +| distributed init (rank 5): env:// +| distributed init (rank 3): env:// +| distributed init (rank 0): env:// +| distributed init (rank 2): env:// +[ 2023-09-21 07:17:52 ] Completed preliminaries 7,570.606 ms, 12.54 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 07:17:52 ] Completed build datasets 11.481 ms, 12.55 s total +[ 2023-09-21 07:17:52 ] Completed build samplers 0.044 ms, 12.55 s total +[ 2023-09-21 07:17:52 ] Completed build dataloaders 0.129 ms, 12.55 s total +[ 2023-09-21 07:17:52 ] Completed generator to device 77.785 ms, 12.63 s total +[ 2023-09-21 07:17:52 ] Completed discriminator to device 7.040 ms, 12.64 s total +[ 2023-09-21 07:17:52 ] Completed loss functions 401.550 ms, 13.04 s total +[ 2023-09-21 07:17:52 ] Completed models prepped for distribution 99.585 ms, 13.14 s total +[ 2023-09-21 07:17:52 ] Completed optimizers 0.682 ms, 13.14 s total +[ 2023-09-21 07:17:52 ] Completed grad scalers 0.037 ms, 13.14 s total +[ 2023-09-21 07:17:53 ] Completed checkpoint retrieval 503.083 ms, 13.64 s total + + +EPOCH :: 45 + + +[ 2023-09-21 07:17:53 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 45 from training step 2 + +Epoch [45] :: gen_loss: 0.502, disc_loss: 0.045 +[ 2023-09-21 07:17:53 ] Completed training generator for epoch 45 112.079 ms, 0.11 s total +[ 2023-09-21 07:17:53 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 45 from eval step 2 + +[ 2023-09-21 07:17:53 ] Completed evaluating generator for epoch 45 119.907 ms, 0.12 s total + + +EPOCH :: 46 + + +[ 2023-09-21 07:17:53 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 46 from training step 0 + +[ 2023-09-21 07:17:57 ] Completed train batch 0 to device 3,582.667 ms, 3.58 s total +[ 2023-09-21 07:17:58 ] Completed train batch 0 generator forward 1,551.750 ms, 5.13 s total +[ 2023-09-21 07:17:58 ] Completed train batch 0 generator loss: 1.505 81.797 ms, 5.22 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 07:18:00 ] Completed train batch 0 generator backward 1,424.238 ms, 6.64 s total +[ 2023-09-21 07:18:00 ] Completed train batch 0 discriminator loss 0.131 55.395 ms, 6.70 s total +[ 2023-09-21 07:18:00 ] Completed train batch 0 discriminator backward 101.391 ms, 6.80 s total +Epoch [46] Step [0/2], gen_loss: 0.497, disc_loss: 0.044 +[ 2023-09-21 07:18:00 ] Completed train batch 0 metrics update 6.389 ms, 6.80 s total +Saving checkpoint at epoch 46 train batch 0 +[ 2023-09-21 07:18:00 ] Completed saving temp checkpoint 444.336 ms, 7.25 s total +[ 2023-09-21 07:18:00 ] Completed replacing temp checkpoint with checkpoint 125.664 ms, 7.37 s total +[ 2023-09-21 07:18:00 ] Completed train batch 1 to device 21.719 ms, 7.40 s total +[ 2023-09-21 07:18:00 ] Completed train batch 1 generator forward 36.451 ms, 7.43 s total +[ 2023-09-21 07:18:01 ] Completed train batch 1 generator loss: 1.556 62.211 ms, 7.49 s total +[ 2023-09-21 07:18:02 ] Completed train batch 1 generator backward 1,315.460 ms, 8.81 s total +[ 2023-09-21 07:18:02 ] Completed train batch 1 discriminator loss 0.115 55.774 ms, 8.87 s total +[ 2023-09-21 07:18:02 ] Completed train batch 1 discriminator backward 71.695 ms, 8.94 s total +Epoch [46] Step [1/2], gen_loss: 0.509, disc_loss: 0.038 +[ 2023-09-21 07:18:02 ] Completed train batch 1 metrics update 0.990 ms, 8.94 s total +Saving checkpoint at epoch 46 train batch 1 +[ 2023-09-21 07:18:02 ] Completed saving temp checkpoint 446.320 ms, 9.38 s total +[ 2023-09-21 07:18:02 ] Completed replacing temp checkpoint with checkpoint 25.707 ms, 9.41 s total +Epoch [46] :: gen_loss: 0.503, disc_loss: 0.041 +[ 2023-09-21 07:18:02 ] Completed training generator for epoch 46 37.110 ms, 9.45 s total +[ 2023-09-21 07:18:02 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 46 from eval step 0 + +[ 2023-09-21 07:18:04 ] Completed eval batch 0 to device 1,133.503 ms, 1.13 s total +[ 2023-09-21 07:18:04 ] Completed eval batch 0 forward 24.993 ms, 1.16 s total +[ 2023-09-21 07:18:04 ] Completed eval batch 0 recons_loss 0.339 ms, 1.16 s total +[ 2023-09-21 07:18:04 ] Completed eval batch 0 metrics update 0.614 ms, 1.16 s total +Saving checkpoint at epoch 46 val batch 0 +[ 2023-09-21 07:18:04 ] Completed saving temp checkpoint 380.225 ms, 1.54 s total +[ 2023-09-21 07:18:04 ] Completed replacing temp checkpoint with checkpoint 28.548 ms, 1.57 s total +[ 2023-09-21 07:18:04 ] Completed eval batch 1 to device 426.021 ms, 1.99 s total +[ 2023-09-21 07:18:05 ] Completed eval batch 1 forward 40.887 ms, 2.04 s total +[ 2023-09-21 07:18:05 ] Completed eval batch 1 recons_loss 0.498 ms, 2.04 s total +[ 2023-09-21 07:18:05 ] Completed eval batch 1 metrics update 193.611 ms, 2.23 s total +Saving checkpoint at epoch 46 val batch 1 +Epoch 46 val loss: 0.0676 +[ 2023-09-21 07:18:05 ] Completed saving temp checkpoint 427.385 ms, 2.66 s total +[ 2023-09-21 07:18:05 ] Completed replacing temp checkpoint with checkpoint 23.389 ms, 2.68 s total +[ 2023-09-21 07:18:05 ] Completed evaluating generator for epoch 46 47.534 ms, 2.73 s total + + +EPOCH :: 47 + + +[ 2023-09-21 07:18:05 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 47 from training step 0 + +[ 2023-09-21 07:18:08 ] Completed train batch 0 to device 3,171.520 ms, 3.17 s total +[ 2023-09-21 07:18:08 ] Completed train batch 0 generator forward 27.453 ms, 3.20 s total +[ 2023-09-21 07:18:08 ] Completed train batch 0 generator loss: 1.459 67.114 ms, 3.27 s total +[ 2023-09-21 07:18:09 ] Completed train batch 0 generator backward 662.138 ms, 3.93 s total +[ 2023-09-21 07:18:09 ] Completed train batch 0 discriminator loss 0.135 57.341 ms, 3.99 s total +[ 2023-09-21 07:18:09 ] Completed train batch 0 discriminator backward 112.715 ms, 4.10 s total +Epoch [47] Step [0/2], gen_loss: 0.498, disc_loss: 0.045 +[ 2023-09-21 07:18:09 ] Completed train batch 0 metrics update 3.998 ms, 4.10 s total +Saving checkpoint at epoch 47 train batch 0 +[ 2023-09-21 07:18:10 ] Completed saving temp checkpoint 423.948 ms, 4.53 s total +[ 2023-09-21 07:18:10 ] Completed replacing temp checkpoint with checkpoint 29.372 ms, 4.56 s total +[ 2023-09-21 07:18:11 ] Completed train batch 1 to device 1,505.213 ms, 6.06 s total +[ 2023-09-21 07:18:11 ] Completed train batch 1 generator forward 39.398 ms, 6.10 s total +[ 2023-09-21 07:18:11 ] Completed train batch 1 generator loss: 1.515 67.652 ms, 6.17 s total +[ 2023-09-21 07:18:13 ] Completed train batch 1 generator backward 1,239.204 ms, 7.41 s total +[ 2023-09-21 07:18:13 ] Completed train batch 1 discriminator loss 0.131 55.819 ms, 7.46 s total +[ 2023-09-21 07:18:13 ] Completed train batch 1 discriminator backward 79.449 ms, 7.54 s total +Epoch [47] Step [1/2], gen_loss: 0.497, disc_loss: 0.044 +[ 2023-09-21 07:18:13 ] Completed train batch 1 metrics update 0.880 ms, 7.54 s total +Saving checkpoint at epoch 47 train batch 1 +[ 2023-09-21 07:18:13 ] Completed saving temp checkpoint 477.477 ms, 8.02 s total +[ 2023-09-21 07:18:13 ] Completed replacing temp checkpoint with checkpoint 31.007 ms, 8.05 s total +Epoch [47] :: gen_loss: 0.497, disc_loss: 0.045 +[ 2023-09-21 07:18:13 ] Completed training generator for epoch 47 52.838 ms, 8.10 s total +[ 2023-09-21 07:18:13 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 47 from eval step 0 + +[ 2023-09-21 07:18:14 ] Completed eval batch 0 to device 1,158.970 ms, 1.16 s total +[ 2023-09-21 07:18:15 ] Completed eval batch 0 forward 43.657 ms, 1.20 s total +[ 2023-09-21 07:18:15 ] Completed eval batch 0 recons_loss 0.369 ms, 1.20 s total +[ 2023-09-21 07:18:15 ] Completed eval batch 0 metrics update 0.862 ms, 1.20 s total +Saving checkpoint at epoch 47 val batch 0 +[ 2023-09-21 07:18:15 ] Completed saving temp checkpoint 388.957 ms, 1.59 s total +[ 2023-09-21 07:18:15 ] Completed replacing temp checkpoint with checkpoint 18.538 ms, 1.61 s total +[ 2023-09-21 07:18:15 ] Completed eval batch 1 to device 460.485 ms, 2.07 s total +[ 2023-09-21 07:18:15 ] Completed eval batch 1 forward 37.754 ms, 2.11 s total +[ 2023-09-21 07:18:15 ] Completed eval batch 1 recons_loss 0.343 ms, 2.11 s total +[ 2023-09-21 07:18:16 ] Completed eval batch 1 metrics update 166.735 ms, 2.28 s total +Saving checkpoint at epoch 47 val batch 1 +Epoch 47 val loss: 0.0661 +[ 2023-09-21 07:18:16 ] Completed saving temp checkpoint 445.364 ms, 2.72 s total +[ 2023-09-21 07:18:16 ] Completed replacing temp checkpoint with checkpoint 21.592 ms, 2.74 s total +[ 2023-09-21 07:18:16 ] Completed evaluating generator for epoch 47 45.692 ms, 2.79 s total + + +EPOCH :: 48 + + +[ 2023-09-21 07:18:16 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 48 from training step 0 + +[ 2023-09-21 07:18:19 ] Completed train batch 0 to device 3,198.751 ms, 3.20 s total +[ 2023-09-21 07:18:19 ] Completed train batch 0 generator forward 28.901 ms, 3.23 s total +[ 2023-09-21 07:18:19 ] Completed train batch 0 generator loss: 1.539 66.530 ms, 3.29 s total +[ 2023-09-21 07:18:20 ] Completed train batch 0 generator backward 685.345 ms, 3.98 s total +[ 2023-09-21 07:18:20 ] Completed train batch 0 discriminator loss 0.129 56.877 ms, 4.04 s total +[ 2023-09-21 07:18:20 ] Completed train batch 0 discriminator backward 116.290 ms, 4.15 s total +Epoch [48] Step [0/2], gen_loss: 0.494, disc_loss: 0.042 +[ 2023-09-21 07:18:20 ] Completed train batch 0 metrics update 5.067 ms, 4.16 s total +Saving checkpoint at epoch 48 train batch 0 +[ 2023-09-21 07:18:21 ] Completed saving temp checkpoint 431.673 ms, 4.59 s total +[ 2023-09-21 07:18:21 ] Completed replacing temp checkpoint with checkpoint 30.364 ms, 4.62 s total +[ 2023-09-21 07:18:22 ] Completed train batch 1 to device 1,746.709 ms, 6.37 s total +[ 2023-09-21 07:18:23 ] Completed train batch 1 generator forward 40.592 ms, 6.41 s total +[ 2023-09-21 07:18:23 ] Completed train batch 1 generator loss: 1.543 63.809 ms, 6.47 s total +[ 2023-09-21 07:18:24 ] Completed train batch 1 generator backward 977.688 ms, 7.45 s total +[ 2023-09-21 07:18:24 ] Completed train batch 1 discriminator loss 0.125 61.987 ms, 7.51 s total +[ 2023-09-21 07:18:24 ] Completed train batch 1 discriminator backward 65.804 ms, 7.58 s total +Epoch [48] Step [1/2], gen_loss: 0.499, disc_loss: 0.039 +[ 2023-09-21 07:18:24 ] Completed train batch 1 metrics update 0.577 ms, 7.58 s total +Saving checkpoint at epoch 48 train batch 1 +[ 2023-09-21 07:18:24 ] Completed saving temp checkpoint 686.801 ms, 8.26 s total +[ 2023-09-21 07:18:24 ] Completed replacing temp checkpoint with checkpoint 23.168 ms, 8.29 s total +Epoch [48] :: gen_loss: 0.497, disc_loss: 0.041 +[ 2023-09-21 07:18:24 ] Completed training generator for epoch 48 49.808 ms, 8.34 s total +[ 2023-09-21 07:18:24 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 48 from eval step 0 + +[ 2023-09-21 07:18:26 ] Completed eval batch 0 to device 1,076.696 ms, 1.08 s total +[ 2023-09-21 07:18:26 ] Completed eval batch 0 forward 37.554 ms, 1.11 s total +[ 2023-09-21 07:18:26 ] Completed eval batch 0 recons_loss 0.365 ms, 1.11 s total +[ 2023-09-21 07:18:26 ] Completed eval batch 0 metrics update 0.736 ms, 1.12 s total +Saving checkpoint at epoch 48 val batch 0 +[ 2023-09-21 07:18:26 ] Completed saving temp checkpoint 521.569 ms, 1.64 s total +[ 2023-09-21 07:18:26 ] Completed replacing temp checkpoint with checkpoint 24.817 ms, 1.66 s total +[ 2023-09-21 07:18:26 ] Completed eval batch 1 to device 270.174 ms, 1.93 s total +[ 2023-09-21 07:18:26 ] Completed eval batch 1 forward 38.461 ms, 1.97 s total +[ 2023-09-21 07:18:26 ] Completed eval batch 1 recons_loss 0.355 ms, 1.97 s total +[ 2023-09-21 07:18:27 ] Completed eval batch 1 metrics update 101.511 ms, 2.07 s total +Saving checkpoint at epoch 48 val batch 1 +Epoch 48 val loss: 0.0680 +[ 2023-09-21 07:18:27 ] Completed saving temp checkpoint 453.318 ms, 2.53 s total +[ 2023-09-21 07:18:27 ] Completed replacing temp checkpoint with checkpoint 23.442 ms, 2.55 s total +[ 2023-09-21 07:18:27 ] Completed evaluating generator for epoch 48 46.677 ms, 2.60 s total + + +EPOCH :: 49 + + +[ 2023-09-21 07:18:27 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 49 from training step 0 + +[ 2023-09-21 07:18:30 ] Completed train batch 0 to device 3,243.125 ms, 3.24 s total +[ 2023-09-21 07:18:30 ] Completed train batch 0 generator forward 30.685 ms, 3.27 s total +[ 2023-09-21 07:18:30 ] Completed train batch 0 generator loss: 1.524 65.423 ms, 3.34 s total +[ 2023-09-21 07:18:31 ] Completed train batch 0 generator backward 513.364 ms, 3.85 s total +[ 2023-09-21 07:18:31 ] Completed train batch 0 discriminator loss 0.114 56.994 ms, 3.91 s total +[ 2023-09-21 07:18:31 ] Completed train batch 0 discriminator backward 102.851 ms, 4.01 s total +Epoch [49] Step [0/2], gen_loss: 0.503, disc_loss: 0.038 +[ 2023-09-21 07:18:31 ] Completed train batch 0 metrics update 7.010 ms, 4.02 s total +Saving checkpoint at epoch 49 train batch 0 +[ 2023-09-21 07:18:31 ] Completed saving temp checkpoint 442.037 ms, 4.46 s total +[ 2023-09-21 07:18:32 ] Completed replacing temp checkpoint with checkpoint 22.113 ms, 4.48 s total +[ 2023-09-21 07:18:33 ] Completed train batch 1 to device 1,926.042 ms, 6.41 s total +[ 2023-09-21 07:18:33 ] Completed train batch 1 generator forward 40.658 ms, 6.45 s total +[ 2023-09-21 07:18:34 ] Completed train batch 1 generator loss: 1.434 63.699 ms, 6.51 s total +[ 2023-09-21 07:18:34 ] Completed train batch 1 generator backward 717.045 ms, 7.23 s total +[ 2023-09-21 07:18:34 ] Completed train batch 1 discriminator loss 0.117 55.496 ms, 7.29 s total +[ 2023-09-21 07:18:34 ] Completed train batch 1 discriminator backward 82.731 ms, 7.37 s total +Epoch [49] Step [1/2], gen_loss: 0.495, disc_loss: 0.039 +[ 2023-09-21 07:18:34 ] Completed train batch 1 metrics update 2.122 ms, 7.37 s total +Saving checkpoint at epoch 49 train batch 1 +[ 2023-09-21 07:18:35 ] Completed saving temp checkpoint 440.995 ms, 7.81 s total +[ 2023-09-21 07:18:35 ] Completed replacing temp checkpoint with checkpoint 23.117 ms, 7.84 s total +Epoch [49] :: gen_loss: 0.499, disc_loss: 0.039 +[ 2023-09-21 07:18:35 ] Completed training generator for epoch 49 49.593 ms, 7.89 s total +[ 2023-09-21 07:18:35 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 49 from eval step 0 + +[ 2023-09-21 07:18:36 ] Completed eval batch 0 to device 1,158.535 ms, 1.16 s total +[ 2023-09-21 07:18:36 ] Completed eval batch 0 forward 43.990 ms, 1.20 s total +[ 2023-09-21 07:18:36 ] Completed eval batch 0 recons_loss 0.308 ms, 1.20 s total +[ 2023-09-21 07:18:36 ] Completed eval batch 0 metrics update 0.590 ms, 1.20 s total +Saving checkpoint at epoch 49 val batch 0 +[ 2023-09-21 07:18:37 ] Completed saving temp checkpoint 451.297 ms, 1.65 s total +[ 2023-09-21 07:18:37 ] Completed replacing temp checkpoint with checkpoint 34.371 ms, 1.69 s total +[ 2023-09-21 07:18:37 ] Completed eval batch 1 to device 376.646 ms, 2.07 s total +[ 2023-09-21 07:18:37 ] Completed eval batch 1 forward 39.838 ms, 2.11 s total +[ 2023-09-21 07:18:37 ] Completed eval batch 1 recons_loss 0.581 ms, 2.11 s total +[ 2023-09-21 07:18:37 ] Completed eval batch 1 metrics update 76.644 ms, 2.18 s total +Saving checkpoint at epoch 49 val batch 1 +Epoch 49 val loss: 0.0695 +[ 2023-09-21 07:18:38 ] Completed saving temp checkpoint 489.921 ms, 2.67 s total +[ 2023-09-21 07:18:38 ] Completed replacing temp checkpoint with checkpoint 26.766 ms, 2.70 s total +[ 2023-09-21 07:18:38 ] Completed evaluating generator for epoch 49 50.867 ms, 2.75 s total + + +EPOCH :: 50 + + +[ 2023-09-21 07:18:38 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 50 from training step 0 + +[ 2023-09-21 07:18:41 ] Completed train batch 0 to device 3,091.808 ms, 3.09 s total +[ 2023-09-21 07:18:41 ] Completed train batch 0 generator forward 43.706 ms, 3.14 s total +[ 2023-09-21 07:18:41 ] Completed train batch 0 generator loss: 1.412 66.672 ms, 3.20 s total +[ 2023-09-21 07:18:41 ] Completed train batch 0 generator backward 570.518 ms, 3.77 s total +[ 2023-09-21 07:18:41 ] Completed train batch 0 discriminator loss 0.114 56.686 ms, 3.83 s total +[ 2023-09-21 07:18:42 ] Completed train batch 0 discriminator backward 115.430 ms, 3.94 s total +Epoch [50] Step [0/2], gen_loss: 0.492, disc_loss: 0.038 +[ 2023-09-21 07:18:42 ] Completed train batch 0 metrics update 21.686 ms, 3.97 s total +Saving checkpoint at epoch 50 train batch 0 +[ 2023-09-21 07:18:42 ] Completed saving temp checkpoint 462.535 ms, 4.43 s total +[ 2023-09-21 07:18:42 ] Completed replacing temp checkpoint with checkpoint 24.433 ms, 4.45 s total +[ 2023-09-21 07:18:44 ] Completed train batch 1 to device 1,519.181 ms, 5.97 s total +[ 2023-09-21 07:18:44 ] Completed train batch 1 generator forward 22.244 ms, 5.99 s total +[ 2023-09-21 07:18:44 ] Completed train batch 1 generator loss: 1.486 64.842 ms, 6.06 s total +[ 2023-09-21 07:18:45 ] Completed train batch 1 generator backward 1,323.729 ms, 7.38 s total +[ 2023-09-21 07:18:45 ] Completed train batch 1 discriminator loss 0.105 55.889 ms, 7.44 s total +[ 2023-09-21 07:18:45 ] Completed train batch 1 discriminator backward 81.862 ms, 7.52 s total +Epoch [50] Step [1/2], gen_loss: 0.492, disc_loss: 0.036 +[ 2023-09-21 07:18:45 ] Completed train batch 1 metrics update 3.746 ms, 7.52 s total +Saving checkpoint at epoch 50 train batch 1 +[ 2023-09-21 07:18:46 ] Completed saving temp checkpoint 507.008 ms, 8.03 s total +[ 2023-09-21 07:18:46 ] Completed replacing temp checkpoint with checkpoint 31.222 ms, 8.06 s total +Epoch [50] :: gen_loss: 0.492, disc_loss: 0.037 +[ 2023-09-21 07:18:46 ] Completed training generator for epoch 50 53.388 ms, 8.12 s total +[ 2023-09-21 07:18:46 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 50 from eval step 0 + +[ 2023-09-21 07:18:47 ] Completed eval batch 0 to device 1,143.930 ms, 1.14 s total +[ 2023-09-21 07:18:47 ] Completed eval batch 0 forward 40.634 ms, 1.18 s total +[ 2023-09-21 07:18:47 ] Completed eval batch 0 recons_loss 0.367 ms, 1.18 s total +[ 2023-09-21 07:18:47 ] Completed eval batch 0 metrics update 0.882 ms, 1.19 s total +Saving checkpoint at epoch 50 val batch 0 +[ 2023-09-21 07:18:47 ] Completed saving temp checkpoint 439.918 ms, 1.63 s total +[ 2023-09-21 07:18:47 ] Completed replacing temp checkpoint with checkpoint 35.674 ms, 1.66 s total +[ 2023-09-21 07:18:48 ] Completed eval batch 1 to device 344.780 ms, 2.01 s total +[ 2023-09-21 07:18:48 ] Completed eval batch 1 forward 22.965 ms, 2.03 s total +[ 2023-09-21 07:18:48 ] Completed eval batch 1 recons_loss 0.349 ms, 2.03 s total +[ 2023-09-21 07:18:48 ] Completed eval batch 1 metrics update 194.772 ms, 2.22 s total +Saving checkpoint at epoch 50 val batch 1 +Epoch 50 val loss: 0.0658 +[ 2023-09-21 07:18:48 ] Completed saving temp checkpoint 446.743 ms, 2.67 s total +[ 2023-09-21 07:18:48 ] Completed replacing temp checkpoint with checkpoint 24.729 ms, 2.70 s total +[ 2023-09-21 07:18:49 ] Completed evaluating generator for epoch 50 47.328 ms, 2.74 s total + + +EPOCH :: 51 + + +[ 2023-09-21 07:18:49 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 51 from training step 0 + +[ 2023-09-21 07:18:52 ] Completed train batch 0 to device 3,098.718 ms, 3.10 s total +[ 2023-09-21 07:18:52 ] Completed train batch 0 generator forward 27.786 ms, 3.13 s total +[ 2023-09-21 07:18:52 ] Completed train batch 0 generator loss: 1.465 67.238 ms, 3.19 s total +[ 2023-09-21 07:18:52 ] Completed train batch 0 generator backward 663.835 ms, 3.86 s total +[ 2023-09-21 07:18:52 ] Completed train batch 0 discriminator loss 0.101 57.713 ms, 3.92 s total +[ 2023-09-21 07:18:53 ] Completed train batch 0 discriminator backward 98.640 ms, 4.01 s total +Epoch [51] Step [0/2], gen_loss: 0.490, disc_loss: 0.034 +[ 2023-09-21 07:18:53 ] Completed train batch 0 metrics update 3.209 ms, 4.02 s total +Saving checkpoint at epoch 51 train batch 0 +[ 2023-09-21 07:18:53 ] Completed saving temp checkpoint 457.031 ms, 4.47 s total +[ 2023-09-21 07:18:53 ] Completed replacing temp checkpoint with checkpoint 25.404 ms, 4.50 s total +[ 2023-09-21 07:18:54 ] Completed train batch 1 to device 1,277.134 ms, 5.78 s total +[ 2023-09-21 07:18:54 ] Completed train batch 1 generator forward 22.341 ms, 5.80 s total +[ 2023-09-21 07:18:54 ] Completed train batch 1 generator loss: 1.461 64.777 ms, 5.86 s total +[ 2023-09-21 07:18:56 ] Completed train batch 1 generator backward 1,492.982 ms, 7.36 s total +[ 2023-09-21 07:18:56 ] Completed train batch 1 discriminator loss 0.102 55.982 ms, 7.41 s total +[ 2023-09-21 07:18:56 ] Completed train batch 1 discriminator backward 79.054 ms, 7.49 s total +Epoch [51] Step [1/2], gen_loss: 0.494, disc_loss: 0.034 +[ 2023-09-21 07:18:56 ] Completed train batch 1 metrics update 4.894 ms, 7.50 s total +Saving checkpoint at epoch 51 train batch 1 +[ 2023-09-21 07:18:56 ] Completed saving temp checkpoint 423.821 ms, 7.92 s total +[ 2023-09-21 07:18:56 ] Completed replacing temp checkpoint with checkpoint 19.746 ms, 7.94 s total +Epoch [51] :: gen_loss: 0.492, disc_loss: 0.034 +[ 2023-09-21 07:18:57 ] Completed training generator for epoch 51 52.641 ms, 7.99 s total +[ 2023-09-21 07:18:57 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 51 from eval step 0 + +[ 2023-09-21 07:18:58 ] Completed eval batch 0 to device 1,112.959 ms, 1.11 s total +[ 2023-09-21 07:18:58 ] Completed eval batch 0 forward 25.001 ms, 1.14 s total +[ 2023-09-21 07:18:58 ] Completed eval batch 0 recons_loss 0.288 ms, 1.14 s total +[ 2023-09-21 07:18:58 ] Completed eval batch 0 metrics update 6.023 ms, 1.14 s total +Saving checkpoint at epoch 51 val batch 0 +[ 2023-09-21 07:18:58 ] Completed saving temp checkpoint 328.733 ms, 1.47 s total +[ 2023-09-21 07:18:58 ] Completed replacing temp checkpoint with checkpoint 20.929 ms, 1.49 s total +[ 2023-09-21 07:18:59 ] Completed eval batch 1 to device 516.116 ms, 2.01 s total +[ 2023-09-21 07:18:59 ] Completed eval batch 1 forward 40.689 ms, 2.05 s total +[ 2023-09-21 07:18:59 ] Completed eval batch 1 recons_loss 0.516 ms, 2.05 s total +[ 2023-09-21 07:18:59 ] Completed eval batch 1 metrics update 338.164 ms, 2.39 s total +Saving checkpoint at epoch 51 val batch 1 +Epoch 51 val loss: 0.0606 +[ 2023-09-21 07:18:59 ] Completed saving temp checkpoint 370.937 ms, 2.76 s total +[ 2023-09-21 07:18:59 ] Completed replacing temp checkpoint with checkpoint 20.192 ms, 2.78 s total +[ 2023-09-21 07:18:59 ] Completed evaluating generator for epoch 51 46.069 ms, 2.83 s total + + +EPOCH :: 52 + + +[ 2023-09-21 07:18:59 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 52 from training step 0 + +[ 2023-09-21 07:19:03 ] Completed train batch 0 to device 3,186.127 ms, 3.19 s total +[ 2023-09-21 07:19:03 ] Completed train batch 0 generator forward 40.662 ms, 3.23 s total +[ 2023-09-21 07:19:03 ] Completed train batch 0 generator loss: 1.502 66.561 ms, 3.29 s total +[ 2023-09-21 07:19:03 ] Completed train batch 0 generator backward 632.934 ms, 3.93 s total +[ 2023-09-21 07:19:03 ] Completed train batch 0 discriminator loss 0.106 57.209 ms, 3.98 s total +[ 2023-09-21 07:19:03 ] Completed train batch 0 discriminator backward 100.358 ms, 4.08 s total +Epoch [52] Step [0/2], gen_loss: 0.491, disc_loss: 0.036 +[ 2023-09-21 07:19:03 ] Completed train batch 0 metrics update 1.102 ms, 4.08 s total +Saving checkpoint at epoch 52 train batch 0 +[ 2023-09-21 07:19:04 ] Completed saving temp checkpoint 490.269 ms, 4.58 s total +[ 2023-09-21 07:19:04 ] Completed replacing temp checkpoint with checkpoint 21.697 ms, 4.60 s total +[ 2023-09-21 07:19:05 ] Completed train batch 1 to device 1,395.165 ms, 5.99 s total +[ 2023-09-21 07:19:05 ] Completed train batch 1 generator forward 40.023 ms, 6.03 s total +[ 2023-09-21 07:19:05 ] Completed train batch 1 generator loss: 1.489 63.631 ms, 6.10 s total +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 07:31:22 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 07:31:22 ] Completed importing Timer 0.025 ms, 0.00 s total +[ 2023-09-21 07:31:27 ] Completed importing everything else 4,582.563 ms, 4.58 s total +| distributed init (rank 1): env:// +| distributed init (rank 2): env:// +| distributed init (rank 3): env:// +| distributed init (rank 0): env:// +| distributed init (rank 4): env:// +| distributed init (rank 5): env:// +[ 2023-09-21 07:31:34 ] Completed preliminaries 7,361.249 ms, 11.94 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 07:31:34 ] Completed build datasets 10.783 ms, 11.95 s total +[ 2023-09-21 07:31:34 ] Completed build samplers 0.044 ms, 11.95 s total +[ 2023-09-21 07:31:34 ] Completed build dataloaders 0.141 ms, 11.95 s total +[ 2023-09-21 07:31:34 ] Completed generator to device 80.119 ms, 12.03 s total +[ 2023-09-21 07:31:34 ] Completed discriminator to device 7.039 ms, 12.04 s total +[ 2023-09-21 07:31:35 ] Completed loss functions 412.549 ms, 12.45 s total +[ 2023-09-21 07:31:35 ] Completed models prepped for distribution 122.574 ms, 12.58 s total +[ 2023-09-21 07:31:35 ] Completed optimizers 0.694 ms, 12.58 s total +[ 2023-09-21 07:31:35 ] Completed grad scalers 0.032 ms, 12.58 s total +[ 2023-09-21 07:31:35 ] Completed checkpoint retrieval 501.205 ms, 13.08 s total + + +EPOCH :: 52 + + +[ 2023-09-21 07:31:35 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 52 from training step 1 + +[ 2023-09-21 07:31:39 ] Completed train batch 1 to device 3,715.823 ms, 3.72 s total +[ 2023-09-21 07:31:40 ] Completed train batch 1 generator forward 1,082.362 ms, 4.80 s total +[ 2023-09-21 07:31:40 ] Completed train batch 1 generator loss: 1.516 73.703 ms, 4.87 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 07:31:41 ] Completed train batch 1 generator backward 1,041.506 ms, 5.91 s total +[ 2023-09-21 07:31:41 ] Completed train batch 1 discriminator loss 0.102 72.417 ms, 5.99 s total +[ 2023-09-21 07:31:41 ] Completed train batch 1 discriminator backward 67.522 ms, 6.05 s total +Epoch [52] Step [1/2], gen_loss: 0.486, disc_loss: 0.034 +[ 2023-09-21 07:31:41 ] Completed train batch 1 metrics update 5.455 ms, 6.06 s total +Saving checkpoint at epoch 52 train batch 1 +[ 2023-09-21 07:31:42 ] Completed saving temp checkpoint 427.516 ms, 6.49 s total +[ 2023-09-21 07:31:42 ] Completed replacing temp checkpoint with checkpoint 119.743 ms, 6.61 s total +Epoch [52] :: gen_loss: 0.488, disc_loss: 0.035 +[ 2023-09-21 07:31:42 ] Completed training generator for epoch 52 55.153 ms, 6.66 s total +[ 2023-09-21 07:31:42 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 52 from eval step 0 + +[ 2023-09-21 07:31:43 ] Completed eval batch 0 to device 1,082.955 ms, 1.08 s total +[ 2023-09-21 07:31:43 ] Completed eval batch 0 forward 25.241 ms, 1.11 s total +[ 2023-09-21 07:31:43 ] Completed eval batch 0 recons_loss 0.293 ms, 1.11 s total +[ 2023-09-21 07:31:43 ] Completed eval batch 0 metrics update 0.638 ms, 1.11 s total +Saving checkpoint at epoch 52 val batch 0 +[ 2023-09-21 07:31:43 ] Completed saving temp checkpoint 453.675 ms, 1.56 s total +[ 2023-09-21 07:31:43 ] Completed replacing temp checkpoint with checkpoint 26.976 ms, 1.59 s total +[ 2023-09-21 07:31:44 ] Completed eval batch 1 to device 346.263 ms, 1.94 s total +[ 2023-09-21 07:31:44 ] Completed eval batch 1 forward 40.800 ms, 1.98 s total +[ 2023-09-21 07:31:44 ] Completed eval batch 1 recons_loss 0.502 ms, 1.98 s total +[ 2023-09-21 07:31:44 ] Completed eval batch 1 metrics update 185.617 ms, 2.16 s total +Saving checkpoint at epoch 52 val batch 1 +Epoch 52 val loss: 0.0605 +[ 2023-09-21 07:31:44 ] Completed saving temp checkpoint 430.936 ms, 2.59 s total +[ 2023-09-21 07:31:45 ] Completed replacing temp checkpoint with checkpoint 29.670 ms, 2.62 s total +[ 2023-09-21 07:31:45 ] Completed evaluating generator for epoch 52 47.591 ms, 2.67 s total + + +EPOCH :: 53 + + +[ 2023-09-21 07:31:45 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 53 from training step 0 + +[ 2023-09-21 07:31:48 ] Completed train batch 0 to device 3,057.031 ms, 3.06 s total +[ 2023-09-21 07:31:48 ] Completed train batch 0 generator forward 41.932 ms, 3.10 s total +[ 2023-09-21 07:31:48 ] Completed train batch 0 generator loss: 1.452 64.831 ms, 3.16 s total +[ 2023-09-21 07:31:49 ] Completed train batch 0 generator backward 799.631 ms, 3.96 s total +[ 2023-09-21 07:31:49 ] Completed train batch 0 discriminator loss 0.097 56.349 ms, 4.02 s total +[ 2023-09-21 07:31:49 ] Completed train batch 0 discriminator backward 111.225 ms, 4.13 s total +Epoch [53] Step [0/2], gen_loss: 0.480, disc_loss: 0.032 +[ 2023-09-21 07:31:49 ] Completed train batch 0 metrics update 6.817 ms, 4.14 s total +Saving checkpoint at epoch 53 train batch 0 +[ 2023-09-21 07:31:49 ] Completed saving temp checkpoint 362.368 ms, 4.50 s total +[ 2023-09-21 07:31:49 ] Completed replacing temp checkpoint with checkpoint 25.178 ms, 4.53 s total +[ 2023-09-21 07:31:50 ] Completed train batch 1 to device 1,186.412 ms, 5.71 s total +[ 2023-09-21 07:31:50 ] Completed train batch 1 generator forward 39.419 ms, 5.75 s total +[ 2023-09-21 07:31:50 ] Completed train batch 1 generator loss: 1.461 63.212 ms, 5.81 s total +[ 2023-09-21 07:31:52 ] Completed train batch 1 generator backward 1,574.853 ms, 7.39 s total +[ 2023-09-21 07:31:52 ] Completed train batch 1 discriminator loss 0.077 56.092 ms, 7.45 s total +[ 2023-09-21 07:31:52 ] Completed train batch 1 discriminator backward 83.733 ms, 7.53 s total +Epoch [53] Step [1/2], gen_loss: 0.491, disc_loss: 0.027 +[ 2023-09-21 07:31:52 ] Completed train batch 1 metrics update 4.809 ms, 7.53 s total +Saving checkpoint at epoch 53 train batch 1 +[ 2023-09-21 07:31:52 ] Completed saving temp checkpoint 355.193 ms, 7.89 s total +[ 2023-09-21 07:31:52 ] Completed replacing temp checkpoint with checkpoint 26.752 ms, 7.92 s total +Epoch [53] :: gen_loss: 0.485, disc_loss: 0.029 +[ 2023-09-21 07:31:53 ] Completed training generator for epoch 53 50.006 ms, 7.97 s total +[ 2023-09-21 07:31:53 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 53 from eval step 0 + +[ 2023-09-21 07:31:54 ] Completed eval batch 0 to device 1,161.867 ms, 1.16 s total +[ 2023-09-21 07:31:54 ] Completed eval batch 0 forward 26.562 ms, 1.19 s total +[ 2023-09-21 07:31:54 ] Completed eval batch 0 recons_loss 0.282 ms, 1.19 s total +[ 2023-09-21 07:31:54 ] Completed eval batch 0 metrics update 21.130 ms, 1.21 s total +Saving checkpoint at epoch 53 val batch 0 +[ 2023-09-21 07:31:54 ] Completed saving temp checkpoint 354.099 ms, 1.56 s total +[ 2023-09-21 07:31:54 ] Completed replacing temp checkpoint with checkpoint 21.856 ms, 1.59 s total +[ 2023-09-21 07:31:55 ] Completed eval batch 1 to device 507.408 ms, 2.09 s total +[ 2023-09-21 07:31:55 ] Completed eval batch 1 forward 26.756 ms, 2.12 s total +[ 2023-09-21 07:31:55 ] Completed eval batch 1 recons_loss 0.339 ms, 2.12 s total +[ 2023-09-21 07:31:55 ] Completed eval batch 1 metrics update 344.941 ms, 2.47 s total +Saving checkpoint at epoch 53 val batch 1 +Epoch 53 val loss: 0.0632 +[ 2023-09-21 07:31:55 ] Completed saving temp checkpoint 423.668 ms, 2.89 s total +[ 2023-09-21 07:31:55 ] Completed replacing temp checkpoint with checkpoint 31.304 ms, 2.92 s total +[ 2023-09-21 07:31:56 ] Completed evaluating generator for epoch 53 47.242 ms, 2.97 s total + + +EPOCH :: 54 + + +[ 2023-09-21 07:31:56 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 54 from training step 0 + +[ 2023-09-21 07:31:59 ] Completed train batch 0 to device 3,236.233 ms, 3.24 s total +[ 2023-09-21 07:31:59 ] Completed train batch 0 generator forward 42.852 ms, 3.28 s total +[ 2023-09-21 07:31:59 ] Completed train batch 0 generator loss: 1.436 66.407 ms, 3.35 s total +[ 2023-09-21 07:31:59 ] Completed train batch 0 generator backward 597.975 ms, 3.94 s total +[ 2023-09-21 07:32:00 ] Completed train batch 0 discriminator loss 0.088 57.759 ms, 4.00 s total +[ 2023-09-21 07:32:00 ] Completed train batch 0 discriminator backward 100.365 ms, 4.10 s total +Epoch [54] Step [0/2], gen_loss: 0.473, disc_loss: 0.030 +[ 2023-09-21 07:32:00 ] Completed train batch 0 metrics update 6.141 ms, 4.11 s total +Saving checkpoint at epoch 54 train batch 0 +[ 2023-09-21 07:32:00 ] Completed saving temp checkpoint 466.110 ms, 4.57 s total +[ 2023-09-21 07:32:00 ] Completed replacing temp checkpoint with checkpoint 17.600 ms, 4.59 s total +[ 2023-09-21 07:32:02 ] Completed train batch 1 to device 1,562.122 ms, 6.15 s total +[ 2023-09-21 07:32:02 ] Completed train batch 1 generator forward 22.465 ms, 6.18 s total +[ 2023-09-21 07:32:02 ] Completed train batch 1 generator loss: 1.433 63.437 ms, 6.24 s total +[ 2023-09-21 07:32:03 ] Completed train batch 1 generator backward 1,351.680 ms, 7.59 s total +[ 2023-09-21 07:32:03 ] Completed train batch 1 discriminator loss 0.092 56.155 ms, 7.65 s total +[ 2023-09-21 07:32:03 ] Completed train batch 1 discriminator backward 84.751 ms, 7.73 s total +Epoch [54] Step [1/2], gen_loss: 0.482, disc_loss: 0.031 +[ 2023-09-21 07:32:03 ] Completed train batch 1 metrics update 5.064 ms, 7.74 s total +Saving checkpoint at epoch 54 train batch 1 +[ 2023-09-21 07:32:04 ] Completed saving temp checkpoint 476.878 ms, 8.21 s total +[ 2023-09-21 07:32:04 ] Completed replacing temp checkpoint with checkpoint 21.124 ms, 8.24 s total +Epoch [54] :: gen_loss: 0.478, disc_loss: 0.031 +[ 2023-09-21 07:32:04 ] Completed training generator for epoch 54 53.952 ms, 8.29 s total +[ 2023-09-21 07:32:04 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 54 from eval step 0 + +[ 2023-09-21 07:32:05 ] Completed eval batch 0 to device 1,174.192 ms, 1.17 s total +[ 2023-09-21 07:32:05 ] Completed eval batch 0 forward 32.634 ms, 1.21 s total +[ 2023-09-21 07:32:05 ] Completed eval batch 0 recons_loss 0.350 ms, 1.21 s total +[ 2023-09-21 07:32:05 ] Completed eval batch 0 metrics update 0.605 ms, 1.21 s total +Saving checkpoint at epoch 54 val batch 0 +[ 2023-09-21 07:32:05 ] Completed saving temp checkpoint 464.844 ms, 1.67 s total +[ 2023-09-21 07:32:06 ] Completed replacing temp checkpoint with checkpoint 27.932 ms, 1.70 s total +[ 2023-09-21 07:32:06 ] Completed eval batch 1 to device 360.109 ms, 2.06 s total +[ 2023-09-21 07:32:06 ] Completed eval batch 1 forward 27.371 ms, 2.09 s total +[ 2023-09-21 07:32:06 ] Completed eval batch 1 recons_loss 0.358 ms, 2.09 s total +[ 2023-09-21 07:32:06 ] Completed eval batch 1 metrics update 113.961 ms, 2.20 s total +Saving checkpoint at epoch 54 val batch 1 +Epoch 54 val loss: 0.0662 +[ 2023-09-21 07:32:06 ] Completed saving temp checkpoint 497.844 ms, 2.70 s total +[ 2023-09-21 07:32:07 ] Completed replacing temp checkpoint with checkpoint 28.102 ms, 2.73 s total +[ 2023-09-21 07:32:07 ] Completed evaluating generator for epoch 54 49.991 ms, 2.78 s total + + +EPOCH :: 55 + + +[ 2023-09-21 07:32:07 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 55 from training step 0 + +[ 2023-09-21 07:32:10 ] Completed train batch 0 to device 3,367.543 ms, 3.37 s total +[ 2023-09-21 07:32:10 ] Completed train batch 0 generator forward 30.456 ms, 3.40 s total +[ 2023-09-21 07:32:10 ] Completed train batch 0 generator loss: 1.432 67.141 ms, 3.47 s total +[ 2023-09-21 07:32:10 ] Completed train batch 0 generator backward 272.765 ms, 3.74 s total +[ 2023-09-21 07:32:10 ] Completed train batch 0 discriminator loss 0.108 57.625 ms, 3.80 s total +[ 2023-09-21 07:32:10 ] Completed train batch 0 discriminator backward 117.722 ms, 3.91 s total +Epoch [55] Step [0/2], gen_loss: 0.474, disc_loss: 0.035 +[ 2023-09-21 07:32:10 ] Completed train batch 0 metrics update 0.888 ms, 3.91 s total +Saving checkpoint at epoch 55 train batch 0 +[ 2023-09-21 07:32:11 ] Completed saving temp checkpoint 480.805 ms, 4.39 s total +[ 2023-09-21 07:32:11 ] Completed replacing temp checkpoint with checkpoint 32.019 ms, 4.43 s total +[ 2023-09-21 07:32:13 ] Completed train batch 1 to device 2,091.206 ms, 6.52 s total +[ 2023-09-21 07:32:13 ] Completed train batch 1 generator forward 40.803 ms, 6.56 s total +[ 2023-09-21 07:32:13 ] Completed train batch 1 generator loss: 1.413 63.515 ms, 6.62 s total +[ 2023-09-21 07:32:14 ] Completed train batch 1 generator backward 602.175 ms, 7.22 s total +[ 2023-09-21 07:32:14 ] Completed train batch 1 discriminator loss 0.102 63.794 ms, 7.29 s total +[ 2023-09-21 07:32:14 ] Completed train batch 1 discriminator backward 72.864 ms, 7.36 s total +Epoch [55] Step [1/2], gen_loss: 0.484, disc_loss: 0.034 +[ 2023-09-21 07:32:14 ] Completed train batch 1 metrics update 4.747 ms, 7.37 s total +Saving checkpoint at epoch 55 train batch 1 +[ 2023-09-21 07:32:14 ] Completed saving temp checkpoint 494.559 ms, 7.86 s total +[ 2023-09-21 07:32:14 ] Completed replacing temp checkpoint with checkpoint 29.797 ms, 7.89 s total +Epoch [55] :: gen_loss: 0.479, disc_loss: 0.035 +[ 2023-09-21 07:32:15 ] Completed training generator for epoch 55 52.486 ms, 7.94 s total +[ 2023-09-21 07:32:15 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 55 from eval step 0 + +[ 2023-09-21 07:32:16 ] Completed eval batch 0 to device 1,179.965 ms, 1.18 s total +[ 2023-09-21 07:32:16 ] Completed eval batch 0 forward 30.921 ms, 1.21 s total +[ 2023-09-21 07:32:16 ] Completed eval batch 0 recons_loss 0.356 ms, 1.21 s total +[ 2023-09-21 07:32:16 ] Completed eval batch 0 metrics update 0.714 ms, 1.21 s total +Saving checkpoint at epoch 55 val batch 0 +[ 2023-09-21 07:32:16 ] Completed saving temp checkpoint 450.441 ms, 1.66 s total +[ 2023-09-21 07:32:16 ] Completed replacing temp checkpoint with checkpoint 25.264 ms, 1.69 s total +[ 2023-09-21 07:32:17 ] Completed eval batch 1 to device 429.650 ms, 2.12 s total +[ 2023-09-21 07:32:17 ] Completed eval batch 1 forward 21.403 ms, 2.14 s total +[ 2023-09-21 07:32:17 ] Completed eval batch 1 recons_loss 0.333 ms, 2.14 s total +[ 2023-09-21 07:32:17 ] Completed eval batch 1 metrics update 66.810 ms, 2.21 s total +Saving checkpoint at epoch 55 val batch 1 +Epoch 55 val loss: 0.0626 +[ 2023-09-21 07:32:17 ] Completed saving temp checkpoint 465.885 ms, 2.67 s total +[ 2023-09-21 07:32:17 ] Completed replacing temp checkpoint with checkpoint 22.262 ms, 2.69 s total +[ 2023-09-21 07:32:17 ] Completed evaluating generator for epoch 55 47.652 ms, 2.74 s total + + +EPOCH :: 56 + + +[ 2023-09-21 07:32:17 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 56 from training step 0 + +[ 2023-09-21 07:32:21 ] Completed train batch 0 to device 3,291.007 ms, 3.29 s total +[ 2023-09-21 07:32:21 ] Completed train batch 0 generator forward 26.777 ms, 3.32 s total +[ 2023-09-21 07:32:21 ] Completed train batch 0 generator loss: 1.442 66.833 ms, 3.38 s total +[ 2023-09-21 07:32:21 ] Completed train batch 0 generator backward 439.205 ms, 3.82 s total +[ 2023-09-21 07:32:21 ] Completed train batch 0 discriminator loss 0.096 79.973 ms, 3.90 s total +[ 2023-09-21 07:32:21 ] Completed train batch 0 discriminator backward 75.560 ms, 3.98 s total +Epoch [56] Step [0/2], gen_loss: 0.480, disc_loss: 0.032 +[ 2023-09-21 07:32:21 ] Completed train batch 0 metrics update 3.804 ms, 3.98 s total +Saving checkpoint at epoch 56 train batch 0 +[ 2023-09-21 07:32:22 ] Completed saving temp checkpoint 492.375 ms, 4.48 s total +[ 2023-09-21 07:32:22 ] Completed replacing temp checkpoint with checkpoint 18.551 ms, 4.49 s total +[ 2023-09-21 07:32:24 ] Completed train batch 1 to device 1,837.028 ms, 6.33 s total +[ 2023-09-21 07:32:24 ] Completed train batch 1 generator forward 23.946 ms, 6.36 s total +[ 2023-09-21 07:32:24 ] Completed train batch 1 generator loss: 1.440 65.893 ms, 6.42 s total +[ 2023-09-21 07:32:25 ] Completed train batch 1 generator backward 934.150 ms, 7.36 s total +[ 2023-09-21 07:32:25 ] Completed train batch 1 discriminator loss 0.087 56.890 ms, 7.41 s total +[ 2023-09-21 07:32:25 ] Completed train batch 1 discriminator backward 79.433 ms, 7.49 s total +Epoch [56] Step [1/2], gen_loss: 0.480, disc_loss: 0.030 +[ 2023-09-21 07:32:25 ] Completed train batch 1 metrics update 0.494 ms, 7.49 s total +Saving checkpoint at epoch 56 train batch 1 +[ 2023-09-21 07:32:25 ] Completed saving temp checkpoint 455.991 ms, 7.95 s total +[ 2023-09-21 07:32:25 ] Completed replacing temp checkpoint with checkpoint 20.489 ms, 7.97 s total +Epoch [56] :: gen_loss: 0.480, disc_loss: 0.031 +[ 2023-09-21 07:32:25 ] Completed training generator for epoch 56 51.575 ms, 8.02 s total +[ 2023-09-21 07:32:25 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 56 from eval step 0 + +[ 2023-09-21 07:32:26 ] Completed eval batch 0 to device 1,175.478 ms, 1.18 s total +[ 2023-09-21 07:32:26 ] Completed eval batch 0 forward 29.751 ms, 1.21 s total +[ 2023-09-21 07:32:26 ] Completed eval batch 0 recons_loss 0.282 ms, 1.21 s total +[ 2023-09-21 07:32:26 ] Completed eval batch 0 metrics update 0.579 ms, 1.21 s total +Saving checkpoint at epoch 56 val batch 0 +[ 2023-09-21 07:32:27 ] Completed saving temp checkpoint 445.294 ms, 1.65 s total +[ 2023-09-21 07:32:27 ] Completed replacing temp checkpoint with checkpoint 26.569 ms, 1.68 s total +[ 2023-09-21 07:32:28 ] Completed eval batch 1 to device 544.718 ms, 2.22 s total +[ 2023-09-21 07:32:28 ] Completed eval batch 1 forward 39.835 ms, 2.26 s total +[ 2023-09-21 07:32:28 ] Completed eval batch 1 recons_loss 0.430 ms, 2.26 s total +[ 2023-09-21 07:32:28 ] Completed eval batch 1 metrics update 0.394 ms, 2.26 s total +Saving checkpoint at epoch 56 val batch 1 +Epoch 56 val loss: 0.0629 +[ 2023-09-21 07:32:28 ] Completed saving temp checkpoint 470.549 ms, 2.73 s total +[ 2023-09-21 07:32:28 ] Completed replacing temp checkpoint with checkpoint 22.174 ms, 2.76 s total +[ 2023-09-21 07:32:28 ] Completed evaluating generator for epoch 56 50.576 ms, 2.81 s total + + +EPOCH :: 57 + + +[ 2023-09-21 07:32:28 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 57 from training step 0 + +[ 2023-09-21 07:32:31 ] Completed train batch 0 to device 3,220.045 ms, 3.22 s total +[ 2023-09-21 07:32:31 ] Completed train batch 0 generator forward 33.912 ms, 3.25 s total +[ 2023-09-21 07:32:31 ] Completed train batch 0 generator loss: 1.455 69.046 ms, 3.32 s total +[ 2023-09-21 07:32:32 ] Completed train batch 0 generator backward 439.825 ms, 3.76 s total +[ 2023-09-21 07:32:32 ] Completed train batch 0 discriminator loss 0.078 57.996 ms, 3.82 s total +[ 2023-09-21 07:32:32 ] Completed train batch 0 discriminator backward 105.143 ms, 3.93 s total +Epoch [57] Step [0/2], gen_loss: 0.474, disc_loss: 0.027 +[ 2023-09-21 07:32:32 ] Completed train batch 0 metrics update 22.851 ms, 3.95 s total +Saving checkpoint at epoch 57 train batch 0 +[ 2023-09-21 07:32:32 ] Completed saving temp checkpoint 422.642 ms, 4.37 s total +[ 2023-09-21 07:32:32 ] Completed replacing temp checkpoint with checkpoint 31.574 ms, 4.40 s total +[ 2023-09-21 07:32:34 ] Completed train batch 1 to device 1,794.883 ms, 6.20 s total +[ 2023-09-21 07:32:34 ] Completed train batch 1 generator forward 39.268 ms, 6.24 s total +[ 2023-09-21 07:32:34 ] Completed train batch 1 generator loss: 1.446 63.523 ms, 6.30 s total +[ 2023-09-21 07:32:35 ] Completed train batch 1 generator backward 891.363 ms, 7.19 s total +[ 2023-09-21 07:32:35 ] Completed train batch 1 discriminator loss 0.075 56.258 ms, 7.25 s total +[ 2023-09-21 07:32:35 ] Completed train batch 1 discriminator backward 73.408 ms, 7.32 s total +Epoch [57] Step [1/2], gen_loss: 0.479, disc_loss: 0.024 +[ 2023-09-21 07:32:35 ] Completed train batch 1 metrics update 5.492 ms, 7.33 s total +Saving checkpoint at epoch 57 train batch 1 +[ 2023-09-21 07:32:36 ] Completed saving temp checkpoint 450.072 ms, 7.78 s total +[ 2023-09-21 07:32:36 ] Completed replacing temp checkpoint with checkpoint 16.748 ms, 7.79 s total +Epoch [57] :: gen_loss: 0.476, disc_loss: 0.026 +[ 2023-09-21 07:32:36 ] Completed training generator for epoch 57 46.521 ms, 7.84 s total +[ 2023-09-21 07:32:36 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 57 from eval step 0 + +[ 2023-09-21 07:32:37 ] Completed eval batch 0 to device 1,157.684 ms, 1.16 s total +[ 2023-09-21 07:32:37 ] Completed eval batch 0 forward 42.979 ms, 1.20 s total +[ 2023-09-21 07:32:37 ] Completed eval batch 0 recons_loss 0.305 ms, 1.20 s total +[ 2023-09-21 07:32:37 ] Completed eval batch 0 metrics update 0.607 ms, 1.20 s total +Saving checkpoint at epoch 57 val batch 0 +[ 2023-09-21 07:32:37 ] Completed saving temp checkpoint 361.704 ms, 1.56 s total +[ 2023-09-21 07:32:38 ] Completed replacing temp checkpoint with checkpoint 23.432 ms, 1.59 s total +[ 2023-09-21 07:32:38 ] Completed eval batch 1 to device 477.549 ms, 2.06 s total +[ 2023-09-21 07:32:38 ] Completed eval batch 1 forward 38.800 ms, 2.10 s total +[ 2023-09-21 07:32:38 ] Completed eval batch 1 recons_loss 0.337 ms, 2.10 s total +[ 2023-09-21 07:32:38 ] Completed eval batch 1 metrics update 118.013 ms, 2.22 s total +Saving checkpoint at epoch 57 val batch 1 +Epoch 57 val loss: 0.0653 +[ 2023-09-21 07:32:39 ] Completed saving temp checkpoint 404.053 ms, 2.63 s total +[ 2023-09-21 07:32:39 ] Completed replacing temp checkpoint with checkpoint 19.728 ms, 2.65 s total +[ 2023-09-21 07:32:39 ] Completed evaluating generator for epoch 57 44.980 ms, 2.69 s total + + +EPOCH :: 58 + + +[ 2023-09-21 07:32:39 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 58 from training step 0 + +[ 2023-09-21 07:32:42 ] Completed train batch 0 to device 3,167.107 ms, 3.17 s total +[ 2023-09-21 07:32:42 ] Completed train batch 0 generator forward 42.651 ms, 3.21 s total +[ 2023-09-21 07:32:42 ] Completed train batch 0 generator loss: 1.480 66.479 ms, 3.28 s total +[ 2023-09-21 07:32:43 ] Completed train batch 0 generator backward 644.372 ms, 3.92 s total +[ 2023-09-21 07:32:43 ] Completed train batch 0 discriminator loss 0.068 58.494 ms, 3.98 s total +[ 2023-09-21 07:32:43 ] Completed train batch 0 discriminator backward 106.858 ms, 4.09 s total +Epoch [58] Step [0/2], gen_loss: 0.488, disc_loss: 0.022 +[ 2023-09-21 07:32:43 ] Completed train batch 0 metrics update 5.353 ms, 4.09 s total +Saving checkpoint at epoch 58 train batch 0 +[ 2023-09-21 07:32:43 ] Completed saving temp checkpoint 494.629 ms, 4.59 s total +[ 2023-09-21 07:32:43 ] Completed replacing temp checkpoint with checkpoint 27.751 ms, 4.61 s total +[ 2023-09-21 07:32:45 ] Completed train batch 1 to device 1,418.980 ms, 6.03 s total +[ 2023-09-21 07:32:45 ] Completed train batch 1 generator forward 26.940 ms, 6.06 s total +[ 2023-09-21 07:32:45 ] Completed train batch 1 generator loss: 1.406 63.258 ms, 6.12 s total +[ 2023-09-21 07:32:46 ] Completed train batch 1 generator backward 1,285.841 ms, 7.41 s total +[ 2023-09-21 07:32:46 ] Completed train batch 1 discriminator loss 0.080 55.964 ms, 7.46 s total +[ 2023-09-21 07:32:46 ] Completed train batch 1 discriminator backward 80.841 ms, 7.55 s total +Epoch [58] Step [1/2], gen_loss: 0.470, disc_loss: 0.027 +[ 2023-09-21 07:32:46 ] Completed train batch 1 metrics update 1.650 ms, 7.55 s total +Saving checkpoint at epoch 58 train batch 1 +[ 2023-09-21 07:32:47 ] Completed saving temp checkpoint 447.198 ms, 7.99 s total +[ 2023-09-21 07:32:47 ] Completed replacing temp checkpoint with checkpoint 22.292 ms, 8.02 s total +Epoch [58] :: gen_loss: 0.479, disc_loss: 0.024 +[ 2023-09-21 07:32:47 ] Completed training generator for epoch 58 51.166 ms, 8.07 s total +[ 2023-09-21 07:32:47 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 58 from eval step 0 + +[ 2023-09-21 07:32:48 ] Completed eval batch 0 to device 1,147.716 ms, 1.15 s total +[ 2023-09-21 07:32:48 ] Completed eval batch 0 forward 24.761 ms, 1.17 s total +[ 2023-09-21 07:32:48 ] Completed eval batch 0 recons_loss 0.324 ms, 1.17 s total +[ 2023-09-21 07:32:48 ] Completed eval batch 0 metrics update 0.565 ms, 1.17 s total +Saving checkpoint at epoch 58 val batch 0 +[ 2023-09-21 07:32:48 ] Completed saving temp checkpoint 456.028 ms, 1.63 s total +[ 2023-09-21 07:32:48 ] Completed replacing temp checkpoint with checkpoint 37.985 ms, 1.67 s total +[ 2023-09-21 07:32:49 ] Completed eval batch 1 to device 425.747 ms, 2.09 s total +[ 2023-09-21 07:32:49 ] Completed eval batch 1 forward 37.867 ms, 2.13 s total +[ 2023-09-21 07:32:49 ] Completed eval batch 1 recons_loss 0.333 ms, 2.13 s total +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 07:45:07 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 07:45:07 ] Completed importing Timer 0.023 ms, 0.00 s total +[ 2023-09-21 07:45:11 ] Completed importing everything else 4,701.407 ms, 4.70 s total +| distributed init (rank 0): env:// +| distributed init (rank 4): env:// +| distributed init (rank 1): env:// +| distributed init (rank 3): env:// +| distributed init (rank 5): env:// +| distributed init (rank 2): env:// +[ 2023-09-21 07:45:19 ] Completed preliminaries 7,421.430 ms, 12.12 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 07:45:19 ] Completed build datasets 11.123 ms, 12.13 s total +[ 2023-09-21 07:45:19 ] Completed build samplers 0.044 ms, 12.13 s total +[ 2023-09-21 07:45:19 ] Completed build dataloaders 0.135 ms, 12.13 s total +[ 2023-09-21 07:45:19 ] Completed generator to device 76.535 ms, 12.21 s total +[ 2023-09-21 07:45:19 ] Completed discriminator to device 7.310 ms, 12.22 s total +[ 2023-09-21 07:45:19 ] Completed loss functions 441.427 ms, 12.66 s total +[ 2023-09-21 07:45:19 ] Completed models prepped for distribution 92.613 ms, 12.75 s total +[ 2023-09-21 07:45:19 ] Completed optimizers 0.712 ms, 12.75 s total +[ 2023-09-21 07:45:19 ] Completed grad scalers 0.032 ms, 12.75 s total +[ 2023-09-21 07:45:20 ] Completed checkpoint retrieval 482.754 ms, 13.24 s total + + +EPOCH :: 58 + + +[ 2023-09-21 07:45:20 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 58 from training step 2 + +Epoch [58] :: gen_loss: 0.479, disc_loss: 0.024 +[ 2023-09-21 07:45:20 ] Completed training generator for epoch 58 134.396 ms, 0.13 s total +[ 2023-09-21 07:45:20 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 58 from eval step 1 + +[ 2023-09-21 07:45:21 ] Completed eval batch 1 to device 1,160.486 ms, 1.16 s total +[ 2023-09-21 07:45:22 ] Completed eval batch 1 forward 1,185.613 ms, 2.35 s total +[ 2023-09-21 07:45:22 ] Completed eval batch 1 recons_loss 0.266 ms, 2.35 s total +[ 2023-09-21 07:45:23 ] Completed eval batch 1 metrics update 686.940 ms, 3.03 s total +Saving checkpoint at epoch 58 val batch 1 +Epoch 58 val loss: 0.0666 +[ 2023-09-21 07:45:24 ] Completed saving temp checkpoint 458.228 ms, 3.49 s total +[ 2023-09-21 07:45:24 ] Completed replacing temp checkpoint with checkpoint 121.169 ms, 3.61 s total +[ 2023-09-21 07:45:24 ] Completed evaluating generator for epoch 58 41.033 ms, 3.65 s total + + +EPOCH :: 59 + + +[ 2023-09-21 07:45:24 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 59 from training step 0 + +[ 2023-09-21 07:45:27 ] Completed train batch 0 to device 2,990.165 ms, 2.99 s total +[ 2023-09-21 07:45:27 ] Completed train batch 0 generator forward 44.006 ms, 3.03 s total +[ 2023-09-21 07:45:27 ] Completed train batch 0 generator loss: 1.461 87.789 ms, 3.12 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 07:45:28 ] Completed train batch 0 generator backward 830.807 ms, 3.95 s total +[ 2023-09-21 07:45:28 ] Completed train batch 0 discriminator loss 0.078 59.709 ms, 4.01 s total +[ 2023-09-21 07:45:28 ] Completed train batch 0 discriminator backward 105.937 ms, 4.12 s total +Epoch [59] Step [0/2], gen_loss: 0.480, disc_loss: 0.027 +[ 2023-09-21 07:45:28 ] Completed train batch 0 metrics update 4.922 ms, 4.12 s total +Saving checkpoint at epoch 59 train batch 0 +[ 2023-09-21 07:45:28 ] Completed saving temp checkpoint 452.953 ms, 4.58 s total +[ 2023-09-21 07:45:28 ] Completed replacing temp checkpoint with checkpoint 20.875 ms, 4.60 s total +[ 2023-09-21 07:45:29 ] Completed train batch 1 to device 1,228.240 ms, 5.83 s total +[ 2023-09-21 07:45:30 ] Completed train batch 1 generator forward 39.438 ms, 5.86 s total +[ 2023-09-21 07:45:30 ] Completed train batch 1 generator loss: 1.463 66.931 ms, 5.93 s total +[ 2023-09-21 07:45:31 ] Completed train batch 1 generator backward 1,537.025 ms, 7.47 s total +[ 2023-09-21 07:45:31 ] Completed train batch 1 discriminator loss 0.083 56.339 ms, 7.53 s total +[ 2023-09-21 07:45:31 ] Completed train batch 1 discriminator backward 81.416 ms, 7.61 s total +Epoch [59] Step [1/2], gen_loss: 0.488, disc_loss: 0.028 +[ 2023-09-21 07:45:31 ] Completed train batch 1 metrics update 1.985 ms, 7.61 s total +Saving checkpoint at epoch 59 train batch 1 +[ 2023-09-21 07:45:32 ] Completed saving temp checkpoint 416.804 ms, 8.03 s total +[ 2023-09-21 07:45:32 ] Completed replacing temp checkpoint with checkpoint 18.221 ms, 8.04 s total +Epoch [59] :: gen_loss: 0.484, disc_loss: 0.027 +[ 2023-09-21 07:45:32 ] Completed training generator for epoch 59 55.324 ms, 8.10 s total +[ 2023-09-21 07:45:32 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 59 from eval step 0 + +[ 2023-09-21 07:45:33 ] Completed eval batch 0 to device 1,058.731 ms, 1.06 s total +[ 2023-09-21 07:45:33 ] Completed eval batch 0 forward 26.831 ms, 1.09 s total +[ 2023-09-21 07:45:33 ] Completed eval batch 0 recons_loss 0.240 ms, 1.09 s total +[ 2023-09-21 07:45:33 ] Completed eval batch 0 metrics update 88.189 ms, 1.17 s total +Saving checkpoint at epoch 59 val batch 0 +[ 2023-09-21 07:45:33 ] Completed saving temp checkpoint 545.752 ms, 1.72 s total +[ 2023-09-21 07:45:34 ] Completed replacing temp checkpoint with checkpoint 34.526 ms, 1.75 s total +[ 2023-09-21 07:45:34 ] Completed eval batch 1 to device 205.842 ms, 1.96 s total +[ 2023-09-21 07:45:34 ] Completed eval batch 1 forward 21.535 ms, 1.98 s total +[ 2023-09-21 07:45:34 ] Completed eval batch 1 recons_loss 0.399 ms, 1.98 s total +[ 2023-09-21 07:45:34 ] Completed eval batch 1 metrics update 446.929 ms, 2.43 s total +Saving checkpoint at epoch 59 val batch 1 +Epoch 59 val loss: 0.0712 +[ 2023-09-21 07:45:35 ] Completed saving temp checkpoint 808.282 ms, 3.24 s total +[ 2023-09-21 07:45:35 ] Completed replacing temp checkpoint with checkpoint 39.370 ms, 3.28 s total +[ 2023-09-21 07:45:35 ] Completed evaluating generator for epoch 59 51.388 ms, 3.33 s total + + +EPOCH :: 60 + + +[ 2023-09-21 07:45:35 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 60 from training step 0 + +[ 2023-09-21 07:45:38 ] Completed train batch 0 to device 3,226.665 ms, 3.23 s total +[ 2023-09-21 07:45:38 ] Completed train batch 0 generator forward 32.167 ms, 3.26 s total +[ 2023-09-21 07:45:38 ] Completed train batch 0 generator loss: 1.462 66.825 ms, 3.33 s total +[ 2023-09-21 07:45:39 ] Completed train batch 0 generator backward 217.508 ms, 3.54 s total +[ 2023-09-21 07:45:39 ] Completed train batch 0 discriminator loss 0.079 57.793 ms, 3.60 s total +[ 2023-09-21 07:45:39 ] Completed train batch 0 discriminator backward 104.056 ms, 3.71 s total +Epoch [60] Step [0/2], gen_loss: 0.478, disc_loss: 0.027 +[ 2023-09-21 07:45:39 ] Completed train batch 0 metrics update 5.776 ms, 3.71 s total +Saving checkpoint at epoch 60 train batch 0 +[ 2023-09-21 07:45:39 ] Completed saving temp checkpoint 543.796 ms, 4.25 s total +[ 2023-09-21 07:45:39 ] Completed replacing temp checkpoint with checkpoint 34.551 ms, 4.29 s total +[ 2023-09-21 07:45:41 ] Completed train batch 1 to device 1,707.821 ms, 6.00 s total +[ 2023-09-21 07:45:41 ] Completed train batch 1 generator forward 22.338 ms, 6.02 s total +[ 2023-09-21 07:45:41 ] Completed train batch 1 generator loss: 1.384 63.586 ms, 6.08 s total +[ 2023-09-21 07:45:42 ] Completed train batch 1 generator backward 1,107.085 ms, 7.19 s total +[ 2023-09-21 07:45:42 ] Completed train batch 1 discriminator loss 0.079 56.983 ms, 7.25 s total +[ 2023-09-21 07:45:42 ] Completed train batch 1 discriminator backward 72.179 ms, 7.32 s total +Epoch [60] Step [1/2], gen_loss: 0.474, disc_loss: 0.027 +[ 2023-09-21 07:45:42 ] Completed train batch 1 metrics update 5.178 ms, 7.32 s total +Saving checkpoint at epoch 60 train batch 1 +[ 2023-09-21 07:45:43 ] Completed saving temp checkpoint 528.335 ms, 7.85 s total +[ 2023-09-21 07:45:43 ] Completed replacing temp checkpoint with checkpoint 28.988 ms, 7.88 s total +Epoch [60] :: gen_loss: 0.476, disc_loss: 0.027 +[ 2023-09-21 07:45:43 ] Completed training generator for epoch 60 64.573 ms, 7.95 s total +[ 2023-09-21 07:45:43 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 60 from eval step 0 + +[ 2023-09-21 07:45:44 ] Completed eval batch 0 to device 1,140.787 ms, 1.14 s total +[ 2023-09-21 07:45:44 ] Completed eval batch 0 forward 25.814 ms, 1.17 s total +[ 2023-09-21 07:45:44 ] Completed eval batch 0 recons_loss 0.354 ms, 1.17 s total +[ 2023-09-21 07:45:44 ] Completed eval batch 0 metrics update 0.795 ms, 1.17 s total +Saving checkpoint at epoch 60 val batch 0 +[ 2023-09-21 07:45:45 ] Completed saving temp checkpoint 508.877 ms, 1.68 s total +[ 2023-09-21 07:45:45 ] Completed replacing temp checkpoint with checkpoint 22.637 ms, 1.70 s total +[ 2023-09-21 07:45:45 ] Completed eval batch 1 to device 339.669 ms, 2.04 s total +[ 2023-09-21 07:45:45 ] Completed eval batch 1 forward 39.374 ms, 2.08 s total +[ 2023-09-21 07:45:45 ] Completed eval batch 1 recons_loss 0.342 ms, 2.08 s total +[ 2023-09-21 07:45:45 ] Completed eval batch 1 metrics update 44.845 ms, 2.12 s total +Saving checkpoint at epoch 60 val batch 1 +Epoch 60 val loss: 0.0698 +[ 2023-09-21 07:45:46 ] Completed saving temp checkpoint 542.991 ms, 2.67 s total +[ 2023-09-21 07:45:46 ] Completed replacing temp checkpoint with checkpoint 20.076 ms, 2.69 s total +[ 2023-09-21 07:45:46 ] Completed evaluating generator for epoch 60 71.870 ms, 2.76 s total + + +EPOCH :: 61 + + +[ 2023-09-21 07:45:46 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 61 from training step 0 + +[ 2023-09-21 07:45:49 ] Completed train batch 0 to device 3,104.430 ms, 3.10 s total +[ 2023-09-21 07:45:49 ] Completed train batch 0 generator forward 26.351 ms, 3.13 s total +[ 2023-09-21 07:45:49 ] Completed train batch 0 generator loss: 1.411 66.235 ms, 3.20 s total +[ 2023-09-21 07:45:50 ] Completed train batch 0 generator backward 562.204 ms, 3.76 s total +[ 2023-09-21 07:45:50 ] Completed train batch 0 discriminator loss 0.080 58.874 ms, 3.82 s total +[ 2023-09-21 07:45:50 ] Completed train batch 0 discriminator backward 78.518 ms, 3.90 s total +Epoch [61] Step [0/2], gen_loss: 0.474, disc_loss: 0.026 +[ 2023-09-21 07:45:50 ] Completed train batch 0 metrics update 8.176 ms, 3.90 s total +Saving checkpoint at epoch 61 train batch 0 +[ 2023-09-21 07:45:50 ] Completed saving temp checkpoint 499.624 ms, 4.40 s total +[ 2023-09-21 07:45:50 ] Completed replacing temp checkpoint with checkpoint 33.839 ms, 4.44 s total +[ 2023-09-21 07:45:52 ] Completed train batch 1 to device 1,878.233 ms, 6.32 s total +[ 2023-09-21 07:45:52 ] Completed train batch 1 generator forward 40.009 ms, 6.36 s total +[ 2023-09-21 07:45:52 ] Completed train batch 1 generator loss: 1.403 63.335 ms, 6.42 s total +[ 2023-09-21 07:45:53 ] Completed train batch 1 generator backward 787.777 ms, 7.21 s total +[ 2023-09-21 07:45:53 ] Completed train batch 1 discriminator loss 0.074 57.788 ms, 7.27 s total +[ 2023-09-21 07:45:53 ] Completed train batch 1 discriminator backward 63.063 ms, 7.33 s total +Epoch [61] Step [1/2], gen_loss: 0.471, disc_loss: 0.025 +[ 2023-09-21 07:45:53 ] Completed train batch 1 metrics update 3.695 ms, 7.33 s total +Saving checkpoint at epoch 61 train batch 1 +[ 2023-09-21 07:45:54 ] Completed saving temp checkpoint 546.667 ms, 7.88 s total +[ 2023-09-21 07:45:54 ] Completed replacing temp checkpoint with checkpoint 22.648 ms, 7.90 s total +Epoch [61] :: gen_loss: 0.473, disc_loss: 0.025 +[ 2023-09-21 07:45:54 ] Completed training generator for epoch 61 60.812 ms, 7.96 s total +[ 2023-09-21 07:45:54 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 61 from eval step 0 + +[ 2023-09-21 07:45:55 ] Completed eval batch 0 to device 1,134.550 ms, 1.13 s total +[ 2023-09-21 07:45:55 ] Completed eval batch 0 forward 25.163 ms, 1.16 s total +[ 2023-09-21 07:45:55 ] Completed eval batch 0 recons_loss 0.338 ms, 1.16 s total +[ 2023-09-21 07:45:55 ] Completed eval batch 0 metrics update 0.724 ms, 1.16 s total +Saving checkpoint at epoch 61 val batch 0 +[ 2023-09-21 07:45:55 ] Completed saving temp checkpoint 502.528 ms, 1.66 s total +[ 2023-09-21 07:45:55 ] Completed replacing temp checkpoint with checkpoint 35.492 ms, 1.70 s total +[ 2023-09-21 07:45:56 ] Completed eval batch 1 to device 318.622 ms, 2.02 s total +[ 2023-09-21 07:45:56 ] Completed eval batch 1 forward 29.398 ms, 2.05 s total +[ 2023-09-21 07:45:56 ] Completed eval batch 1 recons_loss 0.340 ms, 2.05 s total +[ 2023-09-21 07:45:56 ] Completed eval batch 1 metrics update 144.972 ms, 2.19 s total +Saving checkpoint at epoch 61 val batch 1 +Epoch 61 val loss: 0.0690 +[ 2023-09-21 07:45:57 ] Completed saving temp checkpoint 612.369 ms, 2.80 s total +[ 2023-09-21 07:45:57 ] Completed replacing temp checkpoint with checkpoint 44.678 ms, 2.85 s total +[ 2023-09-21 07:45:57 ] Completed evaluating generator for epoch 61 53.910 ms, 2.90 s total + + +EPOCH :: 62 + + +[ 2023-09-21 07:45:57 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 62 from training step 0 + +[ 2023-09-21 07:46:00 ] Completed train batch 0 to device 3,155.076 ms, 3.16 s total +[ 2023-09-21 07:46:00 ] Completed train batch 0 generator forward 31.637 ms, 3.19 s total +[ 2023-09-21 07:46:00 ] Completed train batch 0 generator loss: 1.414 67.051 ms, 3.25 s total +[ 2023-09-21 07:46:00 ] Completed train batch 0 generator backward 477.182 ms, 3.73 s total +[ 2023-09-21 07:46:00 ] Completed train batch 0 discriminator loss 0.071 58.203 ms, 3.79 s total +[ 2023-09-21 07:46:01 ] Completed train batch 0 discriminator backward 105.591 ms, 3.89 s total +Epoch [62] Step [0/2], gen_loss: 0.473, disc_loss: 0.023 +[ 2023-09-21 07:46:01 ] Completed train batch 0 metrics update 5.719 ms, 3.90 s total +Saving checkpoint at epoch 62 train batch 0 +[ 2023-09-21 07:46:01 ] Completed saving temp checkpoint 786.041 ms, 4.69 s total +[ 2023-09-21 07:46:01 ] Completed replacing temp checkpoint with checkpoint 49.905 ms, 4.74 s total +[ 2023-09-21 07:46:03 ] Completed train batch 1 to device 1,574.301 ms, 6.31 s total +[ 2023-09-21 07:46:03 ] Completed train batch 1 generator forward 40.052 ms, 6.35 s total +[ 2023-09-21 07:46:03 ] Completed train batch 1 generator loss: 1.415 63.315 ms, 6.41 s total +[ 2023-09-21 07:46:04 ] Completed train batch 1 generator backward 685.567 ms, 7.10 s total +[ 2023-09-21 07:46:04 ] Completed train batch 1 discriminator loss 0.070 56.231 ms, 7.16 s total +[ 2023-09-21 07:46:04 ] Completed train batch 1 discriminator backward 80.251 ms, 7.24 s total +Epoch [62] Step [1/2], gen_loss: 0.472, disc_loss: 0.024 +[ 2023-09-21 07:46:04 ] Completed train batch 1 metrics update 5.489 ms, 7.24 s total +Saving checkpoint at epoch 62 train batch 1 +[ 2023-09-21 07:46:06 ] Completed saving temp checkpoint 1,710.586 ms, 8.95 s total +[ 2023-09-21 07:46:06 ] Completed replacing temp checkpoint with checkpoint 55.588 ms, 9.01 s total +Epoch [62] :: gen_loss: 0.472, disc_loss: 0.024 +[ 2023-09-21 07:46:06 ] Completed training generator for epoch 62 74.363 ms, 9.08 s total +[ 2023-09-21 07:46:06 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 62 from eval step 0 + +[ 2023-09-21 07:46:07 ] Completed eval batch 0 to device 989.026 ms, 0.99 s total +[ 2023-09-21 07:46:07 ] Completed eval batch 0 forward 42.185 ms, 1.03 s total +[ 2023-09-21 07:46:07 ] Completed eval batch 0 recons_loss 0.208 ms, 1.03 s total +[ 2023-09-21 07:46:07 ] Completed eval batch 0 metrics update 0.473 ms, 1.03 s total +Saving checkpoint at epoch 62 val batch 0 +[ 2023-09-21 07:46:08 ] Completed saving temp checkpoint 1,644.197 ms, 2.68 s total +[ 2023-09-21 07:46:08 ] Completed replacing temp checkpoint with checkpoint 61.229 ms, 2.74 s total +[ 2023-09-21 07:46:08 ] Completed eval batch 1 to device 16.731 ms, 2.75 s total +[ 2023-09-21 07:46:09 ] Completed eval batch 1 forward 21.193 ms, 2.78 s total +[ 2023-09-21 07:46:09 ] Completed eval batch 1 recons_loss 0.324 ms, 2.78 s total +[ 2023-09-21 07:46:09 ] Completed eval batch 1 metrics update 0.298 ms, 2.78 s total +Saving checkpoint at epoch 62 val batch 1 +Epoch 62 val loss: 0.0711 +[ 2023-09-21 07:46:10 ] Completed saving temp checkpoint 1,690.169 ms, 4.47 s total +[ 2023-09-21 07:46:10 ] Completed replacing temp checkpoint with checkpoint 44.283 ms, 4.51 s total +[ 2023-09-21 07:46:10 ] Completed evaluating generator for epoch 62 50.871 ms, 4.56 s total + + +EPOCH :: 63 + + +[ 2023-09-21 07:46:10 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 63 from training step 0 + +[ 2023-09-21 07:46:13 ] Completed train batch 0 to device 3,157.389 ms, 3.16 s total +[ 2023-09-21 07:46:13 ] Completed train batch 0 generator forward 34.018 ms, 3.19 s total +[ 2023-09-21 07:46:14 ] Completed train batch 0 generator loss: 1.409 66.339 ms, 3.26 s total +[ 2023-09-21 07:46:14 ] Completed train batch 0 generator backward 148.786 ms, 3.41 s total +[ 2023-09-21 07:46:14 ] Completed train batch 0 discriminator loss 0.069 58.824 ms, 3.47 s total +[ 2023-09-21 07:46:14 ] Completed train batch 0 discriminator backward 81.808 ms, 3.55 s total +Epoch [63] Step [0/2], gen_loss: 0.471, disc_loss: 0.023 +[ 2023-09-21 07:46:14 ] Completed train batch 0 metrics update 9.314 ms, 3.56 s total +Saving checkpoint at epoch 63 train batch 0 +[ 2023-09-21 07:46:16 ] Completed saving temp checkpoint 1,687.748 ms, 5.24 s total +[ 2023-09-21 07:46:16 ] Completed replacing temp checkpoint with checkpoint 64.970 ms, 5.31 s total +[ 2023-09-21 07:46:16 ] Completed train batch 1 to device 505.510 ms, 5.81 s total +[ 2023-09-21 07:46:16 ] Completed train batch 1 generator forward 24.161 ms, 5.84 s total +[ 2023-09-21 07:46:16 ] Completed train batch 1 generator loss: 1.418 63.298 ms, 5.90 s total +[ 2023-09-21 07:46:16 ] Completed train batch 1 generator backward 215.178 ms, 6.12 s total +[ 2023-09-21 07:46:16 ] Completed train batch 1 discriminator loss 0.067 56.222 ms, 6.17 s total +[ 2023-09-21 07:46:17 ] Completed train batch 1 discriminator backward 68.521 ms, 6.24 s total +Epoch [63] Step [1/2], gen_loss: 0.473, disc_loss: 0.023 +[ 2023-09-21 07:46:17 ] Completed train batch 1 metrics update 5.025 ms, 6.25 s total +Saving checkpoint at epoch 63 train batch 1 +[ 2023-09-21 07:46:18 ] Completed saving temp checkpoint 1,714.156 ms, 7.96 s total +[ 2023-09-21 07:46:18 ] Completed replacing temp checkpoint with checkpoint 47.317 ms, 8.01 s total +Epoch [63] :: gen_loss: 0.472, disc_loss: 0.023 +[ 2023-09-21 07:46:18 ] Completed training generator for epoch 63 55.577 ms, 8.06 s total +[ 2023-09-21 07:46:18 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 63 from eval step 0 + +[ 2023-09-21 07:46:19 ] Completed eval batch 0 to device 980.702 ms, 0.98 s total +[ 2023-09-21 07:46:19 ] Completed eval batch 0 forward 23.724 ms, 1.00 s total +[ 2023-09-21 07:46:19 ] Completed eval batch 0 recons_loss 0.216 ms, 1.00 s total +[ 2023-09-21 07:46:19 ] Completed eval batch 0 metrics update 0.486 ms, 1.01 s total +Saving checkpoint at epoch 63 val batch 0 +[ 2023-09-21 07:46:20 ] Completed saving temp checkpoint 1,067.760 ms, 2.07 s total +[ 2023-09-21 07:46:21 ] Completed replacing temp checkpoint with checkpoint 62.519 ms, 2.14 s total +[ 2023-09-21 07:46:21 ] Completed eval batch 1 to device 18.476 ms, 2.15 s total +[ 2023-09-21 07:46:21 ] Completed eval batch 1 forward 26.240 ms, 2.18 s total +[ 2023-09-21 07:46:21 ] Completed eval batch 1 recons_loss 0.331 ms, 2.18 s total +[ 2023-09-21 07:46:21 ] Completed eval batch 1 metrics update 0.307 ms, 2.18 s total +Saving checkpoint at epoch 63 val batch 1 +Epoch 63 val loss: 0.0709 +[ 2023-09-21 07:46:21 ] Completed saving temp checkpoint 348.221 ms, 2.53 s total +[ 2023-09-21 07:46:21 ] Completed replacing temp checkpoint with checkpoint 26.025 ms, 2.56 s total +[ 2023-09-21 07:46:21 ] Completed evaluating generator for epoch 63 49.744 ms, 2.60 s total + + +EPOCH :: 64 + + +[ 2023-09-21 07:46:21 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 64 from training step 0 + +[ 2023-09-21 07:46:24 ] Completed train batch 0 to device 3,190.575 ms, 3.19 s total +[ 2023-09-21 07:46:24 ] Completed train batch 0 generator forward 26.732 ms, 3.22 s total +[ 2023-09-21 07:46:24 ] Completed train batch 0 generator loss: 1.402 66.087 ms, 3.28 s total +[ 2023-09-21 07:46:25 ] Completed train batch 0 generator backward 705.525 ms, 3.99 s total +[ 2023-09-21 07:46:25 ] Completed train batch 0 discriminator loss 0.072 56.929 ms, 4.05 s total +[ 2023-09-21 07:46:25 ] Completed train batch 0 discriminator backward 97.464 ms, 4.14 s total +Epoch [64] Step [0/2], gen_loss: 0.478, disc_loss: 0.024 +[ 2023-09-21 07:46:25 ] Completed train batch 0 metrics update 3.122 ms, 4.15 s total +Saving checkpoint at epoch 64 train batch 0 +[ 2023-09-21 07:46:26 ] Completed saving temp checkpoint 381.033 ms, 4.53 s total +[ 2023-09-21 07:46:26 ] Completed replacing temp checkpoint with checkpoint 26.860 ms, 4.55 s total +[ 2023-09-21 07:46:27 ] Completed train batch 1 to device 1,587.706 ms, 6.14 s total +[ 2023-09-21 07:46:27 ] Completed train batch 1 generator forward 23.380 ms, 6.17 s total +[ 2023-09-21 07:46:27 ] Completed train batch 1 generator loss: 1.411 65.257 ms, 6.23 s total +[ 2023-09-21 07:46:29 ] Completed train batch 1 generator backward 1,317.843 ms, 7.55 s total +[ 2023-09-21 07:46:29 ] Completed train batch 1 discriminator loss 0.077 56.531 ms, 7.61 s total +[ 2023-09-21 07:46:29 ] Completed train batch 1 discriminator backward 88.860 ms, 7.69 s total +Epoch [64] Step [1/2], gen_loss: 0.466, disc_loss: 0.026 +[ 2023-09-21 07:46:29 ] Completed train batch 1 metrics update 5.524 ms, 7.70 s total +Saving checkpoint at epoch 64 train batch 1 +[ 2023-09-21 07:46:29 ] Completed saving temp checkpoint 359.173 ms, 8.06 s total +[ 2023-09-21 07:46:29 ] Completed replacing temp checkpoint with checkpoint 18.111 ms, 8.08 s total +Epoch [64] :: gen_loss: 0.472, disc_loss: 0.025 +[ 2023-09-21 07:46:29 ] Completed training generator for epoch 64 55.378 ms, 8.13 s total +[ 2023-09-21 07:46:29 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 64 from eval step 0 + +[ 2023-09-21 07:46:30 ] Completed eval batch 0 to device 1,195.489 ms, 1.20 s total +[ 2023-09-21 07:46:30 ] Completed eval batch 0 forward 31.788 ms, 1.23 s total +[ 2023-09-21 07:46:30 ] Completed eval batch 0 recons_loss 0.290 ms, 1.23 s total +[ 2023-09-21 07:46:30 ] Completed eval batch 0 metrics update 0.572 ms, 1.23 s total +Saving checkpoint at epoch 64 val batch 0 +[ 2023-09-21 07:46:31 ] Completed saving temp checkpoint 392.608 ms, 1.62 s total +[ 2023-09-21 07:46:31 ] Completed replacing temp checkpoint with checkpoint 19.662 ms, 1.64 s total +[ 2023-09-21 07:46:31 ] Completed eval batch 1 to device 626.789 ms, 2.27 s total +[ 2023-09-21 07:46:31 ] Completed eval batch 1 forward 39.927 ms, 2.31 s total +[ 2023-09-21 07:46:31 ] Completed eval batch 1 recons_loss 0.553 ms, 2.31 s total +[ 2023-09-21 07:46:31 ] Completed eval batch 1 metrics update 67.580 ms, 2.38 s total +Saving checkpoint at epoch 64 val batch 1 +Epoch 64 val loss: 0.0644 +[ 2023-09-21 07:46:32 ] Completed saving temp checkpoint 639.560 ms, 3.01 s total +[ 2023-09-21 07:46:32 ] Completed replacing temp checkpoint with checkpoint 28.445 ms, 3.04 s total +[ 2023-09-21 07:46:32 ] Completed evaluating generator for epoch 64 53.124 ms, 3.10 s total + + +EPOCH :: 65 + + +[ 2023-09-21 07:46:32 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 65 from training step 0 + +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 07:57:22 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 07:57:22 ] Completed importing Timer 0.024 ms, 0.00 s total +[ 2023-09-21 07:57:27 ] Completed importing everything else 4,755.735 ms, 4.76 s total +| distributed init (rank 1): env:// +| distributed init (rank 0): env:// +| distributed init (rank 2): env:// +| distributed init (rank 5): env:// +| distributed init (rank 4): env:// +| distributed init (rank 3): env:// +[ 2023-09-21 07:57:34 ] Completed preliminaries 7,242.016 ms, 12.00 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 07:57:34 ] Completed build datasets 11.795 ms, 12.01 s total +[ 2023-09-21 07:57:34 ] Completed build samplers 0.044 ms, 12.01 s total +[ 2023-09-21 07:57:34 ] Completed build dataloaders 0.133 ms, 12.01 s total +[ 2023-09-21 07:57:34 ] Completed generator to device 78.040 ms, 12.09 s total +[ 2023-09-21 07:57:34 ] Completed discriminator to device 7.159 ms, 12.09 s total +[ 2023-09-21 07:57:35 ] Completed loss functions 456.607 ms, 12.55 s total +[ 2023-09-21 07:57:35 ] Completed models prepped for distribution 88.907 ms, 12.64 s total +[ 2023-09-21 07:57:35 ] Completed optimizers 0.669 ms, 12.64 s total +[ 2023-09-21 07:57:35 ] Completed grad scalers 0.032 ms, 12.64 s total +[ 2023-09-21 07:57:36 ] Completed checkpoint retrieval 515.327 ms, 13.16 s total + + +EPOCH :: 64 + + +[ 2023-09-21 07:57:36 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 64 from training step 2 + +Epoch [64] :: gen_loss: 0.472, disc_loss: 0.025 +[ 2023-09-21 07:57:36 ] Completed training generator for epoch 64 126.429 ms, 0.13 s total +[ 2023-09-21 07:57:36 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 64 from eval step 2 + +[ 2023-09-21 07:57:36 ] Completed evaluating generator for epoch 64 123.818 ms, 0.12 s total + + +EPOCH :: 65 + + +[ 2023-09-21 07:57:36 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 65 from training step 0 + +[ 2023-09-21 07:57:40 ] Completed train batch 0 to device 3,924.223 ms, 3.92 s total +[ 2023-09-21 07:57:41 ] Completed train batch 0 generator forward 1,639.195 ms, 5.56 s total +[ 2023-09-21 07:57:41 ] Completed train batch 0 generator loss: 1.421 78.163 ms, 5.64 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 07:57:42 ] Completed train batch 0 generator backward 535.484 ms, 6.18 s total +[ 2023-09-21 07:57:42 ] Completed train batch 0 discriminator loss 0.081 62.398 ms, 6.24 s total +[ 2023-09-21 07:57:42 ] Completed train batch 0 discriminator backward 88.723 ms, 6.33 s total +Epoch [65] Step [0/2], gen_loss: 0.463, disc_loss: 0.026 +[ 2023-09-21 07:57:42 ] Completed train batch 0 metrics update 4.782 ms, 6.33 s total +Saving checkpoint at epoch 65 train batch 0 +[ 2023-09-21 07:57:43 ] Completed saving temp checkpoint 410.913 ms, 6.74 s total +[ 2023-09-21 07:57:43 ] Completed replacing temp checkpoint with checkpoint 126.369 ms, 6.87 s total +[ 2023-09-21 07:57:43 ] Completed train batch 1 to device 255.134 ms, 7.13 s total +[ 2023-09-21 07:57:43 ] Completed train batch 1 generator forward 22.289 ms, 7.15 s total +[ 2023-09-21 07:57:43 ] Completed train batch 1 generator loss: 1.374 63.400 ms, 7.21 s total +[ 2023-09-21 07:57:44 ] Completed train batch 1 generator backward 1,153.316 ms, 8.36 s total +[ 2023-09-21 07:57:44 ] Completed train batch 1 discriminator loss 0.079 56.747 ms, 8.42 s total +[ 2023-09-21 07:57:44 ] Completed train batch 1 discriminator backward 69.588 ms, 8.49 s total +Epoch [65] Step [1/2], gen_loss: 0.460, disc_loss: 0.027 +[ 2023-09-21 07:57:44 ] Completed train batch 1 metrics update 1.912 ms, 8.49 s total +Saving checkpoint at epoch 65 train batch 1 +[ 2023-09-21 07:57:45 ] Completed saving temp checkpoint 442.456 ms, 8.94 s total +[ 2023-09-21 07:57:45 ] Completed replacing temp checkpoint with checkpoint 18.870 ms, 8.95 s total +Epoch [65] :: gen_loss: 0.461, disc_loss: 0.027 +[ 2023-09-21 07:57:45 ] Completed training generator for epoch 65 38.525 ms, 8.99 s total +[ 2023-09-21 07:57:45 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 65 from eval step 0 + +[ 2023-09-21 07:57:46 ] Completed eval batch 0 to device 1,167.462 ms, 1.17 s total +[ 2023-09-21 07:57:46 ] Completed eval batch 0 forward 36.284 ms, 1.20 s total +[ 2023-09-21 07:57:46 ] Completed eval batch 0 recons_loss 0.272 ms, 1.20 s total +[ 2023-09-21 07:57:46 ] Completed eval batch 0 metrics update 0.561 ms, 1.20 s total +Saving checkpoint at epoch 65 val batch 0 +[ 2023-09-21 07:57:46 ] Completed saving temp checkpoint 468.114 ms, 1.67 s total +[ 2023-09-21 07:57:46 ] Completed replacing temp checkpoint with checkpoint 30.477 ms, 1.70 s total +[ 2023-09-21 07:57:47 ] Completed eval batch 1 to device 449.251 ms, 2.15 s total +[ 2023-09-21 07:57:47 ] Completed eval batch 1 forward 40.457 ms, 2.19 s total +[ 2023-09-21 07:57:47 ] Completed eval batch 1 recons_loss 0.665 ms, 2.19 s total +[ 2023-09-21 07:57:47 ] Completed eval batch 1 metrics update 84.523 ms, 2.28 s total +Saving checkpoint at epoch 65 val batch 1 +Epoch 65 val loss: 0.0624 +[ 2023-09-21 07:57:48 ] Completed saving temp checkpoint 466.139 ms, 2.74 s total +[ 2023-09-21 07:57:48 ] Completed replacing temp checkpoint with checkpoint 25.501 ms, 2.77 s total +[ 2023-09-21 07:57:48 ] Completed evaluating generator for epoch 65 51.879 ms, 2.82 s total + + +EPOCH :: 66 + + +[ 2023-09-21 07:57:48 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 66 from training step 0 + +[ 2023-09-21 07:57:51 ] Completed train batch 0 to device 3,071.725 ms, 3.07 s total +[ 2023-09-21 07:57:51 ] Completed train batch 0 generator forward 27.393 ms, 3.10 s total +[ 2023-09-21 07:57:51 ] Completed train batch 0 generator loss: 1.368 65.656 ms, 3.16 s total +[ 2023-09-21 07:57:52 ] Completed train batch 0 generator backward 802.108 ms, 3.97 s total +[ 2023-09-21 07:57:52 ] Completed train batch 0 discriminator loss 0.081 58.141 ms, 4.03 s total +[ 2023-09-21 07:57:52 ] Completed train batch 0 discriminator backward 87.451 ms, 4.11 s total +Epoch [66] Step [0/2], gen_loss: 0.452, disc_loss: 0.026 +[ 2023-09-21 07:57:52 ] Completed train batch 0 metrics update 5.418 ms, 4.12 s total +Saving checkpoint at epoch 66 train batch 0 +[ 2023-09-21 07:57:52 ] Completed saving temp checkpoint 430.365 ms, 4.55 s total +[ 2023-09-21 07:57:52 ] Completed replacing temp checkpoint with checkpoint 23.926 ms, 4.57 s total +[ 2023-09-21 07:57:54 ] Completed train batch 1 to device 1,373.882 ms, 5.95 s total +[ 2023-09-21 07:57:54 ] Completed train batch 1 generator forward 39.977 ms, 5.99 s total +[ 2023-09-21 07:57:54 ] Completed train batch 1 generator loss: 1.324 64.688 ms, 6.05 s total +[ 2023-09-21 07:57:55 ] Completed train batch 1 generator backward 1,565.706 ms, 7.62 s total +[ 2023-09-21 07:57:55 ] Completed train batch 1 discriminator loss 0.073 57.426 ms, 7.67 s total +[ 2023-09-21 07:57:55 ] Completed train batch 1 discriminator backward 90.599 ms, 7.76 s total +Epoch [66] Step [1/2], gen_loss: 0.458, disc_loss: 0.025 +[ 2023-09-21 07:57:55 ] Completed train batch 1 metrics update 5.459 ms, 7.77 s total +Saving checkpoint at epoch 66 train batch 1 +[ 2023-09-21 07:57:56 ] Completed saving temp checkpoint 389.900 ms, 8.16 s total +[ 2023-09-21 07:57:56 ] Completed replacing temp checkpoint with checkpoint 20.658 ms, 8.18 s total +Epoch [66] :: gen_loss: 0.455, disc_loss: 0.025 +[ 2023-09-21 07:57:56 ] Completed training generator for epoch 66 48.393 ms, 8.23 s total +[ 2023-09-21 07:57:56 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 66 from eval step 0 + +[ 2023-09-21 07:57:57 ] Completed eval batch 0 to device 1,189.272 ms, 1.19 s total +[ 2023-09-21 07:57:57 ] Completed eval batch 0 forward 26.415 ms, 1.22 s total +[ 2023-09-21 07:57:57 ] Completed eval batch 0 recons_loss 0.339 ms, 1.22 s total +[ 2023-09-21 07:57:57 ] Completed eval batch 0 metrics update 0.607 ms, 1.22 s total +Saving checkpoint at epoch 66 val batch 0 +[ 2023-09-21 07:57:57 ] Completed saving temp checkpoint 396.969 ms, 1.61 s total +[ 2023-09-21 07:57:57 ] Completed replacing temp checkpoint with checkpoint 23.353 ms, 1.64 s total +[ 2023-09-21 07:57:58 ] Completed eval batch 1 to device 630.587 ms, 2.27 s total +[ 2023-09-21 07:57:58 ] Completed eval batch 1 forward 21.279 ms, 2.29 s total +[ 2023-09-21 07:57:58 ] Completed eval batch 1 recons_loss 0.344 ms, 2.29 s total +[ 2023-09-21 07:57:58 ] Completed eval batch 1 metrics update 33.533 ms, 2.32 s total +Saving checkpoint at epoch 66 val batch 1 +Epoch 66 val loss: 0.0673 +[ 2023-09-21 07:57:59 ] Completed saving temp checkpoint 385.726 ms, 2.71 s total +[ 2023-09-21 07:57:59 ] Completed replacing temp checkpoint with checkpoint 19.797 ms, 2.73 s total +[ 2023-09-21 07:57:59 ] Completed evaluating generator for epoch 66 53.046 ms, 2.78 s total + + +EPOCH :: 67 + + +[ 2023-09-21 07:57:59 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 67 from training step 0 + +[ 2023-09-21 07:58:02 ] Completed train batch 0 to device 3,165.403 ms, 3.17 s total +[ 2023-09-21 07:58:02 ] Completed train batch 0 generator forward 41.164 ms, 3.21 s total +[ 2023-09-21 07:58:02 ] Completed train batch 0 generator loss: 1.308 90.277 ms, 3.30 s total +[ 2023-09-21 07:58:03 ] Completed train batch 0 generator backward 652.949 ms, 3.95 s total +[ 2023-09-21 07:58:03 ] Completed train batch 0 discriminator loss 0.094 58.870 ms, 4.01 s total +[ 2023-09-21 07:58:03 ] Completed train batch 0 discriminator backward 91.831 ms, 4.10 s total +Epoch [67] Step [0/2], gen_loss: 0.441, disc_loss: 0.031 +[ 2023-09-21 07:58:03 ] Completed train batch 0 metrics update 24.993 ms, 4.13 s total +Saving checkpoint at epoch 67 train batch 0 +[ 2023-09-21 07:58:03 ] Completed saving temp checkpoint 350.222 ms, 4.48 s total +[ 2023-09-21 07:58:03 ] Completed replacing temp checkpoint with checkpoint 19.123 ms, 4.49 s total +[ 2023-09-21 07:58:05 ] Completed train batch 1 to device 1,575.093 ms, 6.07 s total +[ 2023-09-21 07:58:05 ] Completed train batch 1 generator forward 38.875 ms, 6.11 s total +[ 2023-09-21 07:58:05 ] Completed train batch 1 generator loss: 1.315 63.681 ms, 6.17 s total +[ 2023-09-21 07:58:06 ] Completed train batch 1 generator backward 1,151.062 ms, 7.32 s total +[ 2023-09-21 07:58:06 ] Completed train batch 1 discriminator loss 0.130 56.855 ms, 7.38 s total +[ 2023-09-21 07:58:06 ] Completed train batch 1 discriminator backward 89.108 ms, 7.47 s total +Epoch [67] Step [1/2], gen_loss: 0.441, disc_loss: 0.043 +[ 2023-09-21 07:58:06 ] Completed train batch 1 metrics update 3.902 ms, 7.47 s total +Saving checkpoint at epoch 67 train batch 1 +[ 2023-09-21 07:58:07 ] Completed saving temp checkpoint 448.778 ms, 7.92 s total +[ 2023-09-21 07:58:07 ] Completed replacing temp checkpoint with checkpoint 20.324 ms, 7.94 s total +Epoch [67] :: gen_loss: 0.441, disc_loss: 0.037 +[ 2023-09-21 07:58:07 ] Completed training generator for epoch 67 48.801 ms, 7.99 s total +[ 2023-09-21 07:58:07 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 67 from eval step 0 + +[ 2023-09-21 07:58:08 ] Completed eval batch 0 to device 1,131.437 ms, 1.13 s total +[ 2023-09-21 07:58:08 ] Completed eval batch 0 forward 31.435 ms, 1.16 s total +[ 2023-09-21 07:58:08 ] Completed eval batch 0 recons_loss 0.317 ms, 1.16 s total +[ 2023-09-21 07:58:08 ] Completed eval batch 0 metrics update 0.594 ms, 1.16 s total +Saving checkpoint at epoch 67 val batch 0 +[ 2023-09-21 07:58:08 ] Completed saving temp checkpoint 423.934 ms, 1.59 s total +[ 2023-09-21 07:58:08 ] Completed replacing temp checkpoint with checkpoint 30.295 ms, 1.62 s total +[ 2023-09-21 07:58:09 ] Completed eval batch 1 to device 553.525 ms, 2.17 s total +[ 2023-09-21 07:58:09 ] Completed eval batch 1 forward 39.437 ms, 2.21 s total +[ 2023-09-21 07:58:09 ] Completed eval batch 1 recons_loss 0.335 ms, 2.21 s total +[ 2023-09-21 07:58:09 ] Completed eval batch 1 metrics update 161.407 ms, 2.37 s total +Saving checkpoint at epoch 67 val batch 1 +Epoch 67 val loss: 0.0727 +[ 2023-09-21 07:58:09 ] Completed saving temp checkpoint 481.884 ms, 2.85 s total +[ 2023-09-21 07:58:09 ] Completed replacing temp checkpoint with checkpoint 26.666 ms, 2.88 s total +[ 2023-09-21 07:58:10 ] Completed evaluating generator for epoch 67 47.122 ms, 2.93 s total + + +EPOCH :: 68 + + +[ 2023-09-21 07:58:10 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 68 from training step 0 + +[ 2023-09-21 07:58:13 ] Completed train batch 0 to device 3,201.496 ms, 3.20 s total +[ 2023-09-21 07:58:13 ] Completed train batch 0 generator forward 33.746 ms, 3.24 s total +[ 2023-09-21 07:58:13 ] Completed train batch 0 generator loss: 1.332 66.054 ms, 3.30 s total +[ 2023-09-21 07:58:13 ] Completed train batch 0 generator backward 531.775 ms, 3.83 s total +[ 2023-09-21 07:58:13 ] Completed train batch 0 discriminator loss 0.115 57.361 ms, 3.89 s total +[ 2023-09-21 07:58:14 ] Completed train batch 0 discriminator backward 114.072 ms, 4.00 s total +Epoch [68] Step [0/2], gen_loss: 0.438, disc_loss: 0.039 +[ 2023-09-21 07:58:14 ] Completed train batch 0 metrics update 0.990 ms, 4.01 s total +Saving checkpoint at epoch 68 train batch 0 +[ 2023-09-21 07:58:14 ] Completed saving temp checkpoint 524.974 ms, 4.53 s total +[ 2023-09-21 07:58:14 ] Completed replacing temp checkpoint with checkpoint 24.865 ms, 4.56 s total +[ 2023-09-21 07:58:16 ] Completed train batch 1 to device 1,746.830 ms, 6.30 s total +[ 2023-09-21 07:58:16 ] Completed train batch 1 generator forward 40.431 ms, 6.34 s total +[ 2023-09-21 07:58:16 ] Completed train batch 1 generator loss: 1.448 63.751 ms, 6.41 s total +[ 2023-09-21 07:58:17 ] Completed train batch 1 generator backward 868.392 ms, 7.27 s total +[ 2023-09-21 07:58:17 ] Completed train batch 1 discriminator loss 0.093 56.961 ms, 7.33 s total +[ 2023-09-21 07:58:17 ] Completed train batch 1 discriminator backward 71.342 ms, 7.40 s total +Epoch [68] Step [1/2], gen_loss: 0.458, disc_loss: 0.035 +[ 2023-09-21 07:58:17 ] Completed train batch 1 metrics update 3.233 ms, 7.41 s total +Saving checkpoint at epoch 68 train batch 1 +[ 2023-09-21 07:58:17 ] Completed saving temp checkpoint 510.881 ms, 7.92 s total +[ 2023-09-21 07:58:17 ] Completed replacing temp checkpoint with checkpoint 20.210 ms, 7.94 s total +Epoch [68] :: gen_loss: 0.448, disc_loss: 0.037 +[ 2023-09-21 07:58:18 ] Completed training generator for epoch 68 65.064 ms, 8.00 s total +[ 2023-09-21 07:58:18 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 68 from eval step 0 + +[ 2023-09-21 07:58:19 ] Completed eval batch 0 to device 1,071.219 ms, 1.07 s total +[ 2023-09-21 07:58:19 ] Completed eval batch 0 forward 23.817 ms, 1.10 s total +[ 2023-09-21 07:58:19 ] Completed eval batch 0 recons_loss 0.229 ms, 1.10 s total +[ 2023-09-21 07:58:19 ] Completed eval batch 0 metrics update 0.534 ms, 1.10 s total +Saving checkpoint at epoch 68 val batch 0 +[ 2023-09-21 07:58:19 ] Completed saving temp checkpoint 446.660 ms, 1.54 s total +[ 2023-09-21 07:58:19 ] Completed replacing temp checkpoint with checkpoint 36.085 ms, 1.58 s total +[ 2023-09-21 07:58:20 ] Completed eval batch 1 to device 508.880 ms, 2.09 s total +[ 2023-09-21 07:58:20 ] Completed eval batch 1 forward 21.433 ms, 2.11 s total +[ 2023-09-21 07:58:20 ] Completed eval batch 1 recons_loss 0.332 ms, 2.11 s total +[ 2023-09-21 07:58:20 ] Completed eval batch 1 metrics update 66.442 ms, 2.18 s total +Saving checkpoint at epoch 68 val batch 1 +Epoch 68 val loss: 0.0640 +[ 2023-09-21 07:58:20 ] Completed saving temp checkpoint 487.707 ms, 2.66 s total +[ 2023-09-21 07:58:20 ] Completed replacing temp checkpoint with checkpoint 21.877 ms, 2.69 s total +[ 2023-09-21 07:58:20 ] Completed evaluating generator for epoch 68 51.097 ms, 2.74 s total + + +EPOCH :: 69 + + +[ 2023-09-21 07:58:20 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 69 from training step 0 + +[ 2023-09-21 07:58:23 ] Completed train batch 0 to device 2,991.578 ms, 2.99 s total +[ 2023-09-21 07:58:23 ] Completed train batch 0 generator forward 27.041 ms, 3.02 s total +[ 2023-09-21 07:58:23 ] Completed train batch 0 generator loss: 1.299 69.722 ms, 3.09 s total +[ 2023-09-21 07:58:24 ] Completed train batch 0 generator backward 662.208 ms, 3.75 s total +[ 2023-09-21 07:58:24 ] Completed train batch 0 discriminator loss 0.135 58.301 ms, 3.81 s total +[ 2023-09-21 07:58:24 ] Completed train batch 0 discriminator backward 86.715 ms, 3.90 s total +Epoch [69] Step [0/2], gen_loss: 0.446, disc_loss: 0.044 +[ 2023-09-21 07:58:24 ] Completed train batch 0 metrics update 5.229 ms, 3.90 s total +Saving checkpoint at epoch 69 train batch 0 +[ 2023-09-21 07:58:25 ] Completed saving temp checkpoint 448.222 ms, 4.35 s total +[ 2023-09-21 07:58:25 ] Completed replacing temp checkpoint with checkpoint 24.982 ms, 4.37 s total +[ 2023-09-21 07:58:27 ] Completed train batch 1 to device 1,865.326 ms, 6.24 s total +[ 2023-09-21 07:58:27 ] Completed train batch 1 generator forward 26.428 ms, 6.27 s total +[ 2023-09-21 07:58:27 ] Completed train batch 1 generator loss: 1.357 63.457 ms, 6.33 s total +[ 2023-09-21 07:58:28 ] Completed train batch 1 generator backward 1,011.093 ms, 7.34 s total +[ 2023-09-21 07:58:28 ] Completed train batch 1 discriminator loss 0.128 62.968 ms, 7.40 s total +[ 2023-09-21 07:58:28 ] Completed train batch 1 discriminator backward 58.608 ms, 7.46 s total +Epoch [69] Step [1/2], gen_loss: 0.457, disc_loss: 0.043 +[ 2023-09-21 07:58:28 ] Completed train batch 1 metrics update 0.795 ms, 7.46 s total +Saving checkpoint at epoch 69 train batch 1 +[ 2023-09-21 07:58:28 ] Completed saving temp checkpoint 489.358 ms, 7.95 s total +[ 2023-09-21 07:58:28 ] Completed replacing temp checkpoint with checkpoint 22.628 ms, 7.97 s total +Epoch [69] :: gen_loss: 0.451, disc_loss: 0.044 +[ 2023-09-21 07:58:28 ] Completed training generator for epoch 69 50.364 ms, 8.03 s total +[ 2023-09-21 07:58:28 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 69 from eval step 0 + +[ 2023-09-21 07:58:29 ] Completed eval batch 0 to device 1,160.171 ms, 1.16 s total +[ 2023-09-21 07:58:29 ] Completed eval batch 0 forward 43.831 ms, 1.20 s total +[ 2023-09-21 07:58:29 ] Completed eval batch 0 recons_loss 0.379 ms, 1.20 s total +[ 2023-09-21 07:58:29 ] Completed eval batch 0 metrics update 0.732 ms, 1.21 s total +Saving checkpoint at epoch 69 val batch 0 +[ 2023-09-21 07:58:30 ] Completed saving temp checkpoint 488.500 ms, 1.69 s total +[ 2023-09-21 07:58:30 ] Completed replacing temp checkpoint with checkpoint 36.871 ms, 1.73 s total +[ 2023-09-21 07:58:30 ] Completed eval batch 1 to device 333.524 ms, 2.06 s total +[ 2023-09-21 07:58:30 ] Completed eval batch 1 forward 36.796 ms, 2.10 s total +[ 2023-09-21 07:58:30 ] Completed eval batch 1 recons_loss 0.583 ms, 2.10 s total +[ 2023-09-21 07:58:30 ] Completed eval batch 1 metrics update 100.548 ms, 2.20 s total +Saving checkpoint at epoch 69 val batch 1 +Epoch 69 val loss: 0.0655 +[ 2023-09-21 07:58:31 ] Completed saving temp checkpoint 477.221 ms, 2.68 s total +[ 2023-09-21 07:58:31 ] Completed replacing temp checkpoint with checkpoint 23.654 ms, 2.70 s total +[ 2023-09-21 07:58:31 ] Completed evaluating generator for epoch 69 66.408 ms, 2.77 s total + + +EPOCH :: 70 + + +[ 2023-09-21 07:58:31 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 70 from training step 0 + +[ 2023-09-21 07:58:34 ] Completed train batch 0 to device 3,141.693 ms, 3.14 s total +[ 2023-09-21 07:58:34 ] Completed train batch 0 generator forward 27.267 ms, 3.17 s total +[ 2023-09-21 07:58:34 ] Completed train batch 0 generator loss: 1.375 64.658 ms, 3.23 s total +[ 2023-09-21 07:58:35 ] Completed train batch 0 generator backward 568.531 ms, 3.80 s total +[ 2023-09-21 07:58:35 ] Completed train batch 0 discriminator loss 0.108 57.565 ms, 3.86 s total +[ 2023-09-21 07:58:35 ] Completed train batch 0 discriminator backward 88.898 ms, 3.95 s total +Epoch [70] Step [0/2], gen_loss: 0.463, disc_loss: 0.036 +[ 2023-09-21 07:58:35 ] Completed train batch 0 metrics update 3.056 ms, 3.95 s total +Saving checkpoint at epoch 70 train batch 0 +[ 2023-09-21 07:58:35 ] Completed saving temp checkpoint 461.149 ms, 4.41 s total +[ 2023-09-21 07:58:35 ] Completed replacing temp checkpoint with checkpoint 20.304 ms, 4.43 s total +[ 2023-09-21 07:58:37 ] Completed train batch 1 to device 1,659.915 ms, 6.09 s total +[ 2023-09-21 07:58:37 ] Completed train batch 1 generator forward 22.275 ms, 6.12 s total +[ 2023-09-21 07:58:37 ] Completed train batch 1 generator loss: 1.347 63.017 ms, 6.18 s total +[ 2023-09-21 07:58:38 ] Completed train batch 1 generator backward 805.908 ms, 6.98 s total +[ 2023-09-21 07:58:38 ] Completed train batch 1 discriminator loss 0.109 56.588 ms, 7.04 s total +[ 2023-09-21 07:58:38 ] Completed train batch 1 discriminator backward 75.112 ms, 7.12 s total +Epoch [70] Step [1/2], gen_loss: 0.452, disc_loss: 0.036 +[ 2023-09-21 07:58:38 ] Completed train batch 1 metrics update 4.685 ms, 7.12 s total +Saving checkpoint at epoch 70 train batch 1 +[ 2023-09-21 07:58:39 ] Completed saving temp checkpoint 395.437 ms, 7.52 s total +[ 2023-09-21 07:58:39 ] Completed replacing temp checkpoint with checkpoint 22.855 ms, 7.54 s total +Epoch [70] :: gen_loss: 0.458, disc_loss: 0.036 +[ 2023-09-21 07:58:39 ] Completed training generator for epoch 70 60.674 ms, 7.60 s total +[ 2023-09-21 07:58:39 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 70 from eval step 0 + +[ 2023-09-21 07:58:40 ] Completed eval batch 0 to device 1,074.515 ms, 1.07 s total +[ 2023-09-21 07:58:40 ] Completed eval batch 0 forward 23.678 ms, 1.10 s total +[ 2023-09-21 07:58:40 ] Completed eval batch 0 recons_loss 0.220 ms, 1.10 s total +[ 2023-09-21 07:58:40 ] Completed eval batch 0 metrics update 0.588 ms, 1.10 s total +Saving checkpoint at epoch 70 val batch 0 +[ 2023-09-21 07:58:40 ] Completed saving temp checkpoint 297.500 ms, 1.40 s total +[ 2023-09-21 07:58:40 ] Completed replacing temp checkpoint with checkpoint 24.150 ms, 1.42 s total +[ 2023-09-21 07:58:41 ] Completed eval batch 1 to device 532.310 ms, 1.95 s total +[ 2023-09-21 07:58:41 ] Completed eval batch 1 forward 21.677 ms, 1.97 s total +[ 2023-09-21 07:58:41 ] Completed eval batch 1 recons_loss 0.373 ms, 1.98 s total +[ 2023-09-21 07:58:41 ] Completed eval batch 1 metrics update 320.625 ms, 2.30 s total +Saving checkpoint at epoch 70 val batch 1 +Epoch 70 val loss: 0.0747 +[ 2023-09-21 07:58:41 ] Completed saving temp checkpoint 401.100 ms, 2.70 s total +[ 2023-09-21 07:58:41 ] Completed replacing temp checkpoint with checkpoint 17.812 ms, 2.71 s total +[ 2023-09-21 07:58:41 ] Completed evaluating generator for epoch 70 62.267 ms, 2.78 s total + + +EPOCH :: 71 + + +[ 2023-09-21 07:58:41 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 71 from training step 0 + +[ 2023-09-21 07:58:45 ] Completed train batch 0 to device 3,124.482 ms, 3.12 s total +[ 2023-09-21 07:58:45 ] Completed train batch 0 generator forward 34.011 ms, 3.16 s total +[ 2023-09-21 07:58:45 ] Completed train batch 0 generator loss: 1.362 66.355 ms, 3.22 s total +[ 2023-09-21 07:58:45 ] Completed train batch 0 generator backward 554.264 ms, 3.78 s total +[ 2023-09-21 07:58:45 ] Completed train batch 0 discriminator loss 0.106 57.635 ms, 3.84 s total +[ 2023-09-21 07:58:45 ] Completed train batch 0 discriminator backward 103.922 ms, 3.94 s total +Epoch [71] Step [0/2], gen_loss: 0.459, disc_loss: 0.035 +[ 2023-09-21 07:58:45 ] Completed train batch 0 metrics update 5.941 ms, 3.95 s total +Saving checkpoint at epoch 71 train batch 0 +[ 2023-09-21 07:58:46 ] Completed saving temp checkpoint 433.877 ms, 4.38 s total +[ 2023-09-21 07:58:46 ] Completed replacing temp checkpoint with checkpoint 30.553 ms, 4.41 s total +[ 2023-09-21 07:58:47 ] Completed train batch 1 to device 1,551.116 ms, 5.96 s total +[ 2023-09-21 07:58:47 ] Completed train batch 1 generator forward 22.154 ms, 5.98 s total +[ 2023-09-21 07:58:47 ] Completed train batch 1 generator loss: 1.350 63.407 ms, 6.05 s total +[ 2023-09-21 07:58:49 ] Completed train batch 1 generator backward 1,346.798 ms, 7.39 s total +[ 2023-09-21 07:58:49 ] Completed train batch 1 discriminator loss 0.100 56.385 ms, 7.45 s total +[ 2023-09-21 07:58:49 ] Completed train batch 1 discriminator backward 88.573 ms, 7.54 s total +Epoch [71] Step [1/2], gen_loss: 0.452, disc_loss: 0.034 +[ 2023-09-21 07:58:49 ] Completed train batch 1 metrics update 4.402 ms, 7.54 s total +Saving checkpoint at epoch 71 train batch 1 +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 08:11:13 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 08:11:13 ] Completed importing Timer 0.026 ms, 0.00 s total +[ 2023-09-21 08:11:18 ] Completed importing everything else 4,614.973 ms, 4.61 s total +| distributed init (rank 3): env:// +| distributed init (rank 0): env:// +| distributed init (rank 4): env:// +| distributed init (rank 2): env:// +| distributed init (rank 1): env:// +| distributed init (rank 5): env:// +[ 2023-09-21 08:11:26 ] Completed preliminaries 7,857.885 ms, 12.47 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 08:11:26 ] Completed build datasets 12.437 ms, 12.49 s total +[ 2023-09-21 08:11:26 ] Completed build samplers 0.042 ms, 12.49 s total +[ 2023-09-21 08:11:26 ] Completed build dataloaders 0.127 ms, 12.49 s total +[ 2023-09-21 08:11:26 ] Completed generator to device 90.771 ms, 12.58 s total +[ 2023-09-21 08:11:26 ] Completed discriminator to device 7.047 ms, 12.58 s total +[ 2023-09-21 08:11:26 ] Completed loss functions 396.861 ms, 12.98 s total +[ 2023-09-21 08:11:26 ] Completed models prepped for distribution 100.575 ms, 13.08 s total +[ 2023-09-21 08:11:26 ] Completed optimizers 0.656 ms, 13.08 s total +[ 2023-09-21 08:11:26 ] Completed grad scalers 0.019 ms, 13.08 s total +[ 2023-09-21 08:11:27 ] Completed checkpoint retrieval 437.787 ms, 13.52 s total + + +EPOCH :: 71 + + +[ 2023-09-21 08:11:27 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 71 from training step 1 + +[ 2023-09-21 08:11:30 ] Completed train batch 1 to device 3,541.980 ms, 3.54 s total +[ 2023-09-21 08:11:31 ] Completed train batch 1 generator forward 1,088.314 ms, 4.63 s total +[ 2023-09-21 08:11:31 ] Completed train batch 1 generator loss: 1.296 77.965 ms, 4.71 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 08:11:32 ] Completed train batch 1 generator backward 794.755 ms, 5.50 s total +[ 2023-09-21 08:11:32 ] Completed train batch 1 discriminator loss 0.100 55.693 ms, 5.56 s total +[ 2023-09-21 08:11:32 ] Completed train batch 1 discriminator backward 87.523 ms, 5.65 s total +Epoch [71] Step [1/2], gen_loss: 0.455, disc_loss: 0.034 +[ 2023-09-21 08:11:32 ] Completed train batch 1 metrics update 5.483 ms, 5.65 s total +Saving checkpoint at epoch 71 train batch 1 +[ 2023-09-21 08:11:33 ] Completed saving temp checkpoint 487.428 ms, 6.14 s total +[ 2023-09-21 08:11:33 ] Completed replacing temp checkpoint with checkpoint 131.364 ms, 6.27 s total +Epoch [71] :: gen_loss: 0.457, disc_loss: 0.035 +[ 2023-09-21 08:11:33 ] Completed training generator for epoch 71 36.094 ms, 6.31 s total +[ 2023-09-21 08:11:33 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 71 from eval step 0 + +[ 2023-09-21 08:11:34 ] Completed eval batch 0 to device 1,090.389 ms, 1.09 s total +[ 2023-09-21 08:11:34 ] Completed eval batch 0 forward 42.589 ms, 1.13 s total +[ 2023-09-21 08:11:34 ] Completed eval batch 0 recons_loss 0.347 ms, 1.13 s total +[ 2023-09-21 08:11:34 ] Completed eval batch 0 metrics update 0.626 ms, 1.13 s total +Saving checkpoint at epoch 71 val batch 0 +[ 2023-09-21 08:11:35 ] Completed saving temp checkpoint 446.639 ms, 1.58 s total +[ 2023-09-21 08:11:35 ] Completed replacing temp checkpoint with checkpoint 30.902 ms, 1.61 s total +[ 2023-09-21 08:11:35 ] Completed eval batch 1 to device 293.216 ms, 1.90 s total +[ 2023-09-21 08:11:35 ] Completed eval batch 1 forward 40.259 ms, 1.94 s total +[ 2023-09-21 08:11:35 ] Completed eval batch 1 recons_loss 0.532 ms, 1.95 s total +[ 2023-09-21 08:11:35 ] Completed eval batch 1 metrics update 151.727 ms, 2.10 s total +Saving checkpoint at epoch 71 val batch 1 +Epoch 71 val loss: 0.0817 +[ 2023-09-21 08:11:35 ] Completed saving temp checkpoint 439.025 ms, 2.54 s total +[ 2023-09-21 08:11:35 ] Completed replacing temp checkpoint with checkpoint 20.708 ms, 2.56 s total +[ 2023-09-21 08:11:36 ] Completed evaluating generator for epoch 71 52.904 ms, 2.61 s total + + +EPOCH :: 72 + + +[ 2023-09-21 08:11:36 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 72 from training step 0 + +[ 2023-09-21 08:11:39 ] Completed train batch 0 to device 2,988.386 ms, 2.99 s total +[ 2023-09-21 08:11:39 ] Completed train batch 0 generator forward 28.979 ms, 3.02 s total +[ 2023-09-21 08:11:39 ] Completed train batch 0 generator loss: 1.326 67.123 ms, 3.08 s total +[ 2023-09-21 08:11:39 ] Completed train batch 0 generator backward 818.970 ms, 3.90 s total +[ 2023-09-21 08:11:40 ] Completed train batch 0 discriminator loss 0.099 57.118 ms, 3.96 s total +[ 2023-09-21 08:11:40 ] Completed train batch 0 discriminator backward 99.349 ms, 4.06 s total +Epoch [72] Step [0/2], gen_loss: 0.445, disc_loss: 0.033 +[ 2023-09-21 08:11:40 ] Completed train batch 0 metrics update 28.022 ms, 4.09 s total +Saving checkpoint at epoch 72 train batch 0 +[ 2023-09-21 08:11:40 ] Completed saving temp checkpoint 377.349 ms, 4.47 s total +[ 2023-09-21 08:11:40 ] Completed replacing temp checkpoint with checkpoint 30.498 ms, 4.50 s total +[ 2023-09-21 08:11:41 ] Completed train batch 1 to device 1,297.884 ms, 5.79 s total +[ 2023-09-21 08:11:41 ] Completed train batch 1 generator forward 22.817 ms, 5.82 s total +[ 2023-09-21 08:11:41 ] Completed train batch 1 generator loss: 1.306 63.894 ms, 5.88 s total +[ 2023-09-21 08:11:43 ] Completed train batch 1 generator backward 1,630.351 ms, 7.51 s total +[ 2023-09-21 08:11:43 ] Completed train batch 1 discriminator loss 0.096 56.381 ms, 7.57 s total +[ 2023-09-21 08:11:43 ] Completed train batch 1 discriminator backward 85.316 ms, 7.65 s total +Epoch [72] Step [1/2], gen_loss: 0.447, disc_loss: 0.032 +[ 2023-09-21 08:11:43 ] Completed train batch 1 metrics update 2.208 ms, 7.65 s total +Saving checkpoint at epoch 72 train batch 1 +[ 2023-09-21 08:11:44 ] Completed saving temp checkpoint 408.053 ms, 8.06 s total +[ 2023-09-21 08:11:44 ] Completed replacing temp checkpoint with checkpoint 22.185 ms, 8.08 s total +Epoch [72] :: gen_loss: 0.446, disc_loss: 0.032 +[ 2023-09-21 08:11:44 ] Completed training generator for epoch 72 50.336 ms, 8.14 s total +[ 2023-09-21 08:11:44 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 72 from eval step 0 + +[ 2023-09-21 08:11:45 ] Completed eval batch 0 to device 1,195.798 ms, 1.20 s total +[ 2023-09-21 08:11:45 ] Completed eval batch 0 forward 24.609 ms, 1.22 s total +[ 2023-09-21 08:11:45 ] Completed eval batch 0 recons_loss 0.268 ms, 1.22 s total +[ 2023-09-21 08:11:45 ] Completed eval batch 0 metrics update 0.568 ms, 1.22 s total +Saving checkpoint at epoch 72 val batch 0 +[ 2023-09-21 08:11:45 ] Completed saving temp checkpoint 396.244 ms, 1.62 s total +[ 2023-09-21 08:11:45 ] Completed replacing temp checkpoint with checkpoint 23.093 ms, 1.64 s total +[ 2023-09-21 08:11:46 ] Completed eval batch 1 to device 545.331 ms, 2.19 s total +[ 2023-09-21 08:11:46 ] Completed eval batch 1 forward 39.984 ms, 2.23 s total +[ 2023-09-21 08:11:46 ] Completed eval batch 1 recons_loss 0.437 ms, 2.23 s total +[ 2023-09-21 08:11:46 ] Completed eval batch 1 metrics update 67.760 ms, 2.29 s total +Saving checkpoint at epoch 72 val batch 1 +Epoch 72 val loss: 0.0794 +[ 2023-09-21 08:11:46 ] Completed saving temp checkpoint 412.818 ms, 2.71 s total +[ 2023-09-21 08:11:46 ] Completed replacing temp checkpoint with checkpoint 22.298 ms, 2.73 s total +[ 2023-09-21 08:11:46 ] Completed evaluating generator for epoch 72 50.466 ms, 2.78 s total + + +EPOCH :: 73 + + +[ 2023-09-21 08:11:46 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 73 from training step 0 + +[ 2023-09-21 08:11:50 ] Completed train batch 0 to device 3,181.154 ms, 3.18 s total +[ 2023-09-21 08:11:50 ] Completed train batch 0 generator forward 26.084 ms, 3.21 s total +[ 2023-09-21 08:11:50 ] Completed train batch 0 generator loss: 1.325 66.549 ms, 3.27 s total +[ 2023-09-21 08:11:50 ] Completed train batch 0 generator backward 483.546 ms, 3.76 s total +[ 2023-09-21 08:11:50 ] Completed train batch 0 discriminator loss 0.098 59.189 ms, 3.82 s total +[ 2023-09-21 08:11:50 ] Completed train batch 0 discriminator backward 96.352 ms, 3.91 s total +Epoch [73] Step [0/2], gen_loss: 0.443, disc_loss: 0.033 +[ 2023-09-21 08:11:50 ] Completed train batch 0 metrics update 5.581 ms, 3.92 s total +Saving checkpoint at epoch 73 train batch 0 +[ 2023-09-21 08:11:51 ] Completed saving temp checkpoint 460.586 ms, 4.38 s total +[ 2023-09-21 08:11:51 ] Completed replacing temp checkpoint with checkpoint 34.043 ms, 4.41 s total +[ 2023-09-21 08:11:53 ] Completed train batch 1 to device 2,106.484 ms, 6.52 s total +[ 2023-09-21 08:11:53 ] Completed train batch 1 generator forward 30.064 ms, 6.55 s total +[ 2023-09-21 08:11:53 ] Completed train batch 1 generator loss: 1.291 64.351 ms, 6.61 s total +[ 2023-09-21 08:11:54 ] Completed train batch 1 generator backward 661.737 ms, 7.28 s total +[ 2023-09-21 08:11:54 ] Completed train batch 1 discriminator loss 0.098 56.516 ms, 7.33 s total +[ 2023-09-21 08:11:54 ] Completed train batch 1 discriminator backward 88.479 ms, 7.42 s total +Epoch [73] Step [1/2], gen_loss: 0.442, disc_loss: 0.033 +[ 2023-09-21 08:11:54 ] Completed train batch 1 metrics update 5.269 ms, 7.43 s total +Saving checkpoint at epoch 73 train batch 1 +[ 2023-09-21 08:11:54 ] Completed saving temp checkpoint 455.297 ms, 7.88 s total +[ 2023-09-21 08:11:54 ] Completed replacing temp checkpoint with checkpoint 26.319 ms, 7.91 s total +Epoch [73] :: gen_loss: 0.442, disc_loss: 0.033 +[ 2023-09-21 08:11:54 ] Completed training generator for epoch 73 53.993 ms, 7.96 s total +[ 2023-09-21 08:11:54 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 73 from eval step 0 + +[ 2023-09-21 08:11:56 ] Completed eval batch 0 to device 1,113.735 ms, 1.11 s total +[ 2023-09-21 08:11:56 ] Completed eval batch 0 forward 24.830 ms, 1.14 s total +[ 2023-09-21 08:11:56 ] Completed eval batch 0 recons_loss 0.390 ms, 1.14 s total +[ 2023-09-21 08:11:56 ] Completed eval batch 0 metrics update 0.711 ms, 1.14 s total +Saving checkpoint at epoch 73 val batch 0 +[ 2023-09-21 08:11:56 ] Completed saving temp checkpoint 384.825 ms, 1.52 s total +[ 2023-09-21 08:11:56 ] Completed replacing temp checkpoint with checkpoint 30.683 ms, 1.56 s total +[ 2023-09-21 08:11:56 ] Completed eval batch 1 to device 460.177 ms, 2.02 s total +[ 2023-09-21 08:11:56 ] Completed eval batch 1 forward 21.869 ms, 2.04 s total +[ 2023-09-21 08:11:56 ] Completed eval batch 1 recons_loss 0.346 ms, 2.04 s total +[ 2023-09-21 08:11:57 ] Completed eval batch 1 metrics update 287.086 ms, 2.32 s total +Saving checkpoint at epoch 73 val batch 1 +Epoch 73 val loss: 0.0816 +[ 2023-09-21 08:11:57 ] Completed saving temp checkpoint 429.478 ms, 2.75 s total +[ 2023-09-21 08:11:57 ] Completed replacing temp checkpoint with checkpoint 20.118 ms, 2.77 s total +[ 2023-09-21 08:11:57 ] Completed evaluating generator for epoch 73 48.632 ms, 2.82 s total + + +EPOCH :: 74 + + +[ 2023-09-21 08:11:57 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 74 from training step 0 + +[ 2023-09-21 08:12:00 ] Completed train batch 0 to device 3,226.211 ms, 3.23 s total +[ 2023-09-21 08:12:01 ] Completed train batch 0 generator forward 32.484 ms, 3.26 s total +[ 2023-09-21 08:12:01 ] Completed train batch 0 generator loss: 1.296 66.538 ms, 3.33 s total +[ 2023-09-21 08:12:01 ] Completed train batch 0 generator backward 565.629 ms, 3.89 s total +[ 2023-09-21 08:12:01 ] Completed train batch 0 discriminator loss 0.092 57.858 ms, 3.95 s total +[ 2023-09-21 08:12:01 ] Completed train batch 0 discriminator backward 105.334 ms, 4.05 s total +Epoch [74] Step [0/2], gen_loss: 0.433, disc_loss: 0.031 +[ 2023-09-21 08:12:01 ] Completed train batch 0 metrics update 21.263 ms, 4.08 s total +Saving checkpoint at epoch 74 train batch 0 +[ 2023-09-21 08:12:02 ] Completed saving temp checkpoint 421.005 ms, 4.50 s total +[ 2023-09-21 08:12:02 ] Completed replacing temp checkpoint with checkpoint 21.021 ms, 4.52 s total +[ 2023-09-21 08:12:03 ] Completed train batch 1 to device 1,716.890 ms, 6.23 s total +[ 2023-09-21 08:12:04 ] Completed train batch 1 generator forward 22.992 ms, 6.26 s total +[ 2023-09-21 08:12:04 ] Completed train batch 1 generator loss: 1.431 64.256 ms, 6.32 s total +[ 2023-09-21 08:12:05 ] Completed train batch 1 generator backward 1,025.352 ms, 7.35 s total +[ 2023-09-21 08:12:05 ] Completed train batch 1 discriminator loss 0.088 68.942 ms, 7.42 s total +[ 2023-09-21 08:12:05 ] Completed train batch 1 discriminator backward 61.145 ms, 7.48 s total +Epoch [74] Step [1/2], gen_loss: 0.438, disc_loss: 0.030 +[ 2023-09-21 08:12:05 ] Completed train batch 1 metrics update 2.128 ms, 7.48 s total +Saving checkpoint at epoch 74 train batch 1 +[ 2023-09-21 08:12:05 ] Completed saving temp checkpoint 431.521 ms, 7.91 s total +[ 2023-09-21 08:12:05 ] Completed replacing temp checkpoint with checkpoint 22.538 ms, 7.93 s total +Epoch [74] :: gen_loss: 0.435, disc_loss: 0.030 +[ 2023-09-21 08:12:05 ] Completed training generator for epoch 74 54.696 ms, 7.99 s total +[ 2023-09-21 08:12:05 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 74 from eval step 0 + +[ 2023-09-21 08:12:06 ] Completed eval batch 0 to device 1,143.475 ms, 1.14 s total +[ 2023-09-21 08:12:06 ] Completed eval batch 0 forward 25.599 ms, 1.17 s total +[ 2023-09-21 08:12:06 ] Completed eval batch 0 recons_loss 0.346 ms, 1.17 s total +[ 2023-09-21 08:12:06 ] Completed eval batch 0 metrics update 0.693 ms, 1.17 s total +Saving checkpoint at epoch 74 val batch 0 +[ 2023-09-21 08:12:07 ] Completed saving temp checkpoint 434.375 ms, 1.60 s total +[ 2023-09-21 08:12:07 ] Completed replacing temp checkpoint with checkpoint 33.352 ms, 1.64 s total +[ 2023-09-21 08:12:07 ] Completed eval batch 1 to device 457.752 ms, 2.10 s total +[ 2023-09-21 08:12:07 ] Completed eval batch 1 forward 22.208 ms, 2.12 s total +[ 2023-09-21 08:12:07 ] Completed eval batch 1 recons_loss 0.350 ms, 2.12 s total +[ 2023-09-21 08:12:07 ] Completed eval batch 1 metrics update 100.968 ms, 2.22 s total +Saving checkpoint at epoch 74 val batch 1 +Epoch 74 val loss: 0.0658 +[ 2023-09-21 08:12:08 ] Completed saving temp checkpoint 361.060 ms, 2.58 s total +[ 2023-09-21 08:12:08 ] Completed replacing temp checkpoint with checkpoint 25.610 ms, 2.61 s total +[ 2023-09-21 08:12:08 ] Completed evaluating generator for epoch 74 55.727 ms, 2.66 s total + + +EPOCH :: 75 + + +[ 2023-09-21 08:12:08 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 75 from training step 0 + +[ 2023-09-21 08:12:11 ] Completed train batch 0 to device 2,934.395 ms, 2.93 s total +[ 2023-09-21 08:12:11 ] Completed train batch 0 generator forward 26.872 ms, 2.96 s total +[ 2023-09-21 08:12:11 ] Completed train batch 0 generator loss: 1.286 66.668 ms, 3.03 s total +[ 2023-09-21 08:12:12 ] Completed train batch 0 generator backward 947.278 ms, 3.98 s total +[ 2023-09-21 08:12:12 ] Completed train batch 0 discriminator loss 0.090 62.886 ms, 4.04 s total +[ 2023-09-21 08:12:12 ] Completed train batch 0 discriminator backward 90.037 ms, 4.13 s total +Epoch [75] Step [0/2], gen_loss: 0.436, disc_loss: 0.029 +[ 2023-09-21 08:12:12 ] Completed train batch 0 metrics update 4.243 ms, 4.13 s total +Saving checkpoint at epoch 75 train batch 0 +[ 2023-09-21 08:12:12 ] Completed saving temp checkpoint 428.024 ms, 4.56 s total +[ 2023-09-21 08:12:12 ] Completed replacing temp checkpoint with checkpoint 31.415 ms, 4.59 s total +[ 2023-09-21 08:12:14 ] Completed train batch 1 to device 1,023.425 ms, 5.62 s total +[ 2023-09-21 08:12:14 ] Completed train batch 1 generator forward 23.913 ms, 5.64 s total +[ 2023-09-21 08:12:14 ] Completed train batch 1 generator loss: 1.365 65.011 ms, 5.70 s total +[ 2023-09-21 08:12:16 ] Completed train batch 1 generator backward 1,945.294 ms, 7.65 s total +[ 2023-09-21 08:12:16 ] Completed train batch 1 discriminator loss 0.081 56.625 ms, 7.71 s total +[ 2023-09-21 08:12:16 ] Completed train batch 1 discriminator backward 74.133 ms, 7.78 s total +Epoch [75] Step [1/2], gen_loss: 0.447, disc_loss: 0.027 +[ 2023-09-21 08:12:16 ] Completed train batch 1 metrics update 1.367 ms, 7.78 s total +Saving checkpoint at epoch 75 train batch 1 +[ 2023-09-21 08:12:16 ] Completed saving temp checkpoint 452.463 ms, 8.23 s total +[ 2023-09-21 08:12:16 ] Completed replacing temp checkpoint with checkpoint 23.898 ms, 8.26 s total +Epoch [75] :: gen_loss: 0.441, disc_loss: 0.028 +[ 2023-09-21 08:12:16 ] Completed training generator for epoch 75 52.999 ms, 8.31 s total +[ 2023-09-21 08:12:16 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 75 from eval step 0 + +[ 2023-09-21 08:12:17 ] Completed eval batch 0 to device 1,163.130 ms, 1.16 s total +[ 2023-09-21 08:12:17 ] Completed eval batch 0 forward 41.380 ms, 1.20 s total +[ 2023-09-21 08:12:17 ] Completed eval batch 0 recons_loss 0.296 ms, 1.20 s total +[ 2023-09-21 08:12:17 ] Completed eval batch 0 metrics update 0.627 ms, 1.21 s total +Saving checkpoint at epoch 75 val batch 0 +[ 2023-09-21 08:12:18 ] Completed saving temp checkpoint 405.962 ms, 1.61 s total +[ 2023-09-21 08:12:18 ] Completed replacing temp checkpoint with checkpoint 33.534 ms, 1.64 s total +[ 2023-09-21 08:12:18 ] Completed eval batch 1 to device 375.302 ms, 2.02 s total +[ 2023-09-21 08:12:18 ] Completed eval batch 1 forward 21.859 ms, 2.04 s total +[ 2023-09-21 08:12:18 ] Completed eval batch 1 recons_loss 0.333 ms, 2.04 s total +[ 2023-09-21 08:12:18 ] Completed eval batch 1 metrics update 185.881 ms, 2.23 s total +Saving checkpoint at epoch 75 val batch 1 +Epoch 75 val loss: 0.0618 +[ 2023-09-21 08:12:19 ] Completed saving temp checkpoint 435.086 ms, 2.66 s total +[ 2023-09-21 08:12:19 ] Completed replacing temp checkpoint with checkpoint 33.545 ms, 2.70 s total +[ 2023-09-21 08:12:19 ] Completed evaluating generator for epoch 75 55.952 ms, 2.75 s total + + +EPOCH :: 76 + + +[ 2023-09-21 08:12:19 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 76 from training step 0 + +[ 2023-09-21 08:12:22 ] Completed train batch 0 to device 3,014.040 ms, 3.01 s total +[ 2023-09-21 08:12:22 ] Completed train batch 0 generator forward 26.714 ms, 3.04 s total +[ 2023-09-21 08:12:22 ] Completed train batch 0 generator loss: 1.285 66.636 ms, 3.11 s total +[ 2023-09-21 08:12:23 ] Completed train batch 0 generator backward 643.326 ms, 3.75 s total +[ 2023-09-21 08:12:23 ] Completed train batch 0 discriminator loss 0.081 56.926 ms, 3.81 s total +[ 2023-09-21 08:12:23 ] Completed train batch 0 discriminator backward 106.519 ms, 3.91 s total +Epoch [76] Step [0/2], gen_loss: 0.428, disc_loss: 0.027 +[ 2023-09-21 08:12:23 ] Completed train batch 0 metrics update 3.109 ms, 3.92 s total +Saving checkpoint at epoch 76 train batch 0 +[ 2023-09-21 08:12:23 ] Completed saving temp checkpoint 423.190 ms, 4.34 s total +[ 2023-09-21 08:12:23 ] Completed replacing temp checkpoint with checkpoint 29.092 ms, 4.37 s total +[ 2023-09-21 08:12:25 ] Completed train batch 1 to device 1,385.659 ms, 5.76 s total +[ 2023-09-21 08:12:25 ] Completed train batch 1 generator forward 39.103 ms, 5.79 s total +[ 2023-09-21 08:12:25 ] Completed train batch 1 generator loss: 1.448 63.654 ms, 5.86 s total +[ 2023-09-21 08:12:26 ] Completed train batch 1 generator backward 1,261.234 ms, 7.12 s total +[ 2023-09-21 08:12:26 ] Completed train batch 1 discriminator loss 0.080 56.463 ms, 7.18 s total +[ 2023-09-21 08:12:26 ] Completed train batch 1 discriminator backward 72.639 ms, 7.25 s total +Epoch [76] Step [1/2], gen_loss: 0.465, disc_loss: 0.027 +[ 2023-09-21 08:12:26 ] Completed train batch 1 metrics update 1.084 ms, 7.25 s total +Saving checkpoint at epoch 76 train batch 1 +[ 2023-09-21 08:12:27 ] Completed saving temp checkpoint 464.993 ms, 7.71 s total +[ 2023-09-21 08:12:27 ] Completed replacing temp checkpoint with checkpoint 24.421 ms, 7.74 s total +Epoch [76] :: gen_loss: 0.446, disc_loss: 0.027 +[ 2023-09-21 08:12:27 ] Completed training generator for epoch 76 57.006 ms, 7.80 s total +[ 2023-09-21 08:12:27 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 76 from eval step 0 + +[ 2023-09-21 08:12:28 ] Completed eval batch 0 to device 1,171.629 ms, 1.17 s total +[ 2023-09-21 08:12:28 ] Completed eval batch 0 forward 42.575 ms, 1.21 s total +[ 2023-09-21 08:12:28 ] Completed eval batch 0 recons_loss 0.318 ms, 1.21 s total +[ 2023-09-21 08:12:28 ] Completed eval batch 0 metrics update 0.554 ms, 1.22 s total +Saving checkpoint at epoch 76 val batch 0 +[ 2023-09-21 08:12:28 ] Completed saving temp checkpoint 457.609 ms, 1.67 s total +[ 2023-09-21 08:12:28 ] Completed replacing temp checkpoint with checkpoint 37.383 ms, 1.71 s total +[ 2023-09-21 08:12:29 ] Completed eval batch 1 to device 347.045 ms, 2.06 s total +[ 2023-09-21 08:12:29 ] Completed eval batch 1 forward 40.310 ms, 2.10 s total +[ 2023-09-21 08:12:29 ] Completed eval batch 1 recons_loss 0.519 ms, 2.10 s total +[ 2023-09-21 08:12:29 ] Completed eval batch 1 metrics update 131.453 ms, 2.23 s total +Saving checkpoint at epoch 76 val batch 1 +Epoch 76 val loss: 0.0715 +[ 2023-09-21 08:12:30 ] Completed saving temp checkpoint 539.074 ms, 2.77 s total +[ 2023-09-21 08:12:30 ] Completed replacing temp checkpoint with checkpoint 33.298 ms, 2.80 s total +[ 2023-09-21 08:12:30 ] Completed evaluating generator for epoch 76 72.310 ms, 2.87 s total + + +EPOCH :: 77 + + +[ 2023-09-21 08:12:30 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 77 from training step 0 + +[ 2023-09-21 08:12:33 ] Completed train batch 0 to device 3,064.954 ms, 3.06 s total +[ 2023-09-21 08:12:33 ] Completed train batch 0 generator forward 26.080 ms, 3.09 s total +[ 2023-09-21 08:12:33 ] Completed train batch 0 generator loss: 1.278 65.871 ms, 3.16 s total +[ 2023-09-21 08:12:33 ] Completed train batch 0 generator backward 491.118 ms, 3.65 s total +[ 2023-09-21 08:12:33 ] Completed train batch 0 discriminator loss 0.103 76.216 ms, 3.72 s total +[ 2023-09-21 08:12:33 ] Completed train batch 0 discriminator backward 69.385 ms, 3.79 s total +Epoch [77] Step [0/2], gen_loss: 0.431, disc_loss: 0.034 +[ 2023-09-21 08:12:33 ] Completed train batch 0 metrics update 7.152 ms, 3.80 s total +Saving checkpoint at epoch 77 train batch 0 +[ 2023-09-21 08:12:34 ] Completed saving temp checkpoint 603.802 ms, 4.40 s total +[ 2023-09-21 08:12:34 ] Completed replacing temp checkpoint with checkpoint 38.718 ms, 4.44 s total +[ 2023-09-21 08:12:36 ] Completed train batch 1 to device 1,508.849 ms, 5.95 s total +[ 2023-09-21 08:12:36 ] Completed train batch 1 generator forward 26.731 ms, 5.98 s total +[ 2023-09-21 08:12:36 ] Completed train batch 1 generator loss: 1.314 64.571 ms, 6.04 s total +[ 2023-09-21 08:12:37 ] Completed train batch 1 generator backward 1,001.168 ms, 7.04 s total +[ 2023-09-21 08:12:37 ] Completed train batch 1 discriminator loss 0.107 56.452 ms, 7.10 s total +[ 2023-09-21 08:12:37 ] Completed train batch 1 discriminator backward 72.795 ms, 7.17 s total +Epoch [77] Step [1/2], gen_loss: 0.442, disc_loss: 0.036 +[ 2023-09-21 08:12:37 ] Completed train batch 1 metrics update 0.880 ms, 7.17 s total +Saving checkpoint at epoch 77 train batch 1 +[ 2023-09-21 08:12:37 ] Completed saving temp checkpoint 458.492 ms, 7.63 s total +[ 2023-09-21 08:12:37 ] Completed replacing temp checkpoint with checkpoint 28.692 ms, 7.66 s total +Epoch [77] :: gen_loss: 0.436, disc_loss: 0.035 +[ 2023-09-21 08:12:37 ] Completed training generator for epoch 77 57.193 ms, 7.72 s total +[ 2023-09-21 08:12:37 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 77 from eval step 0 + +[ 2023-09-21 08:12:39 ] Completed eval batch 0 to device 1,339.295 ms, 1.34 s total +[ 2023-09-21 08:12:39 ] Completed eval batch 0 forward 43.233 ms, 1.38 s total +[ 2023-09-21 08:12:39 ] Completed eval batch 0 recons_loss 0.298 ms, 1.38 s total +[ 2023-09-21 08:12:39 ] Completed eval batch 0 metrics update 0.513 ms, 1.38 s total +Saving checkpoint at epoch 77 val batch 0 +[ 2023-09-21 08:12:39 ] Completed saving temp checkpoint 531.855 ms, 1.92 s total +[ 2023-09-21 08:12:39 ] Completed replacing temp checkpoint with checkpoint 32.410 ms, 1.95 s total +[ 2023-09-21 08:12:40 ] Completed eval batch 1 to device 240.498 ms, 2.19 s total +[ 2023-09-21 08:12:40 ] Completed eval batch 1 forward 40.450 ms, 2.23 s total +[ 2023-09-21 08:12:40 ] Completed eval batch 1 recons_loss 0.579 ms, 2.23 s total +[ 2023-09-21 08:12:40 ] Completed eval batch 1 metrics update 0.318 ms, 2.23 s total +Saving checkpoint at epoch 77 val batch 1 +Epoch 77 val loss: 0.0733 +[ 2023-09-21 08:12:40 ] Completed saving temp checkpoint 406.553 ms, 2.64 s total +[ 2023-09-21 08:12:40 ] Completed replacing temp checkpoint with checkpoint 22.236 ms, 2.66 s total +[ 2023-09-21 08:12:40 ] Completed evaluating generator for epoch 77 61.369 ms, 2.72 s total + + +EPOCH :: 78 + + +[ 2023-09-21 08:12:40 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 78 from training step 0 + +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 08:23:28 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 08:23:28 ] Completed importing Timer 0.034 ms, 0.00 s total +[ 2023-09-21 08:23:32 ] Completed importing everything else 4,605.591 ms, 4.61 s total +| distributed init (rank 0): env:// +| distributed init (rank 4): env:// +| distributed init (rank 5): env:// +| distributed init (rank 3): env:// +| distributed init (rank 2): env:// +| distributed init (rank 1): env:// +[ 2023-09-21 08:23:40 ] Completed preliminaries 7,903.409 ms, 12.51 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 08:23:40 ] Completed build datasets 10.761 ms, 12.52 s total +[ 2023-09-21 08:23:40 ] Completed build samplers 0.041 ms, 12.52 s total +[ 2023-09-21 08:23:40 ] Completed build dataloaders 0.131 ms, 12.52 s total +[ 2023-09-21 08:23:40 ] Completed generator to device 95.443 ms, 12.62 s total +[ 2023-09-21 08:23:41 ] Completed discriminator to device 7.251 ms, 12.62 s total +[ 2023-09-21 08:23:41 ] Completed loss functions 371.143 ms, 12.99 s total +[ 2023-09-21 08:23:41 ] Completed models prepped for distribution 90.552 ms, 13.08 s total +[ 2023-09-21 08:23:41 ] Completed optimizers 0.664 ms, 13.09 s total +[ 2023-09-21 08:23:41 ] Completed grad scalers 0.024 ms, 13.09 s total +[ 2023-09-21 08:23:41 ] Completed checkpoint retrieval 451.235 ms, 13.54 s total + + +EPOCH :: 77 + + +[ 2023-09-21 08:23:41 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 77 from training step 2 + +Epoch [77] :: gen_loss: 0.436, disc_loss: 0.035 +[ 2023-09-21 08:23:42 ] Completed training generator for epoch 77 133.966 ms, 0.13 s total +[ 2023-09-21 08:23:42 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 77 from eval step 2 + +[ 2023-09-21 08:23:42 ] Completed evaluating generator for epoch 77 134.040 ms, 0.13 s total + + +EPOCH :: 78 + + +[ 2023-09-21 08:23:42 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 78 from training step 0 + +[ 2023-09-21 08:23:46 ] Completed train batch 0 to device 3,827.840 ms, 3.83 s total +[ 2023-09-21 08:23:47 ] Completed train batch 0 generator forward 1,532.256 ms, 5.36 s total +[ 2023-09-21 08:23:47 ] Completed train batch 0 generator loss: 1.284 75.993 ms, 5.44 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 08:23:48 ] Completed train batch 0 generator backward 812.561 ms, 6.25 s total +[ 2023-09-21 08:23:48 ] Completed train batch 0 discriminator loss 0.087 58.532 ms, 6.31 s total +[ 2023-09-21 08:23:48 ] Completed train batch 0 discriminator backward 112.880 ms, 6.42 s total +Epoch [78] Step [0/2], gen_loss: 0.428, disc_loss: 0.030 +[ 2023-09-21 08:23:48 ] Completed train batch 0 metrics update 15.599 ms, 6.44 s total +Saving checkpoint at epoch 78 train batch 0 +[ 2023-09-21 08:23:49 ] Completed saving temp checkpoint 475.398 ms, 6.91 s total +[ 2023-09-21 08:23:49 ] Completed replacing temp checkpoint with checkpoint 132.559 ms, 7.04 s total +[ 2023-09-21 08:23:49 ] Completed train batch 1 to device 133.143 ms, 7.18 s total +[ 2023-09-21 08:23:49 ] Completed train batch 1 generator forward 40.847 ms, 7.22 s total +[ 2023-09-21 08:23:49 ] Completed train batch 1 generator loss: 1.290 63.243 ms, 7.28 s total +[ 2023-09-21 08:23:50 ] Completed train batch 1 generator backward 971.942 ms, 8.25 s total +[ 2023-09-21 08:23:50 ] Completed train batch 1 discriminator loss 0.080 56.534 ms, 8.31 s total +[ 2023-09-21 08:23:50 ] Completed train batch 1 discriminator backward 72.372 ms, 8.38 s total +Epoch [78] Step [1/2], gen_loss: 0.440, disc_loss: 0.028 +[ 2023-09-21 08:23:50 ] Completed train batch 1 metrics update 0.658 ms, 8.38 s total +Saving checkpoint at epoch 78 train batch 1 +[ 2023-09-21 08:23:51 ] Completed saving temp checkpoint 1,045.206 ms, 9.43 s total +[ 2023-09-21 08:23:51 ] Completed replacing temp checkpoint with checkpoint 25.958 ms, 9.45 s total +Epoch [78] :: gen_loss: 0.434, disc_loss: 0.029 +[ 2023-09-21 08:23:51 ] Completed training generator for epoch 78 40.824 ms, 9.49 s total +[ 2023-09-21 08:23:51 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 78 from eval step 0 + +[ 2023-09-21 08:23:52 ] Completed eval batch 0 to device 1,242.060 ms, 1.24 s total +[ 2023-09-21 08:23:52 ] Completed eval batch 0 forward 29.650 ms, 1.27 s total +[ 2023-09-21 08:23:52 ] Completed eval batch 0 recons_loss 0.289 ms, 1.27 s total +[ 2023-09-21 08:23:52 ] Completed eval batch 0 metrics update 0.590 ms, 1.27 s total +Saving checkpoint at epoch 78 val batch 0 +[ 2023-09-21 08:23:53 ] Completed saving temp checkpoint 922.263 ms, 2.19 s total +[ 2023-09-21 08:23:53 ] Completed replacing temp checkpoint with checkpoint 22.613 ms, 2.22 s total +[ 2023-09-21 08:23:53 ] Completed eval batch 1 to device 14.402 ms, 2.23 s total +[ 2023-09-21 08:23:53 ] Completed eval batch 1 forward 23.463 ms, 2.26 s total +[ 2023-09-21 08:23:53 ] Completed eval batch 1 recons_loss 0.337 ms, 2.26 s total +[ 2023-09-21 08:23:53 ] Completed eval batch 1 metrics update 0.300 ms, 2.26 s total +Saving checkpoint at epoch 78 val batch 1 +Epoch 78 val loss: 0.0694 +[ 2023-09-21 08:23:54 ] Completed saving temp checkpoint 979.155 ms, 3.24 s total +[ 2023-09-21 08:23:54 ] Completed replacing temp checkpoint with checkpoint 26.288 ms, 3.26 s total +[ 2023-09-21 08:23:55 ] Completed evaluating generator for epoch 78 56.873 ms, 3.32 s total + + +EPOCH :: 79 + + +[ 2023-09-21 08:23:55 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 79 from training step 0 + +[ 2023-09-21 08:23:57 ] Completed train batch 0 to device 2,884.299 ms, 2.88 s total +[ 2023-09-21 08:23:57 ] Completed train batch 0 generator forward 29.973 ms, 2.91 s total +[ 2023-09-21 08:23:57 ] Completed train batch 0 generator loss: 1.306 65.165 ms, 2.98 s total +[ 2023-09-21 08:23:58 ] Completed train batch 0 generator backward 269.149 ms, 3.25 s total +[ 2023-09-21 08:23:58 ] Completed train batch 0 discriminator loss 0.083 57.814 ms, 3.31 s total +[ 2023-09-21 08:23:58 ] Completed train batch 0 discriminator backward 81.217 ms, 3.39 s total +Epoch [79] Step [0/2], gen_loss: 0.441, disc_loss: 0.028 +[ 2023-09-21 08:23:58 ] Completed train batch 0 metrics update 2.051 ms, 3.39 s total +Saving checkpoint at epoch 79 train batch 0 +[ 2023-09-21 08:23:59 ] Completed saving temp checkpoint 966.520 ms, 4.36 s total +[ 2023-09-21 08:23:59 ] Completed replacing temp checkpoint with checkpoint 21.086 ms, 4.38 s total +[ 2023-09-21 08:24:00 ] Completed train batch 1 to device 1,233.560 ms, 5.61 s total +[ 2023-09-21 08:24:00 ] Completed train batch 1 generator forward 40.966 ms, 5.65 s total +[ 2023-09-21 08:24:00 ] Completed train batch 1 generator loss: 1.326 63.604 ms, 5.72 s total +[ 2023-09-21 08:24:01 ] Completed train batch 1 generator backward 922.803 ms, 6.64 s total +[ 2023-09-21 08:24:01 ] Completed train batch 1 discriminator loss 0.077 57.170 ms, 6.70 s total +[ 2023-09-21 08:24:01 ] Completed train batch 1 discriminator backward 83.720 ms, 6.78 s total +Epoch [79] Step [1/2], gen_loss: 0.455, disc_loss: 0.026 +[ 2023-09-21 08:24:01 ] Completed train batch 1 metrics update 0.530 ms, 6.78 s total +Saving checkpoint at epoch 79 train batch 1 +[ 2023-09-21 08:24:02 ] Completed saving temp checkpoint 989.977 ms, 7.77 s total +[ 2023-09-21 08:24:02 ] Completed replacing temp checkpoint with checkpoint 38.489 ms, 7.81 s total +Epoch [79] :: gen_loss: 0.448, disc_loss: 0.027 +[ 2023-09-21 08:24:02 ] Completed training generator for epoch 79 53.137 ms, 7.86 s total +[ 2023-09-21 08:24:02 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 79 from eval step 0 + +[ 2023-09-21 08:24:04 ] Completed eval batch 0 to device 1,336.194 ms, 1.34 s total +[ 2023-09-21 08:24:04 ] Completed eval batch 0 forward 39.059 ms, 1.38 s total +[ 2023-09-21 08:24:04 ] Completed eval batch 0 recons_loss 0.226 ms, 1.38 s total +[ 2023-09-21 08:24:04 ] Completed eval batch 0 metrics update 0.516 ms, 1.38 s total +Saving checkpoint at epoch 79 val batch 0 +[ 2023-09-21 08:24:05 ] Completed saving temp checkpoint 919.627 ms, 2.30 s total +[ 2023-09-21 08:24:05 ] Completed replacing temp checkpoint with checkpoint 24.222 ms, 2.32 s total +[ 2023-09-21 08:24:05 ] Completed eval batch 1 to device 14.630 ms, 2.33 s total +[ 2023-09-21 08:24:05 ] Completed eval batch 1 forward 21.765 ms, 2.36 s total +[ 2023-09-21 08:24:05 ] Completed eval batch 1 recons_loss 0.349 ms, 2.36 s total +[ 2023-09-21 08:24:05 ] Completed eval batch 1 metrics update 0.325 ms, 2.36 s total +Saving checkpoint at epoch 79 val batch 1 +Epoch 79 val loss: 0.0730 +[ 2023-09-21 08:24:06 ] Completed saving temp checkpoint 981.806 ms, 3.34 s total +[ 2023-09-21 08:24:06 ] Completed replacing temp checkpoint with checkpoint 29.803 ms, 3.37 s total +[ 2023-09-21 08:24:06 ] Completed evaluating generator for epoch 79 55.828 ms, 3.42 s total + + +EPOCH :: 80 + + +[ 2023-09-21 08:24:06 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 80 from training step 0 + +[ 2023-09-21 08:24:09 ] Completed train batch 0 to device 3,560.598 ms, 3.56 s total +[ 2023-09-21 08:24:09 ] Completed train batch 0 generator forward 30.196 ms, 3.59 s total +[ 2023-09-21 08:24:09 ] Completed train batch 0 generator loss: 1.329 65.396 ms, 3.66 s total +[ 2023-09-21 08:24:10 ] Completed train batch 0 generator backward 144.300 ms, 3.80 s total +[ 2023-09-21 08:24:10 ] Completed train batch 0 discriminator loss 0.074 56.959 ms, 3.86 s total +[ 2023-09-21 08:24:10 ] Completed train batch 0 discriminator backward 93.631 ms, 3.95 s total +Epoch [80] Step [0/2], gen_loss: 0.432, disc_loss: 0.025 +[ 2023-09-21 08:24:10 ] Completed train batch 0 metrics update 4.695 ms, 3.96 s total +Saving checkpoint at epoch 80 train batch 0 +[ 2023-09-21 08:24:10 ] Completed saving temp checkpoint 392.269 ms, 4.35 s total +[ 2023-09-21 08:24:10 ] Completed replacing temp checkpoint with checkpoint 29.280 ms, 4.38 s total +[ 2023-09-21 08:24:12 ] Completed train batch 1 to device 2,197.854 ms, 6.58 s total +[ 2023-09-21 08:24:12 ] Completed train batch 1 generator forward 40.978 ms, 6.62 s total +[ 2023-09-21 08:24:12 ] Completed train batch 1 generator loss: 1.300 64.089 ms, 6.68 s total +[ 2023-09-21 08:24:13 ] Completed train batch 1 generator backward 125.874 ms, 6.81 s total +[ 2023-09-21 08:24:13 ] Completed train batch 1 discriminator loss 0.064 58.102 ms, 6.86 s total +[ 2023-09-21 08:24:13 ] Completed train batch 1 discriminator backward 74.156 ms, 6.94 s total +Epoch [80] Step [1/2], gen_loss: 0.450, disc_loss: 0.023 +[ 2023-09-21 08:24:13 ] Completed train batch 1 metrics update 5.055 ms, 6.94 s total +Saving checkpoint at epoch 80 train batch 1 +[ 2023-09-21 08:24:13 ] Completed saving temp checkpoint 429.206 ms, 7.37 s total +[ 2023-09-21 08:24:13 ] Completed replacing temp checkpoint with checkpoint 25.480 ms, 7.40 s total +Epoch [80] :: gen_loss: 0.441, disc_loss: 0.024 +[ 2023-09-21 08:24:13 ] Completed training generator for epoch 80 53.877 ms, 7.45 s total +[ 2023-09-21 08:24:13 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 80 from eval step 0 + +[ 2023-09-21 08:24:14 ] Completed eval batch 0 to device 1,155.643 ms, 1.16 s total +[ 2023-09-21 08:24:14 ] Completed eval batch 0 forward 25.243 ms, 1.18 s total +[ 2023-09-21 08:24:14 ] Completed eval batch 0 recons_loss 0.353 ms, 1.18 s total +[ 2023-09-21 08:24:14 ] Completed eval batch 0 metrics update 0.641 ms, 1.18 s total +Saving checkpoint at epoch 80 val batch 0 +[ 2023-09-21 08:24:15 ] Completed saving temp checkpoint 326.440 ms, 1.51 s total +[ 2023-09-21 08:24:15 ] Completed replacing temp checkpoint with checkpoint 33.278 ms, 1.54 s total +[ 2023-09-21 08:24:15 ] Completed eval batch 1 to device 538.413 ms, 2.08 s total +[ 2023-09-21 08:24:15 ] Completed eval batch 1 forward 23.520 ms, 2.10 s total +[ 2023-09-21 08:24:15 ] Completed eval batch 1 recons_loss 0.357 ms, 2.10 s total +[ 2023-09-21 08:24:16 ] Completed eval batch 1 metrics update 251.075 ms, 2.35 s total +Saving checkpoint at epoch 80 val batch 1 +Epoch 80 val loss: 0.0816 +[ 2023-09-21 08:24:16 ] Completed saving temp checkpoint 415.822 ms, 2.77 s total +[ 2023-09-21 08:24:16 ] Completed replacing temp checkpoint with checkpoint 20.535 ms, 2.79 s total +[ 2023-09-21 08:24:16 ] Completed evaluating generator for epoch 80 51.350 ms, 2.84 s total + + +EPOCH :: 81 + + +[ 2023-09-21 08:24:16 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 81 from training step 0 + +[ 2023-09-21 08:24:19 ] Completed train batch 0 to device 2,861.946 ms, 2.86 s total +[ 2023-09-21 08:24:19 ] Completed train batch 0 generator forward 25.791 ms, 2.89 s total +[ 2023-09-21 08:24:19 ] Completed train batch 0 generator loss: 1.349 76.612 ms, 2.96 s total +[ 2023-09-21 08:24:20 ] Completed train batch 0 generator backward 817.780 ms, 3.78 s total +[ 2023-09-21 08:24:20 ] Completed train batch 0 discriminator loss 0.069 57.783 ms, 3.84 s total +[ 2023-09-21 08:24:20 ] Completed train batch 0 discriminator backward 107.669 ms, 3.95 s total +Epoch [81] Step [0/2], gen_loss: 0.438, disc_loss: 0.023 +[ 2023-09-21 08:24:20 ] Completed train batch 0 metrics update 7.724 ms, 3.96 s total +Saving checkpoint at epoch 81 train batch 0 +[ 2023-09-21 08:24:20 ] Completed saving temp checkpoint 377.141 ms, 4.33 s total +[ 2023-09-21 08:24:20 ] Completed replacing temp checkpoint with checkpoint 22.075 ms, 4.35 s total +[ 2023-09-21 08:24:22 ] Completed train batch 1 to device 1,294.767 ms, 5.65 s total +[ 2023-09-21 08:24:22 ] Completed train batch 1 generator forward 41.925 ms, 5.69 s total +[ 2023-09-21 08:24:22 ] Completed train batch 1 generator loss: 1.370 63.254 ms, 5.75 s total +[ 2023-09-21 08:24:23 ] Completed train batch 1 generator backward 1,463.059 ms, 7.22 s total +[ 2023-09-21 08:24:23 ] Completed train batch 1 discriminator loss 0.067 56.588 ms, 7.27 s total +[ 2023-09-21 08:24:23 ] Completed train batch 1 discriminator backward 76.091 ms, 7.35 s total +Epoch [81] Step [1/2], gen_loss: 0.449, disc_loss: 0.023 +[ 2023-09-21 08:24:23 ] Completed train batch 1 metrics update 5.704 ms, 7.36 s total +Saving checkpoint at epoch 81 train batch 1 +[ 2023-09-21 08:24:24 ] Completed saving temp checkpoint 365.515 ms, 7.72 s total +[ 2023-09-21 08:24:24 ] Completed replacing temp checkpoint with checkpoint 19.259 ms, 7.74 s total +Epoch [81] :: gen_loss: 0.443, disc_loss: 0.023 +[ 2023-09-21 08:24:24 ] Completed training generator for epoch 81 53.969 ms, 7.79 s total +[ 2023-09-21 08:24:24 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 81 from eval step 0 + +[ 2023-09-21 08:24:25 ] Completed eval batch 0 to device 1,209.739 ms, 1.21 s total +[ 2023-09-21 08:24:25 ] Completed eval batch 0 forward 24.842 ms, 1.23 s total +[ 2023-09-21 08:24:25 ] Completed eval batch 0 recons_loss 0.328 ms, 1.23 s total +[ 2023-09-21 08:24:25 ] Completed eval batch 0 metrics update 0.632 ms, 1.24 s total +Saving checkpoint at epoch 81 val batch 0 +[ 2023-09-21 08:24:25 ] Completed saving temp checkpoint 326.066 ms, 1.56 s total +[ 2023-09-21 08:24:25 ] Completed replacing temp checkpoint with checkpoint 19.708 ms, 1.58 s total +[ 2023-09-21 08:24:26 ] Completed eval batch 1 to device 650.834 ms, 2.23 s total +[ 2023-09-21 08:24:26 ] Completed eval batch 1 forward 22.894 ms, 2.26 s total +[ 2023-09-21 08:24:26 ] Completed eval batch 1 recons_loss 0.443 ms, 2.26 s total +[ 2023-09-21 08:24:26 ] Completed eval batch 1 metrics update 47.785 ms, 2.30 s total +Saving checkpoint at epoch 81 val batch 1 +Epoch 81 val loss: 0.0696 +[ 2023-09-21 08:24:27 ] Completed saving temp checkpoint 428.947 ms, 2.73 s total +[ 2023-09-21 08:24:27 ] Completed replacing temp checkpoint with checkpoint 27.182 ms, 2.76 s total +[ 2023-09-21 08:24:27 ] Completed evaluating generator for epoch 81 53.409 ms, 2.81 s total + + +EPOCH :: 82 + + +[ 2023-09-21 08:24:27 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 82 from training step 0 + +[ 2023-09-21 08:24:30 ] Completed train batch 0 to device 2,839.534 ms, 2.84 s total +[ 2023-09-21 08:24:30 ] Completed train batch 0 generator forward 44.247 ms, 2.88 s total +[ 2023-09-21 08:24:30 ] Completed train batch 0 generator loss: 1.280 66.084 ms, 2.95 s total +[ 2023-09-21 08:24:30 ] Completed train batch 0 generator backward 815.233 ms, 3.77 s total +[ 2023-09-21 08:24:31 ] Completed train batch 0 discriminator loss 0.067 57.360 ms, 3.82 s total +[ 2023-09-21 08:24:31 ] Completed train batch 0 discriminator backward 84.333 ms, 3.91 s total +Epoch [82] Step [0/2], gen_loss: 0.439, disc_loss: 0.023 +[ 2023-09-21 08:24:31 ] Completed train batch 0 metrics update 3.811 ms, 3.91 s total +Saving checkpoint at epoch 82 train batch 0 +[ 2023-09-21 08:24:31 ] Completed saving temp checkpoint 395.677 ms, 4.31 s total +[ 2023-09-21 08:24:31 ] Completed replacing temp checkpoint with checkpoint 24.729 ms, 4.33 s total +[ 2023-09-21 08:24:32 ] Completed train batch 1 to device 1,139.682 ms, 5.47 s total +[ 2023-09-21 08:24:32 ] Completed train batch 1 generator forward 41.393 ms, 5.51 s total +[ 2023-09-21 08:24:32 ] Completed train batch 1 generator loss: 1.361 62.215 ms, 5.57 s total +[ 2023-09-21 08:24:34 ] Completed train batch 1 generator backward 1,686.891 ms, 7.26 s total +[ 2023-09-21 08:24:34 ] Completed train batch 1 discriminator loss 0.073 56.631 ms, 7.32 s total +[ 2023-09-21 08:24:34 ] Completed train batch 1 discriminator backward 71.914 ms, 7.39 s total +Epoch [82] Step [1/2], gen_loss: 0.443, disc_loss: 0.022 +[ 2023-09-21 08:24:34 ] Completed train batch 1 metrics update 1.993 ms, 7.39 s total +Saving checkpoint at epoch 82 train batch 1 +[ 2023-09-21 08:24:35 ] Completed saving temp checkpoint 428.372 ms, 7.82 s total +[ 2023-09-21 08:24:35 ] Completed replacing temp checkpoint with checkpoint 30.879 ms, 7.85 s total +Epoch [82] :: gen_loss: 0.441, disc_loss: 0.023 +[ 2023-09-21 08:24:35 ] Completed training generator for epoch 82 53.851 ms, 7.90 s total +[ 2023-09-21 08:24:35 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 82 from eval step 0 + +[ 2023-09-21 08:24:36 ] Completed eval batch 0 to device 1,081.710 ms, 1.08 s total +[ 2023-09-21 08:24:36 ] Completed eval batch 0 forward 23.689 ms, 1.11 s total +[ 2023-09-21 08:24:36 ] Completed eval batch 0 recons_loss 0.272 ms, 1.11 s total +[ 2023-09-21 08:24:36 ] Completed eval batch 0 metrics update 0.548 ms, 1.11 s total +Saving checkpoint at epoch 82 val batch 0 +[ 2023-09-21 08:24:36 ] Completed saving temp checkpoint 380.521 ms, 1.49 s total +[ 2023-09-21 08:24:36 ] Completed replacing temp checkpoint with checkpoint 29.702 ms, 1.52 s total +[ 2023-09-21 08:24:37 ] Completed eval batch 1 to device 461.535 ms, 1.98 s total +[ 2023-09-21 08:24:37 ] Completed eval batch 1 forward 21.476 ms, 2.00 s total +[ 2023-09-21 08:24:37 ] Completed eval batch 1 recons_loss 0.365 ms, 2.00 s total +[ 2023-09-21 08:24:37 ] Completed eval batch 1 metrics update 221.963 ms, 2.22 s total +Saving checkpoint at epoch 82 val batch 1 +Epoch 82 val loss: 0.0634 +[ 2023-09-21 08:24:37 ] Completed saving temp checkpoint 425.685 ms, 2.65 s total +[ 2023-09-21 08:24:37 ] Completed replacing temp checkpoint with checkpoint 31.340 ms, 2.68 s total +[ 2023-09-21 08:24:37 ] Completed evaluating generator for epoch 82 47.623 ms, 2.73 s total + + +EPOCH :: 83 + + +[ 2023-09-21 08:24:37 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 83 from training step 0 + +[ 2023-09-21 08:24:40 ] Completed train batch 0 to device 3,012.115 ms, 3.01 s total +[ 2023-09-21 08:24:40 ] Completed train batch 0 generator forward 44.323 ms, 3.06 s total +[ 2023-09-21 08:24:40 ] Completed train batch 0 generator loss: 1.388 66.317 ms, 3.12 s total +[ 2023-09-21 08:24:41 ] Completed train batch 0 generator backward 613.751 ms, 3.74 s total +[ 2023-09-21 08:24:41 ] Completed train batch 0 discriminator loss 0.069 57.614 ms, 3.79 s total +[ 2023-09-21 08:24:41 ] Completed train batch 0 discriminator backward 104.452 ms, 3.90 s total +Epoch [83] Step [0/2], gen_loss: 0.453, disc_loss: 0.022 +[ 2023-09-21 08:24:41 ] Completed train batch 0 metrics update 16.740 ms, 3.92 s total +Saving checkpoint at epoch 83 train batch 0 +[ 2023-09-21 08:24:42 ] Completed saving temp checkpoint 438.755 ms, 4.35 s total +[ 2023-09-21 08:24:42 ] Completed replacing temp checkpoint with checkpoint 22.632 ms, 4.38 s total +[ 2023-09-21 08:24:43 ] Completed train batch 1 to device 1,595.004 ms, 5.97 s total +[ 2023-09-21 08:24:43 ] Completed train batch 1 generator forward 22.069 ms, 5.99 s total +[ 2023-09-21 08:24:43 ] Completed train batch 1 generator loss: 1.346 63.175 ms, 6.06 s total +[ 2023-09-21 08:24:44 ] Completed train batch 1 generator backward 943.995 ms, 7.00 s total +[ 2023-09-21 08:24:44 ] Completed train batch 1 discriminator loss 0.065 56.361 ms, 7.06 s total +[ 2023-09-21 08:24:44 ] Completed train batch 1 discriminator backward 88.012 ms, 7.15 s total +Epoch [83] Step [1/2], gen_loss: 0.430, disc_loss: 0.022 +[ 2023-09-21 08:24:44 ] Completed train batch 1 metrics update 5.284 ms, 7.15 s total +Saving checkpoint at epoch 83 train batch 1 +[ 2023-09-21 08:24:45 ] Completed saving temp checkpoint 491.947 ms, 7.64 s total +[ 2023-09-21 08:24:45 ] Completed replacing temp checkpoint with checkpoint 32.319 ms, 7.67 s total +Epoch [83] :: gen_loss: 0.442, disc_loss: 0.022 +[ 2023-09-21 08:24:45 ] Completed training generator for epoch 83 63.397 ms, 7.74 s total +[ 2023-09-21 08:24:45 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 83 from eval step 0 + +[ 2023-09-21 08:24:46 ] Completed eval batch 0 to device 1,097.869 ms, 1.10 s total +[ 2023-09-21 08:24:46 ] Completed eval batch 0 forward 24.549 ms, 1.12 s total +[ 2023-09-21 08:24:46 ] Completed eval batch 0 recons_loss 0.303 ms, 1.12 s total +[ 2023-09-21 08:24:46 ] Completed eval batch 0 metrics update 0.552 ms, 1.12 s total +Saving checkpoint at epoch 83 val batch 0 +[ 2023-09-21 08:24:47 ] Completed saving temp checkpoint 491.824 ms, 1.62 s total +[ 2023-09-21 08:24:47 ] Completed replacing temp checkpoint with checkpoint 28.285 ms, 1.64 s total +[ 2023-09-21 08:24:47 ] Completed eval batch 1 to device 315.130 ms, 1.96 s total +[ 2023-09-21 08:24:47 ] Completed eval batch 1 forward 39.689 ms, 2.00 s total +[ 2023-09-21 08:24:47 ] Completed eval batch 1 recons_loss 0.566 ms, 2.00 s total +[ 2023-09-21 08:24:47 ] Completed eval batch 1 metrics update 67.483 ms, 2.07 s total +Saving checkpoint at epoch 83 val batch 1 +Epoch 83 val loss: 0.0621 +[ 2023-09-21 08:24:48 ] Completed saving temp checkpoint 541.697 ms, 2.61 s total +[ 2023-09-21 08:24:48 ] Completed replacing temp checkpoint with checkpoint 30.848 ms, 2.64 s total +[ 2023-09-21 08:24:48 ] Completed evaluating generator for epoch 83 72.205 ms, 2.71 s total + + +EPOCH :: 84 + + +[ 2023-09-21 08:24:48 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 84 from training step 0 + +[ 2023-09-21 08:24:51 ] Completed train batch 0 to device 3,071.151 ms, 3.07 s total +[ 2023-09-21 08:24:51 ] Completed train batch 0 generator forward 26.905 ms, 3.10 s total +[ 2023-09-21 08:24:51 ] Completed train batch 0 generator loss: 1.305 66.369 ms, 3.16 s total +[ 2023-09-21 08:24:51 ] Completed train batch 0 generator backward 502.222 ms, 3.67 s total +[ 2023-09-21 08:24:51 ] Completed train batch 0 discriminator loss 0.064 59.061 ms, 3.73 s total +[ 2023-09-21 08:24:52 ] Completed train batch 0 discriminator backward 74.967 ms, 3.80 s total +Epoch [84] Step [0/2], gen_loss: 0.434, disc_loss: 0.022 +[ 2023-09-21 08:24:52 ] Completed train batch 0 metrics update 5.606 ms, 3.81 s total +Saving checkpoint at epoch 84 train batch 0 +[ 2023-09-21 08:24:52 ] Completed saving temp checkpoint 449.948 ms, 4.26 s total +[ 2023-09-21 08:24:52 ] Completed replacing temp checkpoint with checkpoint 29.964 ms, 4.29 s total +[ 2023-09-21 08:24:54 ] Completed train batch 1 to device 1,714.652 ms, 6.00 s total +[ 2023-09-21 08:24:54 ] Completed train batch 1 generator forward 41.117 ms, 6.04 s total +[ 2023-09-21 08:24:54 ] Completed train batch 1 generator loss: 1.284 64.494 ms, 6.11 s total +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 08:37:11 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 08:37:11 ] Completed importing Timer 0.023 ms, 0.00 s total +[ 2023-09-21 08:37:15 ] Completed importing everything else 4,560.760 ms, 4.56 s total +| distributed init (rank 1): env:// +| distributed init (rank 4): env:// +| distributed init (rank 2): env:// +| distributed init (rank 3): env:// +| distributed init (rank 5): env:// +| distributed init (rank 0): env:// +[ 2023-09-21 08:37:23 ] Completed preliminaries 8,000.092 ms, 12.56 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 08:37:23 ] Completed build datasets 12.248 ms, 12.57 s total +[ 2023-09-21 08:37:23 ] Completed build samplers 0.048 ms, 12.57 s total +[ 2023-09-21 08:37:23 ] Completed build dataloaders 0.128 ms, 12.57 s total +[ 2023-09-21 08:37:24 ] Completed generator to device 98.323 ms, 12.67 s total +[ 2023-09-21 08:37:24 ] Completed discriminator to device 7.038 ms, 12.68 s total +[ 2023-09-21 08:37:24 ] Completed loss functions 387.323 ms, 13.07 s total +[ 2023-09-21 08:37:24 ] Completed models prepped for distribution 79.678 ms, 13.15 s total +[ 2023-09-21 08:37:24 ] Completed optimizers 0.649 ms, 13.15 s total +[ 2023-09-21 08:37:24 ] Completed grad scalers 0.021 ms, 13.15 s total +[ 2023-09-21 08:37:25 ] Completed checkpoint retrieval 456.307 ms, 13.60 s total + + +EPOCH :: 84 + + +[ 2023-09-21 08:37:25 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 84 from training step 1 + +[ 2023-09-21 08:37:27 ] Completed train batch 1 to device 2,969.141 ms, 2.97 s total +[ 2023-09-21 08:37:29 ] Completed train batch 1 generator forward 1,053.736 ms, 4.02 s total +[ 2023-09-21 08:37:29 ] Completed train batch 1 generator loss: 1.283 72.949 ms, 4.10 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 08:37:30 ] Completed train batch 1 generator backward 1,528.393 ms, 5.62 s total +[ 2023-09-21 08:37:30 ] Completed train batch 1 discriminator loss 0.069 65.458 ms, 5.69 s total +[ 2023-09-21 08:37:30 ] Completed train batch 1 discriminator backward 60.379 ms, 5.75 s total +Epoch [84] Step [1/2], gen_loss: 0.434, disc_loss: 0.022 +[ 2023-09-21 08:37:30 ] Completed train batch 1 metrics update 5.680 ms, 5.76 s total +Saving checkpoint at epoch 84 train batch 1 +[ 2023-09-21 08:37:31 ] Completed saving temp checkpoint 456.080 ms, 6.21 s total +[ 2023-09-21 08:37:31 ] Completed replacing temp checkpoint with checkpoint 140.221 ms, 6.35 s total +Epoch [84] :: gen_loss: 0.434, disc_loss: 0.022 +[ 2023-09-21 08:37:31 ] Completed training generator for epoch 84 41.081 ms, 6.39 s total +[ 2023-09-21 08:37:31 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 84 from eval step 0 + +[ 2023-09-21 08:37:32 ] Completed eval batch 0 to device 1,084.780 ms, 1.08 s total +[ 2023-09-21 08:37:32 ] Completed eval batch 0 forward 32.321 ms, 1.12 s total +[ 2023-09-21 08:37:32 ] Completed eval batch 0 recons_loss 0.394 ms, 1.12 s total +[ 2023-09-21 08:37:32 ] Completed eval batch 0 metrics update 0.730 ms, 1.12 s total +Saving checkpoint at epoch 84 val batch 0 +[ 2023-09-21 08:37:32 ] Completed saving temp checkpoint 423.445 ms, 1.54 s total +[ 2023-09-21 08:37:32 ] Completed replacing temp checkpoint with checkpoint 26.338 ms, 1.57 s total +[ 2023-09-21 08:37:33 ] Completed eval batch 1 to device 445.819 ms, 2.01 s total +[ 2023-09-21 08:37:33 ] Completed eval batch 1 forward 39.346 ms, 2.05 s total +[ 2023-09-21 08:37:33 ] Completed eval batch 1 recons_loss 0.342 ms, 2.05 s total +[ 2023-09-21 08:37:33 ] Completed eval batch 1 metrics update 0.310 ms, 2.05 s total +Saving checkpoint at epoch 84 val batch 1 +Epoch 84 val loss: 0.0690 +[ 2023-09-21 08:37:33 ] Completed saving temp checkpoint 445.363 ms, 2.50 s total +[ 2023-09-21 08:37:33 ] Completed replacing temp checkpoint with checkpoint 18.873 ms, 2.52 s total +[ 2023-09-21 08:37:33 ] Completed evaluating generator for epoch 84 56.829 ms, 2.57 s total + + +EPOCH :: 85 + + +[ 2023-09-21 08:37:33 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 85 from training step 0 + +[ 2023-09-21 08:37:36 ] Completed train batch 0 to device 3,002.618 ms, 3.00 s total +[ 2023-09-21 08:37:37 ] Completed train batch 0 generator forward 32.380 ms, 3.03 s total +[ 2023-09-21 08:37:37 ] Completed train batch 0 generator loss: 1.246 75.447 ms, 3.11 s total +[ 2023-09-21 08:37:37 ] Completed train batch 0 generator backward 593.391 ms, 3.70 s total +[ 2023-09-21 08:37:37 ] Completed train batch 0 discriminator loss 0.064 56.687 ms, 3.76 s total +[ 2023-09-21 08:37:37 ] Completed train batch 0 discriminator backward 95.863 ms, 3.86 s total +Epoch [85] Step [0/2], gen_loss: 0.434, disc_loss: 0.022 +[ 2023-09-21 08:37:37 ] Completed train batch 0 metrics update 3.381 ms, 3.86 s total +Saving checkpoint at epoch 85 train batch 0 +[ 2023-09-21 08:37:38 ] Completed saving temp checkpoint 395.102 ms, 4.25 s total +[ 2023-09-21 08:37:38 ] Completed replacing temp checkpoint with checkpoint 21.464 ms, 4.28 s total +[ 2023-09-21 08:37:39 ] Completed train batch 1 to device 1,571.264 ms, 5.85 s total +[ 2023-09-21 08:37:39 ] Completed train batch 1 generator forward 22.235 ms, 5.87 s total +[ 2023-09-21 08:37:39 ] Completed train batch 1 generator loss: 1.313 62.679 ms, 5.93 s total +[ 2023-09-21 08:37:41 ] Completed train batch 1 generator backward 1,238.011 ms, 7.17 s total +[ 2023-09-21 08:37:41 ] Completed train batch 1 discriminator loss 0.065 56.200 ms, 7.23 s total +[ 2023-09-21 08:37:41 ] Completed train batch 1 discriminator backward 72.238 ms, 7.30 s total +Epoch [85] Step [1/2], gen_loss: 0.432, disc_loss: 0.023 +[ 2023-09-21 08:37:41 ] Completed train batch 1 metrics update 4.912 ms, 7.30 s total +Saving checkpoint at epoch 85 train batch 1 +[ 2023-09-21 08:37:41 ] Completed saving temp checkpoint 435.985 ms, 7.74 s total +[ 2023-09-21 08:37:41 ] Completed replacing temp checkpoint with checkpoint 29.507 ms, 7.77 s total +Epoch [85] :: gen_loss: 0.433, disc_loss: 0.023 +[ 2023-09-21 08:37:41 ] Completed training generator for epoch 85 73.936 ms, 7.84 s total +[ 2023-09-21 08:37:41 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 85 from eval step 0 + +[ 2023-09-21 08:37:42 ] Completed eval batch 0 to device 1,108.904 ms, 1.11 s total +[ 2023-09-21 08:37:42 ] Completed eval batch 0 forward 24.962 ms, 1.13 s total +[ 2023-09-21 08:37:42 ] Completed eval batch 0 recons_loss 0.306 ms, 1.13 s total +[ 2023-09-21 08:37:42 ] Completed eval batch 0 metrics update 0.587 ms, 1.13 s total +Saving checkpoint at epoch 85 val batch 0 +[ 2023-09-21 08:37:43 ] Completed saving temp checkpoint 426.435 ms, 1.56 s total +[ 2023-09-21 08:37:43 ] Completed replacing temp checkpoint with checkpoint 27.424 ms, 1.59 s total +[ 2023-09-21 08:37:43 ] Completed eval batch 1 to device 376.246 ms, 1.96 s total +[ 2023-09-21 08:37:43 ] Completed eval batch 1 forward 39.468 ms, 2.00 s total +[ 2023-09-21 08:37:43 ] Completed eval batch 1 recons_loss 0.567 ms, 2.00 s total +[ 2023-09-21 08:37:43 ] Completed eval batch 1 metrics update 114.613 ms, 2.12 s total +Saving checkpoint at epoch 85 val batch 1 +Epoch 85 val loss: 0.0624 +[ 2023-09-21 08:37:44 ] Completed saving temp checkpoint 429.945 ms, 2.55 s total +[ 2023-09-21 08:37:44 ] Completed replacing temp checkpoint with checkpoint 31.621 ms, 2.58 s total +[ 2023-09-21 08:37:44 ] Completed evaluating generator for epoch 85 52.873 ms, 2.63 s total + + +EPOCH :: 86 + + +[ 2023-09-21 08:37:44 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 86 from training step 0 + +[ 2023-09-21 08:37:47 ] Completed train batch 0 to device 3,002.154 ms, 3.00 s total +[ 2023-09-21 08:37:47 ] Completed train batch 0 generator forward 44.928 ms, 3.05 s total +[ 2023-09-21 08:37:47 ] Completed train batch 0 generator loss: 1.232 65.435 ms, 3.11 s total +[ 2023-09-21 08:37:48 ] Completed train batch 0 generator backward 608.948 ms, 3.72 s total +[ 2023-09-21 08:37:48 ] Completed train batch 0 discriminator loss 0.067 56.595 ms, 3.78 s total +[ 2023-09-21 08:37:48 ] Completed train batch 0 discriminator backward 100.503 ms, 3.88 s total +Epoch [86] Step [0/2], gen_loss: 0.427, disc_loss: 0.022 +[ 2023-09-21 08:37:48 ] Completed train batch 0 metrics update 5.914 ms, 3.88 s total +Saving checkpoint at epoch 86 train batch 0 +[ 2023-09-21 08:37:48 ] Completed saving temp checkpoint 461.155 ms, 4.35 s total +[ 2023-09-21 08:37:48 ] Completed replacing temp checkpoint with checkpoint 32.349 ms, 4.38 s total +[ 2023-09-21 08:37:50 ] Completed train batch 1 to device 1,369.635 ms, 5.75 s total +[ 2023-09-21 08:37:50 ] Completed train batch 1 generator forward 40.617 ms, 5.79 s total +[ 2023-09-21 08:37:50 ] Completed train batch 1 generator loss: 1.168 62.415 ms, 5.85 s total +[ 2023-09-21 08:37:51 ] Completed train batch 1 generator backward 1,379.927 ms, 7.23 s total +[ 2023-09-21 08:37:51 ] Completed train batch 1 discriminator loss 0.064 55.839 ms, 7.29 s total +[ 2023-09-21 08:37:51 ] Completed train batch 1 discriminator backward 79.100 ms, 7.37 s total +Epoch [86] Step [1/2], gen_loss: 0.421, disc_loss: 0.022 +[ 2023-09-21 08:37:51 ] Completed train batch 1 metrics update 5.673 ms, 7.37 s total +Saving checkpoint at epoch 86 train batch 1 +[ 2023-09-21 08:37:52 ] Completed saving temp checkpoint 420.687 ms, 7.79 s total +[ 2023-09-21 08:37:52 ] Completed replacing temp checkpoint with checkpoint 20.134 ms, 7.81 s total +Epoch [86] :: gen_loss: 0.424, disc_loss: 0.022 +[ 2023-09-21 08:37:52 ] Completed training generator for epoch 86 67.529 ms, 7.88 s total +[ 2023-09-21 08:37:52 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 86 from eval step 0 + +[ 2023-09-21 08:37:53 ] Completed eval batch 0 to device 1,112.662 ms, 1.11 s total +[ 2023-09-21 08:37:53 ] Completed eval batch 0 forward 35.351 ms, 1.15 s total +[ 2023-09-21 08:37:53 ] Completed eval batch 0 recons_loss 0.360 ms, 1.15 s total +[ 2023-09-21 08:37:53 ] Completed eval batch 0 metrics update 0.692 ms, 1.15 s total +Saving checkpoint at epoch 86 val batch 0 +[ 2023-09-21 08:37:53 ] Completed saving temp checkpoint 346.958 ms, 1.50 s total +[ 2023-09-21 08:37:53 ] Completed replacing temp checkpoint with checkpoint 26.376 ms, 1.52 s total +[ 2023-09-21 08:37:54 ] Completed eval batch 1 to device 532.656 ms, 2.06 s total +[ 2023-09-21 08:37:54 ] Completed eval batch 1 forward 40.455 ms, 2.10 s total +[ 2023-09-21 08:37:54 ] Completed eval batch 1 recons_loss 0.627 ms, 2.10 s total +[ 2023-09-21 08:37:54 ] Completed eval batch 1 metrics update 56.625 ms, 2.15 s total +Saving checkpoint at epoch 86 val batch 1 +Epoch 86 val loss: 0.0551 +[ 2023-09-21 08:37:54 ] Completed saving temp checkpoint 385.306 ms, 2.54 s total +[ 2023-09-21 08:37:54 ] Completed replacing temp checkpoint with checkpoint 18.097 ms, 2.56 s total +[ 2023-09-21 08:37:54 ] Completed evaluating generator for epoch 86 65.376 ms, 2.62 s total + + +EPOCH :: 87 + + +[ 2023-09-21 08:37:54 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 87 from training step 0 + +[ 2023-09-21 08:37:58 ] Completed train batch 0 to device 3,057.718 ms, 3.06 s total +[ 2023-09-21 08:37:58 ] Completed train batch 0 generator forward 40.051 ms, 3.10 s total +[ 2023-09-21 08:37:58 ] Completed train batch 0 generator loss: 1.276 68.036 ms, 3.17 s total +[ 2023-09-21 08:37:58 ] Completed train batch 0 generator backward 518.593 ms, 3.68 s total +[ 2023-09-21 08:37:58 ] Completed train batch 0 discriminator loss 0.069 61.354 ms, 3.75 s total +[ 2023-09-21 08:37:58 ] Completed train batch 0 discriminator backward 79.619 ms, 3.83 s total +Epoch [87] Step [0/2], gen_loss: 0.428, disc_loss: 0.023 +[ 2023-09-21 08:37:58 ] Completed train batch 0 metrics update 3.643 ms, 3.83 s total +Saving checkpoint at epoch 87 train batch 0 +[ 2023-09-21 08:37:59 ] Completed saving temp checkpoint 437.393 ms, 4.27 s total +[ 2023-09-21 08:37:59 ] Completed replacing temp checkpoint with checkpoint 23.168 ms, 4.29 s total +[ 2023-09-21 08:38:00 ] Completed train batch 1 to device 1,476.512 ms, 5.77 s total +[ 2023-09-21 08:38:00 ] Completed train batch 1 generator forward 41.139 ms, 5.81 s total +[ 2023-09-21 08:38:00 ] Completed train batch 1 generator loss: 1.257 63.657 ms, 5.87 s total +[ 2023-09-21 08:38:02 ] Completed train batch 1 generator backward 1,227.927 ms, 7.10 s total +[ 2023-09-21 08:38:02 ] Completed train batch 1 discriminator loss 0.071 55.779 ms, 7.15 s total +[ 2023-09-21 08:38:02 ] Completed train batch 1 discriminator backward 69.511 ms, 7.22 s total +Epoch [87] Step [1/2], gen_loss: 0.414, disc_loss: 0.025 +[ 2023-09-21 08:38:02 ] Completed train batch 1 metrics update 2.022 ms, 7.23 s total +Saving checkpoint at epoch 87 train batch 1 +[ 2023-09-21 08:38:02 ] Completed saving temp checkpoint 434.943 ms, 7.66 s total +[ 2023-09-21 08:38:02 ] Completed replacing temp checkpoint with checkpoint 22.395 ms, 7.68 s total +Epoch [87] :: gen_loss: 0.421, disc_loss: 0.024 +[ 2023-09-21 08:38:02 ] Completed training generator for epoch 87 63.161 ms, 7.75 s total +[ 2023-09-21 08:38:02 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 87 from eval step 0 + +[ 2023-09-21 08:38:03 ] Completed eval batch 0 to device 1,128.317 ms, 1.13 s total +[ 2023-09-21 08:38:03 ] Completed eval batch 0 forward 31.858 ms, 1.16 s total +[ 2023-09-21 08:38:03 ] Completed eval batch 0 recons_loss 0.340 ms, 1.16 s total +[ 2023-09-21 08:38:03 ] Completed eval batch 0 metrics update 0.624 ms, 1.16 s total +Saving checkpoint at epoch 87 val batch 0 +[ 2023-09-21 08:38:04 ] Completed saving temp checkpoint 410.998 ms, 1.57 s total +[ 2023-09-21 08:38:04 ] Completed replacing temp checkpoint with checkpoint 25.324 ms, 1.60 s total +[ 2023-09-21 08:38:04 ] Completed eval batch 1 to device 484.901 ms, 2.08 s total +[ 2023-09-21 08:38:04 ] Completed eval batch 1 forward 21.453 ms, 2.10 s total +[ 2023-09-21 08:38:04 ] Completed eval batch 1 recons_loss 0.341 ms, 2.10 s total +[ 2023-09-21 08:38:04 ] Completed eval batch 1 metrics update 90.751 ms, 2.19 s total +Saving checkpoint at epoch 87 val batch 1 +Epoch 87 val loss: 0.0602 +[ 2023-09-21 08:38:05 ] Completed saving temp checkpoint 404.916 ms, 2.60 s total +[ 2023-09-21 08:38:05 ] Completed replacing temp checkpoint with checkpoint 20.038 ms, 2.62 s total +[ 2023-09-21 08:38:05 ] Completed evaluating generator for epoch 87 49.208 ms, 2.67 s total + + +EPOCH :: 88 + + +[ 2023-09-21 08:38:05 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 88 from training step 0 + +[ 2023-09-21 08:38:08 ] Completed train batch 0 to device 2,889.790 ms, 2.89 s total +[ 2023-09-21 08:38:08 ] Completed train batch 0 generator forward 25.395 ms, 2.92 s total +[ 2023-09-21 08:38:08 ] Completed train batch 0 generator loss: 1.266 63.676 ms, 2.98 s total +[ 2023-09-21 08:38:08 ] Completed train batch 0 generator backward 608.918 ms, 3.59 s total +[ 2023-09-21 08:38:09 ] Completed train batch 0 discriminator loss 0.072 79.180 ms, 3.67 s total +[ 2023-09-21 08:38:09 ] Completed train batch 0 discriminator backward 78.404 ms, 3.75 s total +Epoch [88] Step [0/2], gen_loss: 0.420, disc_loss: 0.024 +[ 2023-09-21 08:38:09 ] Completed train batch 0 metrics update 24.650 ms, 3.77 s total +Saving checkpoint at epoch 88 train batch 0 +[ 2023-09-21 08:38:09 ] Completed saving temp checkpoint 405.534 ms, 4.18 s total +[ 2023-09-21 08:38:09 ] Completed replacing temp checkpoint with checkpoint 35.664 ms, 4.21 s total +[ 2023-09-21 08:38:10 ] Completed train batch 1 to device 1,297.992 ms, 5.51 s total +[ 2023-09-21 08:38:10 ] Completed train batch 1 generator forward 42.154 ms, 5.55 s total +[ 2023-09-21 08:38:10 ] Completed train batch 1 generator loss: 1.224 62.226 ms, 5.61 s total +[ 2023-09-21 08:38:12 ] Completed train batch 1 generator backward 1,368.258 ms, 6.98 s total +[ 2023-09-21 08:38:12 ] Completed train batch 1 discriminator loss 0.069 56.244 ms, 7.04 s total +[ 2023-09-21 08:38:12 ] Completed train batch 1 discriminator backward 91.421 ms, 7.13 s total +Epoch [88] Step [1/2], gen_loss: 0.426, disc_loss: 0.023 +[ 2023-09-21 08:38:12 ] Completed train batch 1 metrics update 0.410 ms, 7.13 s total +Saving checkpoint at epoch 88 train batch 1 +[ 2023-09-21 08:38:12 ] Completed saving temp checkpoint 480.104 ms, 7.61 s total +[ 2023-09-21 08:38:13 ] Completed replacing temp checkpoint with checkpoint 28.960 ms, 7.64 s total +Epoch [88] :: gen_loss: 0.423, disc_loss: 0.023 +[ 2023-09-21 08:38:13 ] Completed training generator for epoch 88 79.437 ms, 7.72 s total +[ 2023-09-21 08:38:13 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 88 from eval step 0 + +[ 2023-09-21 08:38:14 ] Completed eval batch 0 to device 1,038.056 ms, 1.04 s total +[ 2023-09-21 08:38:14 ] Completed eval batch 0 forward 25.152 ms, 1.06 s total +[ 2023-09-21 08:38:14 ] Completed eval batch 0 recons_loss 0.307 ms, 1.06 s total +[ 2023-09-21 08:38:14 ] Completed eval batch 0 metrics update 0.585 ms, 1.06 s total +Saving checkpoint at epoch 88 val batch 0 +[ 2023-09-21 08:38:14 ] Completed saving temp checkpoint 424.186 ms, 1.49 s total +[ 2023-09-21 08:38:14 ] Completed replacing temp checkpoint with checkpoint 34.263 ms, 1.52 s total +[ 2023-09-21 08:38:14 ] Completed eval batch 1 to device 327.003 ms, 1.85 s total +[ 2023-09-21 08:38:14 ] Completed eval batch 1 forward 21.470 ms, 1.87 s total +[ 2023-09-21 08:38:14 ] Completed eval batch 1 recons_loss 0.337 ms, 1.87 s total +[ 2023-09-21 08:38:15 ] Completed eval batch 1 metrics update 260.805 ms, 2.13 s total +Saving checkpoint at epoch 88 val batch 1 +Epoch 88 val loss: 0.0691 +[ 2023-09-21 08:38:15 ] Completed saving temp checkpoint 439.432 ms, 2.57 s total +[ 2023-09-21 08:38:15 ] Completed replacing temp checkpoint with checkpoint 24.629 ms, 2.60 s total +[ 2023-09-21 08:38:15 ] Completed evaluating generator for epoch 88 51.939 ms, 2.65 s total + + +EPOCH :: 89 + + +[ 2023-09-21 08:38:15 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 89 from training step 0 + +[ 2023-09-21 08:38:18 ] Completed train batch 0 to device 2,942.213 ms, 2.94 s total +[ 2023-09-21 08:38:18 ] Completed train batch 0 generator forward 31.825 ms, 2.97 s total +[ 2023-09-21 08:38:18 ] Completed train batch 0 generator loss: 1.289 65.331 ms, 3.04 s total +[ 2023-09-21 08:38:19 ] Completed train batch 0 generator backward 603.243 ms, 3.64 s total +[ 2023-09-21 08:38:19 ] Completed train batch 0 discriminator loss 0.073 64.733 ms, 3.71 s total +[ 2023-09-21 08:38:19 ] Completed train batch 0 discriminator backward 95.130 ms, 3.80 s total +Epoch [89] Step [0/2], gen_loss: 0.419, disc_loss: 0.024 +[ 2023-09-21 08:38:19 ] Completed train batch 0 metrics update 16.866 ms, 3.82 s total +Saving checkpoint at epoch 89 train batch 0 +[ 2023-09-21 08:38:19 ] Completed saving temp checkpoint 430.833 ms, 4.25 s total +[ 2023-09-21 08:38:20 ] Completed replacing temp checkpoint with checkpoint 31.681 ms, 4.28 s total +[ 2023-09-21 08:38:21 ] Completed train batch 1 to device 1,592.870 ms, 5.87 s total +[ 2023-09-21 08:38:21 ] Completed train batch 1 generator forward 26.398 ms, 5.90 s total +[ 2023-09-21 08:38:21 ] Completed train batch 1 generator loss: 1.280 62.429 ms, 5.96 s total +[ 2023-09-21 08:38:22 ] Completed train batch 1 generator backward 1,119.422 ms, 7.08 s total +[ 2023-09-21 08:38:22 ] Completed train batch 1 discriminator loss 0.075 55.964 ms, 7.14 s total +[ 2023-09-21 08:38:22 ] Completed train batch 1 discriminator backward 90.185 ms, 7.23 s total +Epoch [89] Step [1/2], gen_loss: 0.430, disc_loss: 0.026 +[ 2023-09-21 08:38:22 ] Completed train batch 1 metrics update 5.049 ms, 7.23 s total +Saving checkpoint at epoch 89 train batch 1 +[ 2023-09-21 08:38:23 ] Completed saving temp checkpoint 450.621 ms, 7.68 s total +[ 2023-09-21 08:38:23 ] Completed replacing temp checkpoint with checkpoint 20.642 ms, 7.71 s total +Epoch [89] :: gen_loss: 0.424, disc_loss: 0.025 +[ 2023-09-21 08:38:23 ] Completed training generator for epoch 89 70.188 ms, 7.78 s total +[ 2023-09-21 08:38:23 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 89 from eval step 0 + +[ 2023-09-21 08:38:24 ] Completed eval batch 0 to device 1,093.326 ms, 1.09 s total +[ 2023-09-21 08:38:24 ] Completed eval batch 0 forward 30.590 ms, 1.12 s total +[ 2023-09-21 08:38:24 ] Completed eval batch 0 recons_loss 0.324 ms, 1.12 s total +[ 2023-09-21 08:38:24 ] Completed eval batch 0 metrics update 0.580 ms, 1.12 s total +Saving checkpoint at epoch 89 val batch 0 +[ 2023-09-21 08:38:25 ] Completed saving temp checkpoint 390.913 ms, 1.52 s total +[ 2023-09-21 08:38:25 ] Completed replacing temp checkpoint with checkpoint 33.642 ms, 1.55 s total +[ 2023-09-21 08:38:25 ] Completed eval batch 1 to device 420.094 ms, 1.97 s total +[ 2023-09-21 08:38:25 ] Completed eval batch 1 forward 39.962 ms, 2.01 s total +[ 2023-09-21 08:38:25 ] Completed eval batch 1 recons_loss 0.554 ms, 2.01 s total +[ 2023-09-21 08:38:25 ] Completed eval batch 1 metrics update 136.875 ms, 2.15 s total +Saving checkpoint at epoch 89 val batch 1 +Epoch 89 val loss: 0.0649 +[ 2023-09-21 08:38:26 ] Completed saving temp checkpoint 345.337 ms, 2.49 s total +[ 2023-09-21 08:38:26 ] Completed replacing temp checkpoint with checkpoint 17.623 ms, 2.51 s total +[ 2023-09-21 08:38:26 ] Completed evaluating generator for epoch 89 59.337 ms, 2.57 s total + + +EPOCH :: 90 + + +[ 2023-09-21 08:38:26 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 90 from training step 0 + +[ 2023-09-21 08:38:29 ] Completed train batch 0 to device 2,966.603 ms, 2.97 s total +[ 2023-09-21 08:38:29 ] Completed train batch 0 generator forward 42.630 ms, 3.01 s total +[ 2023-09-21 08:38:29 ] Completed train batch 0 generator loss: 1.292 65.599 ms, 3.07 s total +[ 2023-09-21 08:38:29 ] Completed train batch 0 generator backward 678.595 ms, 3.75 s total +[ 2023-09-21 08:38:29 ] Completed train batch 0 discriminator loss 0.084 58.003 ms, 3.81 s total +[ 2023-09-21 08:38:29 ] Completed train batch 0 discriminator backward 92.737 ms, 3.90 s total +Epoch [90] Step [0/2], gen_loss: 0.415, disc_loss: 0.028 +[ 2023-09-21 08:38:29 ] Completed train batch 0 metrics update 6.498 ms, 3.91 s total +Saving checkpoint at epoch 90 train batch 0 +[ 2023-09-21 08:38:30 ] Completed saving temp checkpoint 377.957 ms, 4.29 s total +[ 2023-09-21 08:38:30 ] Completed replacing temp checkpoint with checkpoint 23.757 ms, 4.31 s total +[ 2023-09-21 08:38:31 ] Completed train batch 1 to device 1,278.076 ms, 5.59 s total +[ 2023-09-21 08:38:31 ] Completed train batch 1 generator forward 38.504 ms, 5.63 s total +[ 2023-09-21 08:38:31 ] Completed train batch 1 generator loss: 1.277 61.971 ms, 5.69 s total +[ 2023-09-21 08:38:33 ] Completed train batch 1 generator backward 1,517.513 ms, 7.21 s total +[ 2023-09-21 08:38:33 ] Completed train batch 1 discriminator loss 0.076 56.034 ms, 7.26 s total +[ 2023-09-21 08:38:33 ] Completed train batch 1 discriminator backward 72.188 ms, 7.34 s total +Epoch [90] Step [1/2], gen_loss: 0.427, disc_loss: 0.026 +[ 2023-09-21 08:38:33 ] Completed train batch 1 metrics update 1.332 ms, 7.34 s total +Saving checkpoint at epoch 90 train batch 1 +[ 2023-09-21 08:38:33 ] Completed saving temp checkpoint 387.382 ms, 7.73 s total +[ 2023-09-21 08:38:33 ] Completed replacing temp checkpoint with checkpoint 21.270 ms, 7.75 s total +Epoch [90] :: gen_loss: 0.421, disc_loss: 0.027 +[ 2023-09-21 08:38:33 ] Completed training generator for epoch 90 76.188 ms, 7.82 s total +[ 2023-09-21 08:38:33 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 90 from eval step 0 + +[ 2023-09-21 08:38:35 ] Completed eval batch 0 to device 1,121.755 ms, 1.12 s total +[ 2023-09-21 08:38:35 ] Completed eval batch 0 forward 40.190 ms, 1.16 s total +[ 2023-09-21 08:38:35 ] Completed eval batch 0 recons_loss 0.310 ms, 1.16 s total +[ 2023-09-21 08:38:35 ] Completed eval batch 0 metrics update 0.602 ms, 1.16 s total +Saving checkpoint at epoch 90 val batch 0 +[ 2023-09-21 08:38:35 ] Completed saving temp checkpoint 564.244 ms, 1.73 s total +[ 2023-09-21 08:38:35 ] Completed replacing temp checkpoint with checkpoint 28.342 ms, 1.76 s total +[ 2023-09-21 08:38:36 ] Completed eval batch 1 to device 382.637 ms, 2.14 s total +[ 2023-09-21 08:38:36 ] Completed eval batch 1 forward 38.893 ms, 2.18 s total +[ 2023-09-21 08:38:36 ] Completed eval batch 1 recons_loss 0.342 ms, 2.18 s total +[ 2023-09-21 08:38:36 ] Completed eval batch 1 metrics update 0.315 ms, 2.18 s total +Saving checkpoint at epoch 90 val batch 1 +Epoch 90 val loss: 0.0634 +[ 2023-09-21 08:38:36 ] Completed saving temp checkpoint 420.783 ms, 2.60 s total +[ 2023-09-21 08:38:36 ] Completed replacing temp checkpoint with checkpoint 30.398 ms, 2.63 s total +[ 2023-09-21 08:38:36 ] Completed evaluating generator for epoch 90 78.627 ms, 2.71 s total + + +EPOCH :: 91 + + +[ 2023-09-21 08:38:36 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 91 from training step 0 + +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 08:49:25 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 08:49:25 ] Completed importing Timer 0.023 ms, 0.00 s total +[ 2023-09-21 08:49:30 ] Completed importing everything else 4,769.650 ms, 4.77 s total +| distributed init (rank 2): env:// +| distributed init (rank 1): env:// +| distributed init (rank 3): env:// +| distributed init (rank 4): env:// +| distributed init (rank 5): env:// +| distributed init (rank 0): env:// +[ 2023-09-21 08:49:38 ] Completed preliminaries 7,886.886 ms, 12.66 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 08:49:38 ] Completed build datasets 11.201 ms, 12.67 s total +[ 2023-09-21 08:49:38 ] Completed build samplers 0.047 ms, 12.67 s total +[ 2023-09-21 08:49:38 ] Completed build dataloaders 0.135 ms, 12.67 s total +[ 2023-09-21 08:49:38 ] Completed generator to device 78.187 ms, 12.75 s total +[ 2023-09-21 08:49:38 ] Completed discriminator to device 7.165 ms, 12.75 s total +[ 2023-09-21 08:49:38 ] Completed loss functions 408.468 ms, 13.16 s total +[ 2023-09-21 08:49:38 ] Completed models prepped for distribution 79.521 ms, 13.24 s total +[ 2023-09-21 08:49:38 ] Completed optimizers 0.678 ms, 13.24 s total +[ 2023-09-21 08:49:38 ] Completed grad scalers 0.021 ms, 13.24 s total +[ 2023-09-21 08:49:39 ] Completed checkpoint retrieval 479.484 ms, 13.72 s total + + +EPOCH :: 90 + + +[ 2023-09-21 08:49:39 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 90 from training step 2 + +Epoch [90] :: gen_loss: 0.421, disc_loss: 0.027 +[ 2023-09-21 08:49:39 ] Completed training generator for epoch 90 142.817 ms, 0.14 s total +[ 2023-09-21 08:49:39 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 90 from eval step 2 + +[ 2023-09-21 08:49:39 ] Completed evaluating generator for epoch 90 114.368 ms, 0.11 s total + + +EPOCH :: 91 + + +[ 2023-09-21 08:49:39 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 91 from training step 0 + +[ 2023-09-21 08:49:43 ] Completed train batch 0 to device 4,072.147 ms, 4.07 s total +[ 2023-09-21 08:49:45 ] Completed train batch 0 generator forward 1,452.056 ms, 5.52 s total +[ 2023-09-21 08:49:45 ] Completed train batch 0 generator loss: 1.250 89.324 ms, 5.61 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 08:49:45 ] Completed train batch 0 generator backward 669.703 ms, 6.28 s total +[ 2023-09-21 08:49:46 ] Completed train batch 0 discriminator loss 0.071 57.166 ms, 6.34 s total +[ 2023-09-21 08:49:46 ] Completed train batch 0 discriminator backward 90.714 ms, 6.43 s total +Epoch [91] Step [0/2], gen_loss: 0.421, disc_loss: 0.024 +[ 2023-09-21 08:49:46 ] Completed train batch 0 metrics update 3.618 ms, 6.43 s total +Saving checkpoint at epoch 91 train batch 0 +[ 2023-09-21 08:49:46 ] Completed saving temp checkpoint 419.153 ms, 6.85 s total +[ 2023-09-21 08:49:46 ] Completed replacing temp checkpoint with checkpoint 132.930 ms, 6.99 s total +[ 2023-09-21 08:49:47 ] Completed train batch 1 to device 403.669 ms, 7.39 s total +[ 2023-09-21 08:49:47 ] Completed train batch 1 generator forward 22.212 ms, 7.41 s total +[ 2023-09-21 08:49:47 ] Completed train batch 1 generator loss: 1.298 61.855 ms, 7.47 s total +[ 2023-09-21 08:49:47 ] Completed train batch 1 generator backward 770.879 ms, 8.25 s total +[ 2023-09-21 08:49:47 ] Completed train batch 1 discriminator loss 0.067 56.664 ms, 8.30 s total +[ 2023-09-21 08:49:48 ] Completed train batch 1 discriminator backward 71.514 ms, 8.37 s total +Epoch [91] Step [1/2], gen_loss: 0.425, disc_loss: 0.022 +[ 2023-09-21 08:49:48 ] Completed train batch 1 metrics update 0.824 ms, 8.37 s total +Saving checkpoint at epoch 91 train batch 1 +[ 2023-09-21 08:49:48 ] Completed saving temp checkpoint 502.739 ms, 8.88 s total +[ 2023-09-21 08:49:48 ] Completed replacing temp checkpoint with checkpoint 35.246 ms, 8.91 s total +Epoch [91] :: gen_loss: 0.423, disc_loss: 0.023 +[ 2023-09-21 08:49:48 ] Completed training generator for epoch 91 53.671 ms, 8.97 s total +[ 2023-09-21 08:49:48 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 91 from eval step 0 + +[ 2023-09-21 08:49:49 ] Completed eval batch 0 to device 1,053.857 ms, 1.05 s total +[ 2023-09-21 08:49:49 ] Completed eval batch 0 forward 41.864 ms, 1.10 s total +[ 2023-09-21 08:49:49 ] Completed eval batch 0 recons_loss 0.379 ms, 1.10 s total +[ 2023-09-21 08:49:49 ] Completed eval batch 0 metrics update 0.699 ms, 1.10 s total +Saving checkpoint at epoch 91 val batch 0 +[ 2023-09-21 08:49:50 ] Completed saving temp checkpoint 594.697 ms, 1.69 s total +[ 2023-09-21 08:49:50 ] Completed replacing temp checkpoint with checkpoint 37.013 ms, 1.73 s total +[ 2023-09-21 08:49:50 ] Completed eval batch 1 to device 176.701 ms, 1.91 s total +[ 2023-09-21 08:49:50 ] Completed eval batch 1 forward 39.966 ms, 1.95 s total +[ 2023-09-21 08:49:50 ] Completed eval batch 1 recons_loss 0.571 ms, 1.95 s total +[ 2023-09-21 08:49:50 ] Completed eval batch 1 metrics update 85.173 ms, 2.03 s total +Saving checkpoint at epoch 91 val batch 1 +Epoch 91 val loss: 0.0741 +[ 2023-09-21 08:49:51 ] Completed saving temp checkpoint 621.084 ms, 2.65 s total +[ 2023-09-21 08:49:51 ] Completed replacing temp checkpoint with checkpoint 32.583 ms, 2.68 s total +[ 2023-09-21 08:49:51 ] Completed evaluating generator for epoch 91 53.094 ms, 2.74 s total + + +EPOCH :: 92 + + +[ 2023-09-21 08:49:51 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 92 from training step 0 + +[ 2023-09-21 08:49:54 ] Completed train batch 0 to device 2,883.975 ms, 2.88 s total +[ 2023-09-21 08:49:54 ] Completed train batch 0 generator forward 30.731 ms, 2.91 s total +[ 2023-09-21 08:49:54 ] Completed train batch 0 generator loss: 1.214 66.723 ms, 2.98 s total +[ 2023-09-21 08:49:54 ] Completed train batch 0 generator backward 500.278 ms, 3.48 s total +[ 2023-09-21 08:49:54 ] Completed train batch 0 discriminator loss 0.065 56.526 ms, 3.54 s total +[ 2023-09-21 08:49:55 ] Completed train batch 0 discriminator backward 104.204 ms, 3.64 s total +Epoch [92] Step [0/2], gen_loss: 0.422, disc_loss: 0.022 +[ 2023-09-21 08:49:55 ] Completed train batch 0 metrics update 5.716 ms, 3.65 s total +Saving checkpoint at epoch 92 train batch 0 +[ 2023-09-21 08:49:55 ] Completed saving temp checkpoint 435.245 ms, 4.08 s total +[ 2023-09-21 08:49:55 ] Completed replacing temp checkpoint with checkpoint 25.642 ms, 4.11 s total +[ 2023-09-21 08:49:57 ] Completed train batch 1 to device 1,529.921 ms, 5.64 s total +[ 2023-09-21 08:49:57 ] Completed train batch 1 generator forward 39.712 ms, 5.68 s total +[ 2023-09-21 08:49:57 ] Completed train batch 1 generator loss: 1.289 63.459 ms, 5.74 s total +[ 2023-09-21 08:49:58 ] Completed train batch 1 generator backward 1,129.666 ms, 6.87 s total +[ 2023-09-21 08:49:58 ] Completed train batch 1 discriminator loss 0.064 55.793 ms, 6.93 s total +[ 2023-09-21 08:49:58 ] Completed train batch 1 discriminator backward 74.619 ms, 7.00 s total +Epoch [92] Step [1/2], gen_loss: 0.426, disc_loss: 0.022 +[ 2023-09-21 08:49:58 ] Completed train batch 1 metrics update 1.875 ms, 7.00 s total +Saving checkpoint at epoch 92 train batch 1 +[ 2023-09-21 08:49:58 ] Completed saving temp checkpoint 483.508 ms, 7.49 s total +[ 2023-09-21 08:49:58 ] Completed replacing temp checkpoint with checkpoint 22.755 ms, 7.51 s total +Epoch [92] :: gen_loss: 0.424, disc_loss: 0.022 +[ 2023-09-21 08:49:58 ] Completed training generator for epoch 92 55.788 ms, 7.57 s total +[ 2023-09-21 08:49:58 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 92 from eval step 0 + +[ 2023-09-21 08:50:00 ] Completed eval batch 0 to device 1,067.960 ms, 1.07 s total +[ 2023-09-21 08:50:00 ] Completed eval batch 0 forward 25.414 ms, 1.09 s total +[ 2023-09-21 08:50:00 ] Completed eval batch 0 recons_loss 0.304 ms, 1.09 s total +[ 2023-09-21 08:50:00 ] Completed eval batch 0 metrics update 0.586 ms, 1.09 s total +Saving checkpoint at epoch 92 val batch 0 +[ 2023-09-21 08:50:00 ] Completed saving temp checkpoint 400.325 ms, 1.49 s total +[ 2023-09-21 08:50:00 ] Completed replacing temp checkpoint with checkpoint 26.536 ms, 1.52 s total +[ 2023-09-21 08:50:00 ] Completed eval batch 1 to device 396.832 ms, 1.92 s total +[ 2023-09-21 08:50:00 ] Completed eval batch 1 forward 21.551 ms, 1.94 s total +[ 2023-09-21 08:50:00 ] Completed eval batch 1 recons_loss 0.340 ms, 1.94 s total +[ 2023-09-21 08:50:01 ] Completed eval batch 1 metrics update 285.019 ms, 2.22 s total +Saving checkpoint at epoch 92 val batch 1 +Epoch 92 val loss: 0.0771 +[ 2023-09-21 08:50:01 ] Completed saving temp checkpoint 418.460 ms, 2.64 s total +[ 2023-09-21 08:50:01 ] Completed replacing temp checkpoint with checkpoint 22.137 ms, 2.67 s total +[ 2023-09-21 08:50:01 ] Completed evaluating generator for epoch 92 50.349 ms, 2.72 s total + + +EPOCH :: 93 + + +[ 2023-09-21 08:50:01 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 93 from training step 0 + +[ 2023-09-21 08:50:04 ] Completed train batch 0 to device 2,965.656 ms, 2.97 s total +[ 2023-09-21 08:50:04 ] Completed train batch 0 generator forward 33.314 ms, 3.00 s total +[ 2023-09-21 08:50:04 ] Completed train batch 0 generator loss: 1.285 66.512 ms, 3.07 s total +[ 2023-09-21 08:50:05 ] Completed train batch 0 generator backward 681.252 ms, 3.75 s total +[ 2023-09-21 08:50:05 ] Completed train batch 0 discriminator loss 0.064 56.422 ms, 3.80 s total +[ 2023-09-21 08:50:05 ] Completed train batch 0 discriminator backward 105.179 ms, 3.91 s total +Epoch [93] Step [0/2], gen_loss: 0.422, disc_loss: 0.021 +[ 2023-09-21 08:50:05 ] Completed train batch 0 metrics update 15.974 ms, 3.92 s total +Saving checkpoint at epoch 93 train batch 0 +[ 2023-09-21 08:50:06 ] Completed saving temp checkpoint 422.942 ms, 4.35 s total +[ 2023-09-21 08:50:06 ] Completed replacing temp checkpoint with checkpoint 32.483 ms, 4.38 s total +[ 2023-09-21 08:50:07 ] Completed train batch 1 to device 1,308.132 ms, 5.69 s total +[ 2023-09-21 08:50:07 ] Completed train batch 1 generator forward 23.112 ms, 5.71 s total +[ 2023-09-21 08:50:07 ] Completed train batch 1 generator loss: 1.289 64.847 ms, 5.78 s total +[ 2023-09-21 08:50:08 ] Completed train batch 1 generator backward 1,460.318 ms, 7.24 s total +[ 2023-09-21 08:50:08 ] Completed train batch 1 discriminator loss 0.061 56.436 ms, 7.29 s total +[ 2023-09-21 08:50:09 ] Completed train batch 1 discriminator backward 86.575 ms, 7.38 s total +Epoch [93] Step [1/2], gen_loss: 0.426, disc_loss: 0.020 +[ 2023-09-21 08:50:09 ] Completed train batch 1 metrics update 5.310 ms, 7.38 s total +Saving checkpoint at epoch 93 train batch 1 +[ 2023-09-21 08:50:09 ] Completed saving temp checkpoint 423.224 ms, 7.81 s total +[ 2023-09-21 08:50:09 ] Completed replacing temp checkpoint with checkpoint 21.512 ms, 7.83 s total +Epoch [93] :: gen_loss: 0.424, disc_loss: 0.021 +[ 2023-09-21 08:50:09 ] Completed training generator for epoch 93 51.936 ms, 7.88 s total +[ 2023-09-21 08:50:09 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 93 from eval step 0 + +[ 2023-09-21 08:50:10 ] Completed eval batch 0 to device 1,125.783 ms, 1.13 s total +[ 2023-09-21 08:50:10 ] Completed eval batch 0 forward 31.697 ms, 1.16 s total +[ 2023-09-21 08:50:10 ] Completed eval batch 0 recons_loss 0.321 ms, 1.16 s total +[ 2023-09-21 08:50:10 ] Completed eval batch 0 metrics update 0.579 ms, 1.16 s total +Saving checkpoint at epoch 93 val batch 0 +[ 2023-09-21 08:50:11 ] Completed saving temp checkpoint 392.276 ms, 1.55 s total +[ 2023-09-21 08:50:11 ] Completed replacing temp checkpoint with checkpoint 22.299 ms, 1.57 s total +[ 2023-09-21 08:50:11 ] Completed eval batch 1 to device 457.164 ms, 2.03 s total +[ 2023-09-21 08:50:11 ] Completed eval batch 1 forward 21.511 ms, 2.05 s total +[ 2023-09-21 08:50:11 ] Completed eval batch 1 recons_loss 0.344 ms, 2.05 s total +[ 2023-09-21 08:50:11 ] Completed eval batch 1 metrics update 146.902 ms, 2.20 s total +Saving checkpoint at epoch 93 val batch 1 +Epoch 93 val loss: 0.0753 +[ 2023-09-21 08:50:12 ] Completed saving temp checkpoint 452.113 ms, 2.65 s total +[ 2023-09-21 08:50:12 ] Completed replacing temp checkpoint with checkpoint 22.818 ms, 2.67 s total +[ 2023-09-21 08:50:12 ] Completed evaluating generator for epoch 93 51.088 ms, 2.72 s total + + +EPOCH :: 94 + + +[ 2023-09-21 08:50:12 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 94 from training step 0 + +[ 2023-09-21 08:50:15 ] Completed train batch 0 to device 3,063.700 ms, 3.06 s total +[ 2023-09-21 08:50:15 ] Completed train batch 0 generator forward 28.348 ms, 3.09 s total +[ 2023-09-21 08:50:15 ] Completed train batch 0 generator loss: 1.236 66.632 ms, 3.16 s total +[ 2023-09-21 08:50:15 ] Completed train batch 0 generator backward 450.609 ms, 3.61 s total +[ 2023-09-21 08:50:15 ] Completed train batch 0 discriminator loss 0.059 75.608 ms, 3.68 s total +[ 2023-09-21 08:50:16 ] Completed train batch 0 discriminator backward 82.230 ms, 3.77 s total +Epoch [94] Step [0/2], gen_loss: 0.420, disc_loss: 0.020 +[ 2023-09-21 08:50:16 ] Completed train batch 0 metrics update 8.298 ms, 3.78 s total +Saving checkpoint at epoch 94 train batch 0 +[ 2023-09-21 08:50:16 ] Completed saving temp checkpoint 418.287 ms, 4.19 s total +[ 2023-09-21 08:50:16 ] Completed replacing temp checkpoint with checkpoint 25.319 ms, 4.22 s total +[ 2023-09-21 08:50:18 ] Completed train batch 1 to device 1,614.854 ms, 5.83 s total +[ 2023-09-21 08:50:18 ] Completed train batch 1 generator forward 23.832 ms, 5.86 s total +[ 2023-09-21 08:50:18 ] Completed train batch 1 generator loss: 1.274 65.963 ms, 5.92 s total +[ 2023-09-21 08:50:19 ] Completed train batch 1 generator backward 1,104.187 ms, 7.03 s total +[ 2023-09-21 08:50:19 ] Completed train batch 1 discriminator loss 0.059 59.019 ms, 7.09 s total +[ 2023-09-21 08:50:19 ] Completed train batch 1 discriminator backward 58.181 ms, 7.15 s total +Epoch [94] Step [1/2], gen_loss: 0.428, disc_loss: 0.019 +[ 2023-09-21 08:50:19 ] Completed train batch 1 metrics update 5.017 ms, 7.15 s total +Saving checkpoint at epoch 94 train batch 1 +[ 2023-09-21 08:50:19 ] Completed saving temp checkpoint 466.656 ms, 7.62 s total +[ 2023-09-21 08:50:19 ] Completed replacing temp checkpoint with checkpoint 21.747 ms, 7.64 s total +Epoch [94] :: gen_loss: 0.424, disc_loss: 0.019 +[ 2023-09-21 08:50:19 ] Completed training generator for epoch 94 57.259 ms, 7.70 s total +[ 2023-09-21 08:50:19 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 94 from eval step 0 + +[ 2023-09-21 08:50:21 ] Completed eval batch 0 to device 1,128.398 ms, 1.13 s total +[ 2023-09-21 08:50:21 ] Completed eval batch 0 forward 25.459 ms, 1.15 s total +[ 2023-09-21 08:50:21 ] Completed eval batch 0 recons_loss 0.320 ms, 1.15 s total +[ 2023-09-21 08:50:21 ] Completed eval batch 0 metrics update 0.606 ms, 1.15 s total +Saving checkpoint at epoch 94 val batch 0 +[ 2023-09-21 08:50:21 ] Completed saving temp checkpoint 351.894 ms, 1.51 s total +[ 2023-09-21 08:50:21 ] Completed replacing temp checkpoint with checkpoint 30.848 ms, 1.54 s total +[ 2023-09-21 08:50:21 ] Completed eval batch 1 to device 468.216 ms, 2.01 s total +[ 2023-09-21 08:50:21 ] Completed eval batch 1 forward 21.547 ms, 2.03 s total +[ 2023-09-21 08:50:21 ] Completed eval batch 1 recons_loss 0.338 ms, 2.03 s total +[ 2023-09-21 08:50:22 ] Completed eval batch 1 metrics update 91.798 ms, 2.12 s total +Saving checkpoint at epoch 94 val batch 1 +Epoch 94 val loss: 0.0711 +[ 2023-09-21 08:50:22 ] Completed saving temp checkpoint 414.122 ms, 2.53 s total +[ 2023-09-21 08:50:22 ] Completed replacing temp checkpoint with checkpoint 20.486 ms, 2.55 s total +[ 2023-09-21 08:50:22 ] Completed evaluating generator for epoch 94 49.853 ms, 2.60 s total + + +EPOCH :: 95 + + +[ 2023-09-21 08:50:22 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 95 from training step 0 + +[ 2023-09-21 08:50:25 ] Completed train batch 0 to device 3,136.137 ms, 3.14 s total +[ 2023-09-21 08:50:25 ] Completed train batch 0 generator forward 41.057 ms, 3.18 s total +[ 2023-09-21 08:50:25 ] Completed train batch 0 generator loss: 1.203 65.557 ms, 3.24 s total +[ 2023-09-21 08:50:26 ] Completed train batch 0 generator backward 545.893 ms, 3.79 s total +[ 2023-09-21 08:50:26 ] Completed train batch 0 discriminator loss 0.058 56.472 ms, 3.85 s total +[ 2023-09-21 08:50:26 ] Completed train batch 0 discriminator backward 95.229 ms, 3.94 s total +Epoch [95] Step [0/2], gen_loss: 0.418, disc_loss: 0.019 +[ 2023-09-21 08:50:26 ] Completed train batch 0 metrics update 5.165 ms, 3.95 s total +Saving checkpoint at epoch 95 train batch 0 +[ 2023-09-21 08:50:26 ] Completed saving temp checkpoint 358.983 ms, 4.30 s total +[ 2023-09-21 08:50:26 ] Completed replacing temp checkpoint with checkpoint 18.578 ms, 4.32 s total +[ 2023-09-21 08:50:28 ] Completed train batch 1 to device 1,568.036 ms, 5.89 s total +[ 2023-09-21 08:50:28 ] Completed train batch 1 generator forward 37.903 ms, 5.93 s total +[ 2023-09-21 08:50:28 ] Completed train batch 1 generator loss: 1.223 63.637 ms, 5.99 s total +[ 2023-09-21 08:50:29 ] Completed train batch 1 generator backward 1,224.784 ms, 7.22 s total +[ 2023-09-21 08:50:29 ] Completed train batch 1 discriminator loss 0.054 55.883 ms, 7.27 s total +[ 2023-09-21 08:50:29 ] Completed train batch 1 discriminator backward 80.620 ms, 7.35 s total +Epoch [95] Step [1/2], gen_loss: 0.421, disc_loss: 0.018 +[ 2023-09-21 08:50:29 ] Completed train batch 1 metrics update 2.873 ms, 7.36 s total +Saving checkpoint at epoch 95 train batch 1 +[ 2023-09-21 08:50:30 ] Completed saving temp checkpoint 470.625 ms, 7.83 s total +[ 2023-09-21 08:50:30 ] Completed replacing temp checkpoint with checkpoint 22.603 ms, 7.85 s total +Epoch [95] :: gen_loss: 0.420, disc_loss: 0.019 +[ 2023-09-21 08:50:30 ] Completed training generator for epoch 95 49.735 ms, 7.90 s total +[ 2023-09-21 08:50:30 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 95 from eval step 0 + +[ 2023-09-21 08:50:31 ] Completed eval batch 0 to device 1,082.916 ms, 1.08 s total +[ 2023-09-21 08:50:31 ] Completed eval batch 0 forward 41.575 ms, 1.12 s total +[ 2023-09-21 08:50:31 ] Completed eval batch 0 recons_loss 0.250 ms, 1.12 s total +[ 2023-09-21 08:50:31 ] Completed eval batch 0 metrics update 0.542 ms, 1.13 s total +Saving checkpoint at epoch 95 val batch 0 +[ 2023-09-21 08:50:31 ] Completed saving temp checkpoint 399.327 ms, 1.52 s total +[ 2023-09-21 08:50:32 ] Completed replacing temp checkpoint with checkpoint 26.569 ms, 1.55 s total +[ 2023-09-21 08:50:32 ] Completed eval batch 1 to device 365.562 ms, 1.92 s total +[ 2023-09-21 08:50:32 ] Completed eval batch 1 forward 40.143 ms, 1.96 s total +[ 2023-09-21 08:50:32 ] Completed eval batch 1 recons_loss 0.574 ms, 1.96 s total +[ 2023-09-21 08:50:32 ] Completed eval batch 1 metrics update 163.984 ms, 2.12 s total +Saving checkpoint at epoch 95 val batch 1 +Epoch 95 val loss: 0.0623 +[ 2023-09-21 08:50:33 ] Completed saving temp checkpoint 451.803 ms, 2.57 s total +[ 2023-09-21 08:50:33 ] Completed replacing temp checkpoint with checkpoint 21.917 ms, 2.60 s total +[ 2023-09-21 08:50:33 ] Completed evaluating generator for epoch 95 53.738 ms, 2.65 s total + + +EPOCH :: 96 + + +[ 2023-09-21 08:50:33 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 96 from training step 0 + +[ 2023-09-21 08:50:36 ] Completed train batch 0 to device 2,992.449 ms, 2.99 s total +[ 2023-09-21 08:50:36 ] Completed train batch 0 generator forward 26.961 ms, 3.02 s total +[ 2023-09-21 08:50:36 ] Completed train batch 0 generator loss: 1.224 66.396 ms, 3.09 s total +[ 2023-09-21 08:50:36 ] Completed train batch 0 generator backward 503.869 ms, 3.59 s total +[ 2023-09-21 08:50:36 ] Completed train batch 0 discriminator loss 0.051 56.577 ms, 3.65 s total +[ 2023-09-21 08:50:36 ] Completed train batch 0 discriminator backward 87.840 ms, 3.73 s total +Epoch [96] Step [0/2], gen_loss: 0.421, disc_loss: 0.017 +[ 2023-09-21 08:50:36 ] Completed train batch 0 metrics update 12.389 ms, 3.75 s total +Saving checkpoint at epoch 96 train batch 0 +[ 2023-09-21 08:50:37 ] Completed saving temp checkpoint 367.978 ms, 4.11 s total +[ 2023-09-21 08:50:37 ] Completed replacing temp checkpoint with checkpoint 21.711 ms, 4.14 s total +[ 2023-09-21 08:50:38 ] Completed train batch 1 to device 1,629.278 ms, 5.77 s total +[ 2023-09-21 08:50:38 ] Completed train batch 1 generator forward 22.273 ms, 5.79 s total +[ 2023-09-21 08:50:38 ] Completed train batch 1 generator loss: 1.276 63.434 ms, 5.85 s total +[ 2023-09-21 08:50:40 ] Completed train batch 1 generator backward 1,133.581 ms, 6.98 s total +[ 2023-09-21 08:50:40 ] Completed train batch 1 discriminator loss 0.053 55.728 ms, 7.04 s total +[ 2023-09-21 08:50:40 ] Completed train batch 1 discriminator backward 69.563 ms, 7.11 s total +Epoch [96] Step [1/2], gen_loss: 0.423, disc_loss: 0.018 +[ 2023-09-21 08:50:40 ] Completed train batch 1 metrics update 1.950 ms, 7.11 s total +Saving checkpoint at epoch 96 train batch 1 +[ 2023-09-21 08:50:40 ] Completed saving temp checkpoint 464.833 ms, 7.58 s total +[ 2023-09-21 08:50:40 ] Completed replacing temp checkpoint with checkpoint 26.742 ms, 7.60 s total +Epoch [96] :: gen_loss: 0.422, disc_loss: 0.018 +[ 2023-09-21 08:50:40 ] Completed training generator for epoch 96 63.028 ms, 7.67 s total +[ 2023-09-21 08:50:40 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 96 from eval step 0 + +[ 2023-09-21 08:50:41 ] Completed eval batch 0 to device 1,077.419 ms, 1.08 s total +[ 2023-09-21 08:50:41 ] Completed eval batch 0 forward 43.355 ms, 1.12 s total +[ 2023-09-21 08:50:41 ] Completed eval batch 0 recons_loss 0.377 ms, 1.12 s total +[ 2023-09-21 08:50:41 ] Completed eval batch 0 metrics update 0.578 ms, 1.12 s total +Saving checkpoint at epoch 96 val batch 0 +[ 2023-09-21 08:50:42 ] Completed saving temp checkpoint 383.158 ms, 1.50 s total +[ 2023-09-21 08:50:42 ] Completed replacing temp checkpoint with checkpoint 22.101 ms, 1.53 s total +[ 2023-09-21 08:50:42 ] Completed eval batch 1 to device 400.853 ms, 1.93 s total +[ 2023-09-21 08:50:42 ] Completed eval batch 1 forward 33.008 ms, 1.96 s total +[ 2023-09-21 08:50:42 ] Completed eval batch 1 recons_loss 0.390 ms, 1.96 s total +[ 2023-09-21 08:50:42 ] Completed eval batch 1 metrics update 165.486 ms, 2.13 s total +Saving checkpoint at epoch 96 val batch 1 +Epoch 96 val loss: 0.0653 +[ 2023-09-21 08:50:43 ] Completed saving temp checkpoint 429.637 ms, 2.56 s total +[ 2023-09-21 08:50:43 ] Completed replacing temp checkpoint with checkpoint 30.061 ms, 2.59 s total +[ 2023-09-21 08:50:43 ] Completed evaluating generator for epoch 96 53.936 ms, 2.64 s total + + +EPOCH :: 97 + + +[ 2023-09-21 08:50:43 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 97 from training step 0 + +[ 2023-09-21 08:50:46 ] Completed train batch 0 to device 3,024.185 ms, 3.02 s total +[ 2023-09-21 08:50:46 ] Completed train batch 0 generator forward 44.007 ms, 3.07 s total +[ 2023-09-21 08:50:46 ] Completed train batch 0 generator loss: 1.244 66.023 ms, 3.13 s total +[ 2023-09-21 08:50:47 ] Completed train batch 0 generator backward 599.942 ms, 3.73 s total +[ 2023-09-21 08:50:47 ] Completed train batch 0 discriminator loss 0.054 75.593 ms, 3.81 s total +[ 2023-09-21 08:50:47 ] Completed train batch 0 discriminator backward 71.806 ms, 3.88 s total +Epoch [97] Step [0/2], gen_loss: 0.420, disc_loss: 0.018 +[ 2023-09-21 08:50:47 ] Completed train batch 0 metrics update 0.684 ms, 3.88 s total +Saving checkpoint at epoch 97 train batch 0 +[ 2023-09-21 08:50:47 ] Completed saving temp checkpoint 396.388 ms, 4.28 s total +[ 2023-09-21 08:50:47 ] Completed replacing temp checkpoint with checkpoint 30.673 ms, 4.31 s total +[ 2023-09-21 08:50:49 ] Completed train batch 1 to device 1,530.837 ms, 5.84 s total +[ 2023-09-21 08:50:49 ] Completed train batch 1 generator forward 41.378 ms, 5.88 s total +[ 2023-09-21 08:50:49 ] Completed train batch 1 generator loss: 1.248 63.925 ms, 5.95 s total +[ 2023-09-21 08:50:50 ] Completed train batch 1 generator backward 1,215.917 ms, 7.16 s total +[ 2023-09-21 08:50:50 ] Completed train batch 1 discriminator loss 0.052 55.929 ms, 7.22 s total +[ 2023-09-21 08:50:50 ] Completed train batch 1 discriminator backward 73.024 ms, 7.29 s total +Epoch [97] Step [1/2], gen_loss: 0.427, disc_loss: 0.018 +[ 2023-09-21 08:50:50 ] Completed train batch 1 metrics update 0.400 ms, 7.29 s total +Saving checkpoint at epoch 97 train batch 1 +[ 2023-09-21 08:50:51 ] Completed saving temp checkpoint 425.599 ms, 7.72 s total +[ 2023-09-21 08:50:51 ] Completed replacing temp checkpoint with checkpoint 25.642 ms, 7.74 s total +Epoch [97] :: gen_loss: 0.423, disc_loss: 0.018 +[ 2023-09-21 08:50:51 ] Completed training generator for epoch 97 67.135 ms, 7.81 s total +[ 2023-09-21 08:50:51 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 97 from eval step 0 + +[ 2023-09-21 08:50:52 ] Completed eval batch 0 to device 1,081.840 ms, 1.08 s total +[ 2023-09-21 08:50:52 ] Completed eval batch 0 forward 44.007 ms, 1.13 s total +[ 2023-09-21 08:50:52 ] Completed eval batch 0 recons_loss 0.313 ms, 1.13 s total +[ 2023-09-21 08:50:52 ] Completed eval batch 0 metrics update 0.573 ms, 1.13 s total +Saving checkpoint at epoch 97 val batch 0 +[ 2023-09-21 08:50:52 ] Completed saving temp checkpoint 416.440 ms, 1.54 s total +[ 2023-09-21 08:50:52 ] Completed replacing temp checkpoint with checkpoint 25.529 ms, 1.57 s total +[ 2023-09-21 08:50:53 ] Completed eval batch 1 to device 355.590 ms, 1.92 s total +[ 2023-09-21 08:50:53 ] Completed eval batch 1 forward 39.866 ms, 1.96 s total +[ 2023-09-21 08:50:53 ] Completed eval batch 1 recons_loss 0.341 ms, 1.96 s total +[ 2023-09-21 08:50:53 ] Completed eval batch 1 metrics update 111.569 ms, 2.08 s total +Saving checkpoint at epoch 97 val batch 1 +Epoch 97 val loss: 0.0613 +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 09:04:39 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 09:04:39 ] Completed importing Timer 0.020 ms, 0.00 s total +[ 2023-09-21 09:04:44 ] Completed importing everything else 4,945.590 ms, 4.95 s total +| distributed init (rank 2): env:// +| distributed init (rank 1): env:// +| distributed init (rank 5): env:// +| distributed init (rank 4): env:// +| distributed init (rank 0): env:// +| distributed init (rank 3): env:// +[ 2023-09-21 09:04:52 ] Completed preliminaries 7,978.812 ms, 12.92 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 09:04:52 ] Completed build datasets 11.549 ms, 12.94 s total +[ 2023-09-21 09:04:52 ] Completed build samplers 0.049 ms, 12.94 s total +[ 2023-09-21 09:04:52 ] Completed build dataloaders 0.136 ms, 12.94 s total +[ 2023-09-21 09:04:52 ] Completed generator to device 80.354 ms, 13.02 s total +[ 2023-09-21 09:04:52 ] Completed discriminator to device 7.366 ms, 13.02 s total +[ 2023-09-21 09:04:52 ] Completed loss functions 383.310 ms, 13.41 s total +[ 2023-09-21 09:04:52 ] Completed models prepped for distribution 99.896 ms, 13.51 s total +[ 2023-09-21 09:04:52 ] Completed optimizers 0.655 ms, 13.51 s total +[ 2023-09-21 09:04:52 ] Completed grad scalers 0.029 ms, 13.51 s total +[ 2023-09-21 09:04:53 ] Completed checkpoint retrieval 474.925 ms, 13.98 s total + + +EPOCH :: 97 + + +[ 2023-09-21 09:04:53 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 97 from training step 2 + +Epoch [97] :: gen_loss: 0.423, disc_loss: 0.018 +[ 2023-09-21 09:04:53 ] Completed training generator for epoch 97 128.915 ms, 0.13 s total +[ 2023-09-21 09:04:53 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 97 from eval step 1 + +[ 2023-09-21 09:04:54 ] Completed eval batch 1 to device 1,547.702 ms, 1.55 s total +[ 2023-09-21 09:04:56 ] Completed eval batch 1 forward 1,111.353 ms, 2.66 s total +[ 2023-09-21 09:04:56 ] Completed eval batch 1 recons_loss 0.262 ms, 2.66 s total +[ 2023-09-21 09:04:56 ] Completed eval batch 1 metrics update 453.418 ms, 3.11 s total +Saving checkpoint at epoch 97 val batch 1 +Epoch 97 val loss: 0.0597 +[ 2023-09-21 09:04:56 ] Completed saving temp checkpoint 350.483 ms, 3.46 s total +[ 2023-09-21 09:04:56 ] Completed replacing temp checkpoint with checkpoint 121.903 ms, 3.59 s total +[ 2023-09-21 09:04:56 ] Completed evaluating generator for epoch 97 44.084 ms, 3.63 s total + + +EPOCH :: 98 + + +[ 2023-09-21 09:04:56 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 98 from training step 0 + +[ 2023-09-21 09:05:00 ] Completed train batch 0 to device 3,103.570 ms, 3.10 s total +[ 2023-09-21 09:05:00 ] Completed train batch 0 generator forward 33.859 ms, 3.14 s total +[ 2023-09-21 09:05:00 ] Completed train batch 0 generator loss: 1.289 70.038 ms, 3.21 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 09:05:00 ] Completed train batch 0 generator backward 686.476 ms, 3.89 s total +[ 2023-09-21 09:05:00 ] Completed train batch 0 discriminator loss 0.054 55.546 ms, 3.95 s total +[ 2023-09-21 09:05:01 ] Completed train batch 0 discriminator backward 102.629 ms, 4.05 s total +Epoch [98] Step [0/2], gen_loss: 0.431, disc_loss: 0.018 +[ 2023-09-21 09:05:01 ] Completed train batch 0 metrics update 6.263 ms, 4.06 s total +Saving checkpoint at epoch 98 train batch 0 +[ 2023-09-21 09:05:01 ] Completed saving temp checkpoint 443.161 ms, 4.50 s total +[ 2023-09-21 09:05:01 ] Completed replacing temp checkpoint with checkpoint 21.252 ms, 4.52 s total +[ 2023-09-21 09:05:03 ] Completed train batch 1 to device 1,627.861 ms, 6.15 s total +[ 2023-09-21 09:05:03 ] Completed train batch 1 generator forward 25.533 ms, 6.18 s total +[ 2023-09-21 09:05:03 ] Completed train batch 1 generator loss: 1.283 62.189 ms, 6.24 s total +[ 2023-09-21 09:05:04 ] Completed train batch 1 generator backward 1,041.873 ms, 7.28 s total +[ 2023-09-21 09:05:04 ] Completed train batch 1 discriminator loss 0.056 55.876 ms, 7.34 s total +[ 2023-09-21 09:05:04 ] Completed train batch 1 discriminator backward 66.381 ms, 7.40 s total +Epoch [98] Step [1/2], gen_loss: 0.420, disc_loss: 0.019 +[ 2023-09-21 09:05:04 ] Completed train batch 1 metrics update 2.823 ms, 7.41 s total +Saving checkpoint at epoch 98 train batch 1 +[ 2023-09-21 09:05:04 ] Completed saving temp checkpoint 438.400 ms, 7.84 s total +[ 2023-09-21 09:05:04 ] Completed replacing temp checkpoint with checkpoint 23.734 ms, 7.87 s total +Epoch [98] :: gen_loss: 0.426, disc_loss: 0.018 +[ 2023-09-21 09:05:04 ] Completed training generator for epoch 98 55.206 ms, 7.92 s total +[ 2023-09-21 09:05:04 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 98 from eval step 0 + +[ 2023-09-21 09:05:06 ] Completed eval batch 0 to device 1,096.301 ms, 1.10 s total +[ 2023-09-21 09:05:06 ] Completed eval batch 0 forward 24.919 ms, 1.12 s total +[ 2023-09-21 09:05:06 ] Completed eval batch 0 recons_loss 0.295 ms, 1.12 s total +[ 2023-09-21 09:05:06 ] Completed eval batch 0 metrics update 0.593 ms, 1.12 s total +Saving checkpoint at epoch 98 val batch 0 +[ 2023-09-21 09:05:06 ] Completed saving temp checkpoint 381.706 ms, 1.50 s total +[ 2023-09-21 09:05:06 ] Completed replacing temp checkpoint with checkpoint 22.987 ms, 1.53 s total +[ 2023-09-21 09:05:06 ] Completed eval batch 1 to device 452.805 ms, 1.98 s total +[ 2023-09-21 09:05:06 ] Completed eval batch 1 forward 39.107 ms, 2.02 s total +[ 2023-09-21 09:05:06 ] Completed eval batch 1 recons_loss 0.553 ms, 2.02 s total +[ 2023-09-21 09:05:07 ] Completed eval batch 1 metrics update 253.018 ms, 2.27 s total +Saving checkpoint at epoch 98 val batch 1 +Epoch 98 val loss: 0.0687 +[ 2023-09-21 09:05:07 ] Completed saving temp checkpoint 411.712 ms, 2.68 s total +[ 2023-09-21 09:05:07 ] Completed replacing temp checkpoint with checkpoint 27.734 ms, 2.71 s total +[ 2023-09-21 09:05:07 ] Completed evaluating generator for epoch 98 51.729 ms, 2.76 s total + + +EPOCH :: 99 + + +[ 2023-09-21 09:05:07 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 99 from training step 0 + +[ 2023-09-21 09:05:10 ] Completed train batch 0 to device 2,980.073 ms, 2.98 s total +[ 2023-09-21 09:05:10 ] Completed train batch 0 generator forward 24.550 ms, 3.00 s total +[ 2023-09-21 09:05:10 ] Completed train batch 0 generator loss: 1.302 63.461 ms, 3.07 s total +[ 2023-09-21 09:05:11 ] Completed train batch 0 generator backward 741.768 ms, 3.81 s total +[ 2023-09-21 09:05:11 ] Completed train batch 0 discriminator loss 0.058 56.786 ms, 3.87 s total +[ 2023-09-21 09:05:11 ] Completed train batch 0 discriminator backward 96.073 ms, 3.96 s total +Epoch [99] Step [0/2], gen_loss: 0.426, disc_loss: 0.019 +[ 2023-09-21 09:05:11 ] Completed train batch 0 metrics update 5.610 ms, 3.97 s total +Saving checkpoint at epoch 99 train batch 0 +[ 2023-09-21 09:05:12 ] Completed saving temp checkpoint 384.608 ms, 4.35 s total +[ 2023-09-21 09:05:12 ] Completed replacing temp checkpoint with checkpoint 18.394 ms, 4.37 s total +[ 2023-09-21 09:05:13 ] Completed train batch 1 to device 1,181.832 ms, 5.55 s total +[ 2023-09-21 09:05:13 ] Completed train batch 1 generator forward 41.383 ms, 5.59 s total +[ 2023-09-21 09:05:13 ] Completed train batch 1 generator loss: 1.314 61.957 ms, 5.66 s total +[ 2023-09-21 09:05:14 ] Completed train batch 1 generator backward 1,448.958 ms, 7.11 s total +[ 2023-09-21 09:05:14 ] Completed train batch 1 discriminator loss 0.069 56.044 ms, 7.16 s total +[ 2023-09-21 09:05:14 ] Completed train batch 1 discriminator backward 69.443 ms, 7.23 s total +Epoch [99] Step [1/2], gen_loss: 0.430, disc_loss: 0.019 +[ 2023-09-21 09:05:14 ] Completed train batch 1 metrics update 2.796 ms, 7.23 s total +Saving checkpoint at epoch 99 train batch 1 +[ 2023-09-21 09:05:15 ] Completed saving temp checkpoint 426.307 ms, 7.66 s total +[ 2023-09-21 09:05:15 ] Completed replacing temp checkpoint with checkpoint 21.965 ms, 7.68 s total +Epoch [99] :: gen_loss: 0.428, disc_loss: 0.019 +[ 2023-09-21 09:05:15 ] Completed training generator for epoch 99 55.724 ms, 7.74 s total +[ 2023-09-21 09:05:15 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 99 from eval step 0 + +[ 2023-09-21 09:05:16 ] Completed eval batch 0 to device 1,094.851 ms, 1.09 s total +[ 2023-09-21 09:05:16 ] Completed eval batch 0 forward 31.004 ms, 1.13 s total +[ 2023-09-21 09:05:16 ] Completed eval batch 0 recons_loss 0.345 ms, 1.13 s total +[ 2023-09-21 09:05:16 ] Completed eval batch 0 metrics update 0.640 ms, 1.13 s total +Saving checkpoint at epoch 99 val batch 0 +[ 2023-09-21 09:05:16 ] Completed saving temp checkpoint 447.265 ms, 1.57 s total +[ 2023-09-21 09:05:17 ] Completed replacing temp checkpoint with checkpoint 22.580 ms, 1.60 s total +[ 2023-09-21 09:05:17 ] Completed eval batch 1 to device 397.169 ms, 1.99 s total +[ 2023-09-21 09:05:17 ] Completed eval batch 1 forward 27.069 ms, 2.02 s total +[ 2023-09-21 09:05:17 ] Completed eval batch 1 recons_loss 0.395 ms, 2.02 s total +[ 2023-09-21 09:05:17 ] Completed eval batch 1 metrics update 274.357 ms, 2.30 s total +Saving checkpoint at epoch 99 val batch 1 +Epoch 99 val loss: 0.0747 +[ 2023-09-21 09:05:18 ] Completed saving temp checkpoint 492.923 ms, 2.79 s total +[ 2023-09-21 09:05:18 ] Completed replacing temp checkpoint with checkpoint 19.645 ms, 2.81 s total +[ 2023-09-21 09:05:18 ] Completed evaluating generator for epoch 99 52.333 ms, 2.86 s total + + +EPOCH :: 100 + + +[ 2023-09-21 09:05:18 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 100 from training step 0 + +[ 2023-09-21 09:05:21 ] Completed train batch 0 to device 2,891.001 ms, 2.89 s total +[ 2023-09-21 09:05:21 ] Completed train batch 0 generator forward 24.771 ms, 2.92 s total +[ 2023-09-21 09:05:21 ] Completed train batch 0 generator loss: 1.276 64.803 ms, 2.98 s total +[ 2023-09-21 09:05:21 ] Completed train batch 0 generator backward 751.054 ms, 3.73 s total +[ 2023-09-21 09:05:22 ] Completed train batch 0 discriminator loss 0.062 75.276 ms, 3.81 s total +[ 2023-09-21 09:05:22 ] Completed train batch 0 discriminator backward 82.037 ms, 3.89 s total +Epoch [100] Step [0/2], gen_loss: 0.427, disc_loss: 0.021 +[ 2023-09-21 09:05:22 ] Completed train batch 0 metrics update 12.213 ms, 3.90 s total +Saving checkpoint at epoch 100 train batch 0 +[ 2023-09-21 09:05:22 ] Completed saving temp checkpoint 437.107 ms, 4.34 s total +[ 2023-09-21 09:05:22 ] Completed replacing temp checkpoint with checkpoint 22.387 ms, 4.36 s total +[ 2023-09-21 09:05:23 ] Completed train batch 1 to device 1,290.804 ms, 5.65 s total +[ 2023-09-21 09:05:23 ] Completed train batch 1 generator forward 38.132 ms, 5.69 s total +[ 2023-09-21 09:05:24 ] Completed train batch 1 generator loss: 1.293 64.538 ms, 5.75 s total +[ 2023-09-21 09:05:25 ] Completed train batch 1 generator backward 1,542.999 ms, 7.30 s total +[ 2023-09-21 09:05:25 ] Completed train batch 1 discriminator loss 0.067 55.804 ms, 7.35 s total +[ 2023-09-21 09:05:25 ] Completed train batch 1 discriminator backward 71.175 ms, 7.42 s total +Epoch [100] Step [1/2], gen_loss: 0.446, disc_loss: 0.021 +[ 2023-09-21 09:05:25 ] Completed train batch 1 metrics update 2.032 ms, 7.43 s total +Saving checkpoint at epoch 100 train batch 1 +[ 2023-09-21 09:05:26 ] Completed saving temp checkpoint 449.042 ms, 7.88 s total +[ 2023-09-21 09:05:26 ] Completed replacing temp checkpoint with checkpoint 23.054 ms, 7.90 s total +Epoch [100] :: gen_loss: 0.437, disc_loss: 0.021 +[ 2023-09-21 09:05:26 ] Completed training generator for epoch 100 49.971 ms, 7.95 s total +[ 2023-09-21 09:05:26 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 100 from eval step 0 + +[ 2023-09-21 09:05:27 ] Completed eval batch 0 to device 1,097.957 ms, 1.10 s total +[ 2023-09-21 09:05:27 ] Completed eval batch 0 forward 24.411 ms, 1.12 s total +[ 2023-09-21 09:05:27 ] Completed eval batch 0 recons_loss 0.388 ms, 1.12 s total +[ 2023-09-21 09:05:27 ] Completed eval batch 0 metrics update 0.671 ms, 1.12 s total +Saving checkpoint at epoch 100 val batch 0 +[ 2023-09-21 09:05:27 ] Completed saving temp checkpoint 378.759 ms, 1.50 s total +[ 2023-09-21 09:05:27 ] Completed replacing temp checkpoint with checkpoint 21.271 ms, 1.52 s total +[ 2023-09-21 09:05:28 ] Completed eval batch 1 to device 490.156 ms, 2.01 s total +[ 2023-09-21 09:05:28 ] Completed eval batch 1 forward 21.177 ms, 2.03 s total +[ 2023-09-21 09:05:28 ] Completed eval batch 1 recons_loss 0.342 ms, 2.04 s total +[ 2023-09-21 09:05:28 ] Completed eval batch 1 metrics update 216.298 ms, 2.25 s total +Saving checkpoint at epoch 100 val batch 1 +Epoch 100 val loss: 0.0564 +[ 2023-09-21 09:05:28 ] Completed saving temp checkpoint 497.737 ms, 2.75 s total +[ 2023-09-21 09:05:28 ] Completed replacing temp checkpoint with checkpoint 22.628 ms, 2.77 s total +[ 2023-09-21 09:05:29 ] Completed evaluating generator for epoch 100 61.719 ms, 2.83 s total + + +EPOCH :: 101 + + +[ 2023-09-21 09:05:29 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 101 from training step 0 + +[ 2023-09-21 09:05:32 ] Completed train batch 0 to device 3,122.790 ms, 3.12 s total +[ 2023-09-21 09:05:32 ] Completed train batch 0 generator forward 37.788 ms, 3.16 s total +[ 2023-09-21 09:05:32 ] Completed train batch 0 generator loss: 1.324 81.898 ms, 3.24 s total +[ 2023-09-21 09:05:32 ] Completed train batch 0 generator backward 520.331 ms, 3.76 s total +[ 2023-09-21 09:05:32 ] Completed train batch 0 discriminator loss 0.061 56.570 ms, 3.82 s total +[ 2023-09-21 09:05:32 ] Completed train batch 0 discriminator backward 99.804 ms, 3.92 s total +Epoch [101] Step [0/2], gen_loss: 0.423, disc_loss: 0.020 +[ 2023-09-21 09:05:32 ] Completed train batch 0 metrics update 26.518 ms, 3.95 s total +Saving checkpoint at epoch 101 train batch 0 +[ 2023-09-21 09:05:33 ] Completed saving temp checkpoint 439.259 ms, 4.38 s total +[ 2023-09-21 09:05:33 ] Completed replacing temp checkpoint with checkpoint 21.629 ms, 4.41 s total +[ 2023-09-21 09:05:35 ] Completed train batch 1 to device 1,649.682 ms, 6.06 s total +[ 2023-09-21 09:05:35 ] Completed train batch 1 generator forward 41.247 ms, 6.10 s total +[ 2023-09-21 09:05:35 ] Completed train batch 1 generator loss: 1.236 64.109 ms, 6.16 s total +[ 2023-09-21 09:05:36 ] Completed train batch 1 generator backward 1,175.436 ms, 7.34 s total +[ 2023-09-21 09:05:36 ] Completed train batch 1 discriminator loss 0.061 56.161 ms, 7.39 s total +[ 2023-09-21 09:05:36 ] Completed train batch 1 discriminator backward 87.835 ms, 7.48 s total +Epoch [101] Step [1/2], gen_loss: 0.414, disc_loss: 0.020 +[ 2023-09-21 09:05:36 ] Completed train batch 1 metrics update 5.363 ms, 7.49 s total +Saving checkpoint at epoch 101 train batch 1 +[ 2023-09-21 09:05:37 ] Completed saving temp checkpoint 493.990 ms, 7.98 s total +[ 2023-09-21 09:05:37 ] Completed replacing temp checkpoint with checkpoint 18.647 ms, 8.00 s total +Epoch [101] :: gen_loss: 0.419, disc_loss: 0.020 +[ 2023-09-21 09:05:37 ] Completed training generator for epoch 101 55.961 ms, 8.06 s total +[ 2023-09-21 09:05:37 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 101 from eval step 0 + +[ 2023-09-21 09:05:38 ] Completed eval batch 0 to device 1,063.648 ms, 1.06 s total +[ 2023-09-21 09:05:38 ] Completed eval batch 0 forward 31.081 ms, 1.09 s total +[ 2023-09-21 09:05:38 ] Completed eval batch 0 recons_loss 0.479 ms, 1.10 s total +[ 2023-09-21 09:05:38 ] Completed eval batch 0 metrics update 0.606 ms, 1.10 s total +Saving checkpoint at epoch 101 val batch 0 +[ 2023-09-21 09:05:38 ] Completed saving temp checkpoint 393.530 ms, 1.49 s total +[ 2023-09-21 09:05:38 ] Completed replacing temp checkpoint with checkpoint 23.116 ms, 1.51 s total +[ 2023-09-21 09:05:39 ] Completed eval batch 1 to device 427.068 ms, 1.94 s total +[ 2023-09-21 09:05:39 ] Completed eval batch 1 forward 21.093 ms, 1.96 s total +[ 2023-09-21 09:05:39 ] Completed eval batch 1 recons_loss 0.332 ms, 1.96 s total +[ 2023-09-21 09:05:39 ] Completed eval batch 1 metrics update 290.689 ms, 2.25 s total +Saving checkpoint at epoch 101 val batch 1 +Epoch 101 val loss: 0.0689 +[ 2023-09-21 09:05:39 ] Completed saving temp checkpoint 438.752 ms, 2.69 s total +[ 2023-09-21 09:05:39 ] Completed replacing temp checkpoint with checkpoint 23.247 ms, 2.71 s total +[ 2023-09-21 09:05:39 ] Completed evaluating generator for epoch 101 48.632 ms, 2.76 s total + + +EPOCH :: 102 + + +[ 2023-09-21 09:05:39 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 102 from training step 0 + +[ 2023-09-21 09:05:42 ] Completed train batch 0 to device 2,988.940 ms, 2.99 s total +[ 2023-09-21 09:05:42 ] Completed train batch 0 generator forward 32.924 ms, 3.02 s total +[ 2023-09-21 09:05:42 ] Completed train batch 0 generator loss: 1.303 70.749 ms, 3.09 s total +[ 2023-09-21 09:05:43 ] Completed train batch 0 generator backward 840.396 ms, 3.93 s total +[ 2023-09-21 09:05:43 ] Completed train batch 0 discriminator loss 0.063 56.352 ms, 3.99 s total +[ 2023-09-21 09:05:43 ] Completed train batch 0 discriminator backward 89.530 ms, 4.08 s total +Epoch [102] Step [0/2], gen_loss: 0.420, disc_loss: 0.021 +[ 2023-09-21 09:05:43 ] Completed train batch 0 metrics update 5.531 ms, 4.08 s total +Saving checkpoint at epoch 102 train batch 0 +[ 2023-09-21 09:05:44 ] Completed saving temp checkpoint 398.136 ms, 4.48 s total +[ 2023-09-21 09:05:44 ] Completed replacing temp checkpoint with checkpoint 29.949 ms, 4.51 s total +[ 2023-09-21 09:05:45 ] Completed train batch 1 to device 1,309.919 ms, 5.82 s total +[ 2023-09-21 09:05:45 ] Completed train batch 1 generator forward 23.978 ms, 5.85 s total +[ 2023-09-21 09:05:45 ] Completed train batch 1 generator loss: 1.316 62.174 ms, 5.91 s total +[ 2023-09-21 09:05:47 ] Completed train batch 1 generator backward 1,569.532 ms, 7.48 s total +[ 2023-09-21 09:05:47 ] Completed train batch 1 discriminator loss 0.061 57.146 ms, 7.54 s total +[ 2023-09-21 09:05:47 ] Completed train batch 1 discriminator backward 67.695 ms, 7.60 s total +Epoch [102] Step [1/2], gen_loss: 0.435, disc_loss: 0.020 +[ 2023-09-21 09:05:47 ] Completed train batch 1 metrics update 0.346 ms, 7.60 s total +Saving checkpoint at epoch 102 train batch 1 +[ 2023-09-21 09:05:47 ] Completed saving temp checkpoint 431.587 ms, 8.03 s total +[ 2023-09-21 09:05:47 ] Completed replacing temp checkpoint with checkpoint 15.982 ms, 8.05 s total +Epoch [102] :: gen_loss: 0.428, disc_loss: 0.021 +[ 2023-09-21 09:05:47 ] Completed training generator for epoch 102 60.162 ms, 8.11 s total +[ 2023-09-21 09:05:47 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 102 from eval step 0 + +[ 2023-09-21 09:05:49 ] Completed eval batch 0 to device 1,088.676 ms, 1.09 s total +[ 2023-09-21 09:05:49 ] Completed eval batch 0 forward 35.919 ms, 1.12 s total +[ 2023-09-21 09:05:49 ] Completed eval batch 0 recons_loss 0.301 ms, 1.12 s total +[ 2023-09-21 09:05:49 ] Completed eval batch 0 metrics update 0.555 ms, 1.13 s total +Saving checkpoint at epoch 102 val batch 0 +[ 2023-09-21 09:05:49 ] Completed saving temp checkpoint 407.814 ms, 1.53 s total +[ 2023-09-21 09:05:49 ] Completed replacing temp checkpoint with checkpoint 27.393 ms, 1.56 s total +[ 2023-09-21 09:05:49 ] Completed eval batch 1 to device 373.202 ms, 1.93 s total +[ 2023-09-21 09:05:49 ] Completed eval batch 1 forward 20.983 ms, 1.95 s total +[ 2023-09-21 09:05:49 ] Completed eval batch 1 recons_loss 0.327 ms, 1.96 s total +[ 2023-09-21 09:05:50 ] Completed eval batch 1 metrics update 325.189 ms, 2.28 s total +Saving checkpoint at epoch 102 val batch 1 +Epoch 102 val loss: 0.0679 +[ 2023-09-21 09:05:50 ] Completed saving temp checkpoint 379.258 ms, 2.66 s total +[ 2023-09-21 09:05:50 ] Completed replacing temp checkpoint with checkpoint 28.764 ms, 2.69 s total +[ 2023-09-21 09:05:50 ] Completed evaluating generator for epoch 102 55.988 ms, 2.74 s total + + +EPOCH :: 103 + + +[ 2023-09-21 09:05:50 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 103 from training step 0 + +[ 2023-09-21 09:05:53 ] Completed train batch 0 to device 3,189.901 ms, 3.19 s total +[ 2023-09-21 09:05:53 ] Completed train batch 0 generator forward 29.903 ms, 3.22 s total +[ 2023-09-21 09:05:54 ] Completed train batch 0 generator loss: 1.297 65.242 ms, 3.29 s total +[ 2023-09-21 09:05:54 ] Completed train batch 0 generator backward 587.118 ms, 3.87 s total +[ 2023-09-21 09:05:54 ] Completed train batch 0 discriminator loss 0.063 59.645 ms, 3.93 s total +[ 2023-09-21 09:05:54 ] Completed train batch 0 discriminator backward 98.898 ms, 4.03 s total +Epoch [103] Step [0/2], gen_loss: 0.413, disc_loss: 0.021 +[ 2023-09-21 09:05:54 ] Completed train batch 0 metrics update 4.318 ms, 4.04 s total +Saving checkpoint at epoch 103 train batch 0 +[ 2023-09-21 09:05:55 ] Completed saving temp checkpoint 369.789 ms, 4.40 s total +[ 2023-09-21 09:05:55 ] Completed replacing temp checkpoint with checkpoint 26.961 ms, 4.43 s total +[ 2023-09-21 09:05:56 ] Completed train batch 1 to device 1,704.911 ms, 6.14 s total +[ 2023-09-21 09:05:56 ] Completed train batch 1 generator forward 41.647 ms, 6.18 s total +[ 2023-09-21 09:05:56 ] Completed train batch 1 generator loss: 1.334 62.267 ms, 6.24 s total +[ 2023-09-21 09:05:58 ] Completed train batch 1 generator backward 1,280.121 ms, 7.52 s total +[ 2023-09-21 09:05:58 ] Completed train batch 1 discriminator loss 0.079 55.960 ms, 7.58 s total +[ 2023-09-21 09:05:58 ] Completed train batch 1 discriminator backward 75.147 ms, 7.65 s total +Epoch [103] Step [1/2], gen_loss: 0.420, disc_loss: 0.026 +[ 2023-09-21 09:05:58 ] Completed train batch 1 metrics update 1.933 ms, 7.65 s total +Saving checkpoint at epoch 103 train batch 1 +[ 2023-09-21 09:05:58 ] Completed saving temp checkpoint 431.677 ms, 8.09 s total +[ 2023-09-21 09:05:58 ] Completed replacing temp checkpoint with checkpoint 22.061 ms, 8.11 s total +Epoch [103] :: gen_loss: 0.417, disc_loss: 0.024 +[ 2023-09-21 09:05:58 ] Completed training generator for epoch 103 55.741 ms, 8.16 s total +[ 2023-09-21 09:05:58 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 103 from eval step 0 + +[ 2023-09-21 09:05:59 ] Completed eval batch 0 to device 1,046.044 ms, 1.05 s total +[ 2023-09-21 09:05:59 ] Completed eval batch 0 forward 23.653 ms, 1.07 s total +[ 2023-09-21 09:05:59 ] Completed eval batch 0 recons_loss 0.255 ms, 1.07 s total +[ 2023-09-21 09:05:59 ] Completed eval batch 0 metrics update 3.817 ms, 1.07 s total +Saving checkpoint at epoch 103 val batch 0 +[ 2023-09-21 09:06:00 ] Completed saving temp checkpoint 386.366 ms, 1.46 s total +[ 2023-09-21 09:06:00 ] Completed replacing temp checkpoint with checkpoint 30.874 ms, 1.49 s total +[ 2023-09-21 09:06:00 ] Completed eval batch 1 to device 499.912 ms, 1.99 s total +[ 2023-09-21 09:06:00 ] Completed eval batch 1 forward 21.123 ms, 2.01 s total +[ 2023-09-21 09:06:00 ] Completed eval batch 1 recons_loss 0.327 ms, 2.01 s total +[ 2023-09-21 09:06:01 ] Completed eval batch 1 metrics update 283.606 ms, 2.30 s total +Saving checkpoint at epoch 103 val batch 1 +Epoch 103 val loss: 0.0602 +[ 2023-09-21 09:06:01 ] Completed saving temp checkpoint 625.393 ms, 2.92 s total +[ 2023-09-21 09:06:01 ] Completed replacing temp checkpoint with checkpoint 28.276 ms, 2.95 s total +[ 2023-09-21 09:06:01 ] Completed evaluating generator for epoch 103 59.947 ms, 3.01 s total + + +EPOCH :: 104 + + +[ 2023-09-21 09:06:01 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 104 from training step 0 + +[ 2023-09-21 09:06:04 ] Completed train batch 0 to device 2,968.447 ms, 2.97 s total +[ 2023-09-21 09:06:04 ] Completed train batch 0 generator forward 27.982 ms, 3.00 s total +[ 2023-09-21 09:06:04 ] Completed train batch 0 generator loss: 1.256 64.689 ms, 3.06 s total +[ 2023-09-21 09:06:05 ] Completed train batch 0 generator backward 603.054 ms, 3.66 s total +[ 2023-09-21 09:06:05 ] Completed train batch 0 discriminator loss 0.098 58.169 ms, 3.72 s total +[ 2023-09-21 09:06:05 ] Completed train batch 0 discriminator backward 86.747 ms, 3.81 s total +Epoch [104] Step [0/2], gen_loss: 0.427, disc_loss: 0.032 +[ 2023-09-21 09:06:05 ] Completed train batch 0 metrics update 5.896 ms, 3.81 s total +Saving checkpoint at epoch 104 train batch 0 +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 09:16:53 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 09:16:53 ] Completed importing Timer 0.027 ms, 0.00 s total +[ 2023-09-21 09:16:58 ] Completed importing everything else 4,703.070 ms, 4.70 s total +| distributed init (rank 4): env:// +| distributed init (rank 3): env:// +| distributed init (rank 5): env:// +| distributed init (rank 1): env:// +| distributed init (rank 0): env:// +| distributed init (rank 2): env:// +[ 2023-09-21 09:17:05 ] Completed preliminaries 7,577.703 ms, 12.28 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 09:17:05 ] Completed build datasets 10.365 ms, 12.29 s total +[ 2023-09-21 09:17:05 ] Completed build samplers 0.046 ms, 12.29 s total +[ 2023-09-21 09:17:05 ] Completed build dataloaders 0.130 ms, 12.29 s total +[ 2023-09-21 09:17:05 ] Completed generator to device 92.476 ms, 12.38 s total +[ 2023-09-21 09:17:05 ] Completed discriminator to device 7.022 ms, 12.39 s total +[ 2023-09-21 09:17:06 ] Completed loss functions 389.794 ms, 12.78 s total +[ 2023-09-21 09:17:06 ] Completed models prepped for distribution 81.892 ms, 12.86 s total +[ 2023-09-21 09:17:06 ] Completed optimizers 0.657 ms, 12.86 s total +[ 2023-09-21 09:17:06 ] Completed grad scalers 0.024 ms, 12.86 s total +[ 2023-09-21 09:17:06 ] Completed checkpoint retrieval 489.597 ms, 13.35 s total + + +EPOCH :: 103 + + +[ 2023-09-21 09:17:06 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 103 from training step 2 + +Epoch [103] :: gen_loss: 0.417, disc_loss: 0.024 +[ 2023-09-21 09:17:07 ] Completed training generator for epoch 103 133.774 ms, 0.13 s total +[ 2023-09-21 09:17:07 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 103 from eval step 2 + +[ 2023-09-21 09:17:07 ] Completed evaluating generator for epoch 103 131.391 ms, 0.13 s total + + +EPOCH :: 104 + + +[ 2023-09-21 09:17:07 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 104 from training step 0 + +[ 2023-09-21 09:17:10 ] Completed train batch 0 to device 3,491.082 ms, 3.49 s total +[ 2023-09-21 09:17:12 ] Completed train batch 0 generator forward 1,432.966 ms, 4.92 s total +[ 2023-09-21 09:17:12 ] Completed train batch 0 generator loss: 1.272 102.698 ms, 5.03 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 09:17:13 ] Completed train batch 0 generator backward 1,136.538 ms, 6.16 s total +[ 2023-09-21 09:17:13 ] Completed train batch 0 discriminator loss 0.095 56.365 ms, 6.22 s total +[ 2023-09-21 09:17:13 ] Completed train batch 0 discriminator backward 94.419 ms, 6.31 s total +Epoch [104] Step [0/2], gen_loss: 0.432, disc_loss: 0.032 +[ 2023-09-21 09:17:13 ] Completed train batch 0 metrics update 5.468 ms, 6.32 s total +Saving checkpoint at epoch 104 train batch 0 +[ 2023-09-21 09:17:13 ] Completed saving temp checkpoint 437.389 ms, 6.76 s total +[ 2023-09-21 09:17:14 ] Completed replacing temp checkpoint with checkpoint 134.987 ms, 6.89 s total +[ 2023-09-21 09:17:14 ] Completed train batch 1 to device 16.722 ms, 6.91 s total +[ 2023-09-21 09:17:14 ] Completed train batch 1 generator forward 23.071 ms, 6.93 s total +[ 2023-09-21 09:17:14 ] Completed train batch 1 generator loss: 1.265 64.497 ms, 7.00 s total +[ 2023-09-21 09:17:15 ] Completed train batch 1 generator backward 1,395.457 ms, 8.39 s total +[ 2023-09-21 09:17:15 ] Completed train batch 1 discriminator loss 0.107 74.888 ms, 8.47 s total +[ 2023-09-21 09:17:15 ] Completed train batch 1 discriminator backward 55.972 ms, 8.52 s total +Epoch [104] Step [1/2], gen_loss: 0.445, disc_loss: 0.033 +[ 2023-09-21 09:17:15 ] Completed train batch 1 metrics update 0.759 ms, 8.52 s total +Saving checkpoint at epoch 104 train batch 1 +[ 2023-09-21 09:17:16 ] Completed saving temp checkpoint 427.909 ms, 8.95 s total +[ 2023-09-21 09:17:16 ] Completed replacing temp checkpoint with checkpoint 17.704 ms, 8.97 s total +Epoch [104] :: gen_loss: 0.439, disc_loss: 0.032 +[ 2023-09-21 09:17:16 ] Completed training generator for epoch 104 46.471 ms, 9.02 s total +[ 2023-09-21 09:17:16 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 104 from eval step 0 + +[ 2023-09-21 09:17:17 ] Completed eval batch 0 to device 1,126.812 ms, 1.13 s total +[ 2023-09-21 09:17:17 ] Completed eval batch 0 forward 25.378 ms, 1.15 s total +[ 2023-09-21 09:17:17 ] Completed eval batch 0 recons_loss 0.313 ms, 1.15 s total +[ 2023-09-21 09:17:17 ] Completed eval batch 0 metrics update 0.570 ms, 1.15 s total +Saving checkpoint at epoch 104 val batch 0 +[ 2023-09-21 09:17:17 ] Completed saving temp checkpoint 427.872 ms, 1.58 s total +[ 2023-09-21 09:17:17 ] Completed replacing temp checkpoint with checkpoint 19.680 ms, 1.60 s total +[ 2023-09-21 09:17:18 ] Completed eval batch 1 to device 415.800 ms, 2.02 s total +[ 2023-09-21 09:17:18 ] Completed eval batch 1 forward 39.407 ms, 2.06 s total +[ 2023-09-21 09:17:18 ] Completed eval batch 1 recons_loss 0.350 ms, 2.06 s total +[ 2023-09-21 09:17:18 ] Completed eval batch 1 metrics update 130.654 ms, 2.19 s total +Saving checkpoint at epoch 104 val batch 1 +Epoch 104 val loss: 0.0713 +[ 2023-09-21 09:17:18 ] Completed saving temp checkpoint 447.219 ms, 2.63 s total +[ 2023-09-21 09:17:18 ] Completed replacing temp checkpoint with checkpoint 17.953 ms, 2.65 s total +[ 2023-09-21 09:17:18 ] Completed evaluating generator for epoch 104 71.275 ms, 2.72 s total + + +EPOCH :: 105 + + +[ 2023-09-21 09:17:18 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 105 from training step 0 + +[ 2023-09-21 09:17:21 ] Completed train batch 0 to device 2,925.706 ms, 2.93 s total +[ 2023-09-21 09:17:21 ] Completed train batch 0 generator forward 25.216 ms, 2.95 s total +[ 2023-09-21 09:17:21 ] Completed train batch 0 generator loss: 1.252 64.793 ms, 3.02 s total +[ 2023-09-21 09:17:22 ] Completed train batch 0 generator backward 693.121 ms, 3.71 s total +[ 2023-09-21 09:17:22 ] Completed train batch 0 discriminator loss 0.074 56.663 ms, 3.77 s total +[ 2023-09-21 09:17:22 ] Completed train batch 0 discriminator backward 102.815 ms, 3.87 s total +Epoch [105] Step [0/2], gen_loss: 0.429, disc_loss: 0.024 +[ 2023-09-21 09:17:22 ] Completed train batch 0 metrics update 5.957 ms, 3.87 s total +Saving checkpoint at epoch 105 train batch 0 +[ 2023-09-21 09:17:23 ] Completed saving temp checkpoint 411.038 ms, 4.29 s total +[ 2023-09-21 09:17:23 ] Completed replacing temp checkpoint with checkpoint 27.329 ms, 4.31 s total +[ 2023-09-21 09:17:24 ] Completed train batch 1 to device 1,210.763 ms, 5.52 s total +[ 2023-09-21 09:17:24 ] Completed train batch 1 generator forward 40.550 ms, 5.56 s total +[ 2023-09-21 09:17:24 ] Completed train batch 1 generator loss: 1.351 61.963 ms, 5.63 s total +[ 2023-09-21 09:17:26 ] Completed train batch 1 generator backward 1,573.357 ms, 7.20 s total +[ 2023-09-21 09:17:26 ] Completed train batch 1 discriminator loss 0.067 55.998 ms, 7.26 s total +[ 2023-09-21 09:17:26 ] Completed train batch 1 discriminator backward 68.353 ms, 7.32 s total +Epoch [105] Step [1/2], gen_loss: 0.440, disc_loss: 0.022 +[ 2023-09-21 09:17:26 ] Completed train batch 1 metrics update 5.151 ms, 7.33 s total +Saving checkpoint at epoch 105 train batch 1 +[ 2023-09-21 09:17:26 ] Completed saving temp checkpoint 447.619 ms, 7.78 s total +[ 2023-09-21 09:17:26 ] Completed replacing temp checkpoint with checkpoint 21.579 ms, 7.80 s total +Epoch [105] :: gen_loss: 0.435, disc_loss: 0.023 +[ 2023-09-21 09:17:26 ] Completed training generator for epoch 105 56.655 ms, 7.85 s total +[ 2023-09-21 09:17:26 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 105 from eval step 0 + +[ 2023-09-21 09:17:27 ] Completed eval batch 0 to device 1,040.964 ms, 1.04 s total +[ 2023-09-21 09:17:27 ] Completed eval batch 0 forward 40.816 ms, 1.08 s total +[ 2023-09-21 09:17:27 ] Completed eval batch 0 recons_loss 0.361 ms, 1.08 s total +[ 2023-09-21 09:17:27 ] Completed eval batch 0 metrics update 0.614 ms, 1.08 s total +Saving checkpoint at epoch 105 val batch 0 +[ 2023-09-21 09:17:28 ] Completed saving temp checkpoint 435.042 ms, 1.52 s total +[ 2023-09-21 09:17:28 ] Completed replacing temp checkpoint with checkpoint 27.927 ms, 1.55 s total +[ 2023-09-21 09:17:28 ] Completed eval batch 1 to device 320.966 ms, 1.87 s total +[ 2023-09-21 09:17:28 ] Completed eval batch 1 forward 39.139 ms, 1.91 s total +[ 2023-09-21 09:17:28 ] Completed eval batch 1 recons_loss 0.365 ms, 1.91 s total +[ 2023-09-21 09:17:29 ] Completed eval batch 1 metrics update 351.710 ms, 2.26 s total +Saving checkpoint at epoch 105 val batch 1 +Epoch 105 val loss: 0.0811 +[ 2023-09-21 09:17:29 ] Completed saving temp checkpoint 470.187 ms, 2.73 s total +[ 2023-09-21 09:17:29 ] Completed replacing temp checkpoint with checkpoint 30.349 ms, 2.76 s total +[ 2023-09-21 09:17:29 ] Completed evaluating generator for epoch 105 55.149 ms, 2.81 s total + + +EPOCH :: 106 + + +[ 2023-09-21 09:17:29 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 106 from training step 0 + +[ 2023-09-21 09:17:32 ] Completed train batch 0 to device 2,825.473 ms, 2.83 s total +[ 2023-09-21 09:17:32 ] Completed train batch 0 generator forward 26.387 ms, 2.85 s total +[ 2023-09-21 09:17:32 ] Completed train batch 0 generator loss: 1.353 66.170 ms, 2.92 s total +[ 2023-09-21 09:17:33 ] Completed train batch 0 generator backward 974.867 ms, 3.89 s total +[ 2023-09-21 09:17:33 ] Completed train batch 0 discriminator loss 0.067 56.847 ms, 3.95 s total +[ 2023-09-21 09:17:33 ] Completed train batch 0 discriminator backward 99.101 ms, 4.05 s total +Epoch [106] Step [0/2], gen_loss: 0.448, disc_loss: 0.022 +[ 2023-09-21 09:17:33 ] Completed train batch 0 metrics update 6.585 ms, 4.06 s total +Saving checkpoint at epoch 106 train batch 0 +[ 2023-09-21 09:17:34 ] Completed saving temp checkpoint 383.525 ms, 4.44 s total +[ 2023-09-21 09:17:34 ] Completed replacing temp checkpoint with checkpoint 26.062 ms, 4.47 s total +[ 2023-09-21 09:17:35 ] Completed train batch 1 to device 1,060.469 ms, 5.53 s total +[ 2023-09-21 09:17:35 ] Completed train batch 1 generator forward 40.411 ms, 5.57 s total +[ 2023-09-21 09:17:35 ] Completed train batch 1 generator loss: 1.328 64.310 ms, 5.63 s total +[ 2023-09-21 09:17:37 ] Completed train batch 1 generator backward 1,980.092 ms, 7.61 s total +[ 2023-09-21 09:17:37 ] Completed train batch 1 discriminator loss 0.063 56.045 ms, 7.67 s total +[ 2023-09-21 09:17:37 ] Completed train batch 1 discriminator backward 87.934 ms, 7.75 s total +Epoch [106] Step [1/2], gen_loss: 0.438, disc_loss: 0.021 +[ 2023-09-21 09:17:37 ] Completed train batch 1 metrics update 5.733 ms, 7.76 s total +Saving checkpoint at epoch 106 train batch 1 +[ 2023-09-21 09:17:37 ] Completed saving temp checkpoint 397.696 ms, 8.16 s total +[ 2023-09-21 09:17:37 ] Completed replacing temp checkpoint with checkpoint 18.307 ms, 8.18 s total +Epoch [106] :: gen_loss: 0.443, disc_loss: 0.022 +[ 2023-09-21 09:17:37 ] Completed training generator for epoch 106 64.150 ms, 8.24 s total +[ 2023-09-21 09:17:37 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 106 from eval step 0 + +[ 2023-09-21 09:17:38 ] Completed eval batch 0 to device 1,080.504 ms, 1.08 s total +[ 2023-09-21 09:17:38 ] Completed eval batch 0 forward 30.507 ms, 1.11 s total +[ 2023-09-21 09:17:38 ] Completed eval batch 0 recons_loss 0.355 ms, 1.11 s total +[ 2023-09-21 09:17:38 ] Completed eval batch 0 metrics update 0.592 ms, 1.11 s total +Saving checkpoint at epoch 106 val batch 0 +[ 2023-09-21 09:17:39 ] Completed saving temp checkpoint 420.331 ms, 1.53 s total +[ 2023-09-21 09:17:39 ] Completed replacing temp checkpoint with checkpoint 28.374 ms, 1.56 s total +[ 2023-09-21 09:17:39 ] Completed eval batch 1 to device 406.590 ms, 1.97 s total +[ 2023-09-21 09:17:39 ] Completed eval batch 1 forward 39.469 ms, 2.01 s total +[ 2023-09-21 09:17:39 ] Completed eval batch 1 recons_loss 0.350 ms, 2.01 s total +[ 2023-09-21 09:17:40 ] Completed eval batch 1 metrics update 237.137 ms, 2.24 s total +Saving checkpoint at epoch 106 val batch 1 +Epoch 106 val loss: 0.0601 +[ 2023-09-21 09:17:40 ] Completed saving temp checkpoint 482.221 ms, 2.73 s total +[ 2023-09-21 09:17:40 ] Completed replacing temp checkpoint with checkpoint 27.473 ms, 2.75 s total +[ 2023-09-21 09:17:40 ] Completed evaluating generator for epoch 106 56.351 ms, 2.81 s total + + +EPOCH :: 107 + + +[ 2023-09-21 09:17:40 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 107 from training step 0 + +[ 2023-09-21 09:17:43 ] Completed train batch 0 to device 2,931.009 ms, 2.93 s total +[ 2023-09-21 09:17:43 ] Completed train batch 0 generator forward 26.728 ms, 2.96 s total +[ 2023-09-21 09:17:43 ] Completed train batch 0 generator loss: 1.273 66.058 ms, 3.02 s total +[ 2023-09-21 09:17:44 ] Completed train batch 0 generator backward 707.040 ms, 3.73 s total +[ 2023-09-21 09:17:44 ] Completed train batch 0 discriminator loss 0.065 57.848 ms, 3.79 s total +[ 2023-09-21 09:17:44 ] Completed train batch 0 discriminator backward 101.132 ms, 3.89 s total +Epoch [107] Step [0/2], gen_loss: 0.425, disc_loss: 0.022 +[ 2023-09-21 09:17:44 ] Completed train batch 0 metrics update 20.268 ms, 3.91 s total +Saving checkpoint at epoch 107 train batch 0 +[ 2023-09-21 09:17:44 ] Completed saving temp checkpoint 435.768 ms, 4.35 s total +[ 2023-09-21 09:17:45 ] Completed replacing temp checkpoint with checkpoint 24.070 ms, 4.37 s total +[ 2023-09-21 09:17:46 ] Completed train batch 1 to device 1,162.653 ms, 5.53 s total +[ 2023-09-21 09:17:46 ] Completed train batch 1 generator forward 40.851 ms, 5.57 s total +[ 2023-09-21 09:17:46 ] Completed train batch 1 generator loss: 1.350 63.652 ms, 5.64 s total +[ 2023-09-21 09:17:47 ] Completed train batch 1 generator backward 1,476.316 ms, 7.11 s total +[ 2023-09-21 09:17:47 ] Completed train batch 1 discriminator loss 0.069 55.974 ms, 7.17 s total +[ 2023-09-21 09:17:47 ] Completed train batch 1 discriminator backward 79.339 ms, 7.25 s total +Epoch [107] Step [1/2], gen_loss: 0.434, disc_loss: 0.023 +[ 2023-09-21 09:17:47 ] Completed train batch 1 metrics update 2.570 ms, 7.25 s total +Saving checkpoint at epoch 107 train batch 1 +[ 2023-09-21 09:17:48 ] Completed saving temp checkpoint 452.107 ms, 7.70 s total +[ 2023-09-21 09:17:48 ] Completed replacing temp checkpoint with checkpoint 20.848 ms, 7.72 s total +Epoch [107] :: gen_loss: 0.430, disc_loss: 0.022 +[ 2023-09-21 09:17:48 ] Completed training generator for epoch 107 58.452 ms, 7.78 s total +[ 2023-09-21 09:17:48 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 107 from eval step 0 + +[ 2023-09-21 09:17:49 ] Completed eval batch 0 to device 1,120.561 ms, 1.12 s total +[ 2023-09-21 09:17:49 ] Completed eval batch 0 forward 28.837 ms, 1.15 s total +[ 2023-09-21 09:17:49 ] Completed eval batch 0 recons_loss 0.276 ms, 1.15 s total +[ 2023-09-21 09:17:49 ] Completed eval batch 0 metrics update 0.730 ms, 1.15 s total +Saving checkpoint at epoch 107 val batch 0 +[ 2023-09-21 09:17:49 ] Completed saving temp checkpoint 424.929 ms, 1.58 s total +[ 2023-09-21 09:17:50 ] Completed replacing temp checkpoint with checkpoint 25.087 ms, 1.60 s total +[ 2023-09-21 09:17:50 ] Completed eval batch 1 to device 462.461 ms, 2.06 s total +[ 2023-09-21 09:17:50 ] Completed eval batch 1 forward 22.138 ms, 2.09 s total +[ 2023-09-21 09:17:50 ] Completed eval batch 1 recons_loss 0.412 ms, 2.09 s total +[ 2023-09-21 09:17:50 ] Completed eval batch 1 metrics update 190.967 ms, 2.28 s total +Saving checkpoint at epoch 107 val batch 1 +Epoch 107 val loss: 0.0555 +[ 2023-09-21 09:17:51 ] Completed saving temp checkpoint 475.392 ms, 2.75 s total +[ 2023-09-21 09:17:51 ] Completed replacing temp checkpoint with checkpoint 29.985 ms, 2.78 s total +[ 2023-09-21 09:17:51 ] Completed evaluating generator for epoch 107 60.941 ms, 2.84 s total + + +EPOCH :: 108 + + +[ 2023-09-21 09:17:51 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 108 from training step 0 + +[ 2023-09-21 09:17:54 ] Completed train batch 0 to device 2,966.358 ms, 2.97 s total +[ 2023-09-21 09:17:54 ] Completed train batch 0 generator forward 42.785 ms, 3.01 s total +[ 2023-09-21 09:17:54 ] Completed train batch 0 generator loss: 1.317 67.067 ms, 3.08 s total +[ 2023-09-21 09:17:55 ] Completed train batch 0 generator backward 669.834 ms, 3.75 s total +[ 2023-09-21 09:17:55 ] Completed train batch 0 discriminator loss 0.062 78.572 ms, 3.82 s total +[ 2023-09-21 09:17:55 ] Completed train batch 0 discriminator backward 69.244 ms, 3.89 s total +Epoch [108] Step [0/2], gen_loss: 0.427, disc_loss: 0.021 +[ 2023-09-21 09:17:55 ] Completed train batch 0 metrics update 7.249 ms, 3.90 s total +Saving checkpoint at epoch 108 train batch 0 +[ 2023-09-21 09:17:55 ] Completed saving temp checkpoint 443.965 ms, 4.35 s total +[ 2023-09-21 09:17:55 ] Completed replacing temp checkpoint with checkpoint 30.461 ms, 4.38 s total +[ 2023-09-21 09:17:57 ] Completed train batch 1 to device 1,416.422 ms, 5.79 s total +[ 2023-09-21 09:17:57 ] Completed train batch 1 generator forward 22.202 ms, 5.81 s total +[ 2023-09-21 09:17:57 ] Completed train batch 1 generator loss: 1.171 63.529 ms, 5.88 s total +[ 2023-09-21 09:17:58 ] Completed train batch 1 generator backward 1,395.786 ms, 7.27 s total +[ 2023-09-21 09:17:58 ] Completed train batch 1 discriminator loss 0.059 75.385 ms, 7.35 s total +[ 2023-09-21 09:17:58 ] Completed train batch 1 discriminator backward 59.752 ms, 7.41 s total +Epoch [108] Step [1/2], gen_loss: 0.424, disc_loss: 0.020 +[ 2023-09-21 09:17:58 ] Completed train batch 1 metrics update 0.764 ms, 7.41 s total +Saving checkpoint at epoch 108 train batch 1 +[ 2023-09-21 09:17:59 ] Completed saving temp checkpoint 462.027 ms, 7.87 s total +[ 2023-09-21 09:17:59 ] Completed replacing temp checkpoint with checkpoint 33.831 ms, 7.91 s total +Epoch [108] :: gen_loss: 0.426, disc_loss: 0.020 +[ 2023-09-21 09:17:59 ] Completed training generator for epoch 108 54.650 ms, 7.96 s total +[ 2023-09-21 09:17:59 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 108 from eval step 0 + +[ 2023-09-21 09:18:00 ] Completed eval batch 0 to device 1,059.971 ms, 1.06 s total +[ 2023-09-21 09:18:00 ] Completed eval batch 0 forward 25.360 ms, 1.09 s total +[ 2023-09-21 09:18:00 ] Completed eval batch 0 recons_loss 0.308 ms, 1.09 s total +[ 2023-09-21 09:18:00 ] Completed eval batch 0 metrics update 0.639 ms, 1.09 s total +Saving checkpoint at epoch 108 val batch 0 +[ 2023-09-21 09:18:00 ] Completed saving temp checkpoint 615.016 ms, 1.70 s total +[ 2023-09-21 09:18:00 ] Completed replacing temp checkpoint with checkpoint 24.679 ms, 1.73 s total +[ 2023-09-21 09:18:01 ] Completed eval batch 1 to device 178.253 ms, 1.90 s total +[ 2023-09-21 09:18:01 ] Completed eval batch 1 forward 39.397 ms, 1.94 s total +[ 2023-09-21 09:18:01 ] Completed eval batch 1 recons_loss 0.366 ms, 1.94 s total +[ 2023-09-21 09:18:01 ] Completed eval batch 1 metrics update 279.392 ms, 2.22 s total +Saving checkpoint at epoch 108 val batch 1 +Epoch 108 val loss: 0.0677 +[ 2023-09-21 09:18:02 ] Completed saving temp checkpoint 665.188 ms, 2.89 s total +[ 2023-09-21 09:18:02 ] Completed replacing temp checkpoint with checkpoint 56.300 ms, 2.94 s total +[ 2023-09-21 09:18:02 ] Completed evaluating generator for epoch 108 59.365 ms, 3.00 s total + + +EPOCH :: 109 + + +[ 2023-09-21 09:18:02 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 109 from training step 0 + +[ 2023-09-21 09:18:05 ] Completed train batch 0 to device 2,793.331 ms, 2.79 s total +[ 2023-09-21 09:18:05 ] Completed train batch 0 generator forward 31.728 ms, 2.83 s total +[ 2023-09-21 09:18:05 ] Completed train batch 0 generator loss: 1.203 66.579 ms, 2.89 s total +[ 2023-09-21 09:18:05 ] Completed train batch 0 generator backward 737.781 ms, 3.63 s total +[ 2023-09-21 09:18:05 ] Completed train batch 0 discriminator loss 0.055 57.727 ms, 3.69 s total +[ 2023-09-21 09:18:05 ] Completed train batch 0 discriminator backward 81.582 ms, 3.77 s total +Epoch [109] Step [0/2], gen_loss: 0.417, disc_loss: 0.019 +[ 2023-09-21 09:18:06 ] Completed train batch 0 metrics update 5.569 ms, 3.77 s total +Saving checkpoint at epoch 109 train batch 0 +[ 2023-09-21 09:18:06 ] Completed saving temp checkpoint 445.431 ms, 4.22 s total +[ 2023-09-21 09:18:06 ] Completed replacing temp checkpoint with checkpoint 29.725 ms, 4.25 s total +[ 2023-09-21 09:18:07 ] Completed train batch 1 to device 1,344.046 ms, 5.59 s total +[ 2023-09-21 09:18:07 ] Completed train batch 1 generator forward 22.302 ms, 5.62 s total +[ 2023-09-21 09:18:07 ] Completed train batch 1 generator loss: 1.284 63.993 ms, 5.68 s total +[ 2023-09-21 09:18:09 ] Completed train batch 1 generator backward 1,572.896 ms, 7.25 s total +[ 2023-09-21 09:18:09 ] Completed train batch 1 discriminator loss 0.060 55.978 ms, 7.31 s total +[ 2023-09-21 09:18:09 ] Completed train batch 1 discriminator backward 68.997 ms, 7.38 s total +Epoch [109] Step [1/2], gen_loss: 0.433, disc_loss: 0.019 +[ 2023-09-21 09:18:09 ] Completed train batch 1 metrics update 1.063 ms, 7.38 s total +Saving checkpoint at epoch 109 train batch 1 +[ 2023-09-21 09:18:10 ] Completed saving temp checkpoint 473.140 ms, 7.85 s total +[ 2023-09-21 09:18:10 ] Completed replacing temp checkpoint with checkpoint 22.917 ms, 7.87 s total +Epoch [109] :: gen_loss: 0.425, disc_loss: 0.019 +[ 2023-09-21 09:18:10 ] Completed training generator for epoch 109 57.350 ms, 7.93 s total +[ 2023-09-21 09:18:10 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 109 from eval step 0 + +[ 2023-09-21 09:18:11 ] Completed eval batch 0 to device 1,120.096 ms, 1.12 s total +[ 2023-09-21 09:18:11 ] Completed eval batch 0 forward 42.745 ms, 1.16 s total +[ 2023-09-21 09:18:11 ] Completed eval batch 0 recons_loss 0.314 ms, 1.16 s total +[ 2023-09-21 09:18:11 ] Completed eval batch 0 metrics update 0.615 ms, 1.16 s total +Saving checkpoint at epoch 109 val batch 0 +[ 2023-09-21 09:18:11 ] Completed saving temp checkpoint 435.866 ms, 1.60 s total +[ 2023-09-21 09:18:11 ] Completed replacing temp checkpoint with checkpoint 26.851 ms, 1.63 s total +[ 2023-09-21 09:18:12 ] Completed eval batch 1 to device 360.741 ms, 1.99 s total +[ 2023-09-21 09:18:12 ] Completed eval batch 1 forward 21.529 ms, 2.01 s total +[ 2023-09-21 09:18:12 ] Completed eval batch 1 recons_loss 0.340 ms, 2.01 s total +[ 2023-09-21 09:18:12 ] Completed eval batch 1 metrics update 168.435 ms, 2.18 s total +Saving checkpoint at epoch 109 val batch 1 +Epoch 109 val loss: 0.0657 +[ 2023-09-21 09:18:12 ] Completed saving temp checkpoint 443.287 ms, 2.62 s total +[ 2023-09-21 09:18:12 ] Completed replacing temp checkpoint with checkpoint 24.200 ms, 2.65 s total +[ 2023-09-21 09:18:12 ] Completed evaluating generator for epoch 109 56.589 ms, 2.70 s total + + +EPOCH :: 110 + + +[ 2023-09-21 09:18:12 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 110 from training step 0 + +[ 2023-09-21 09:18:15 ] Completed train batch 0 to device 3,016.631 ms, 3.02 s total +[ 2023-09-21 09:18:15 ] Completed train batch 0 generator forward 33.138 ms, 3.05 s total +[ 2023-09-21 09:18:15 ] Completed train batch 0 generator loss: 1.266 72.191 ms, 3.12 s total +[ 2023-09-21 09:18:16 ] Completed train batch 0 generator backward 590.816 ms, 3.71 s total +[ 2023-09-21 09:18:16 ] Completed train batch 0 discriminator loss 0.059 57.539 ms, 3.77 s total +[ 2023-09-21 09:18:16 ] Completed train batch 0 discriminator backward 94.061 ms, 3.86 s total +Epoch [110] Step [0/2], gen_loss: 0.421, disc_loss: 0.020 +[ 2023-09-21 09:18:16 ] Completed train batch 0 metrics update 3.869 ms, 3.87 s total +Saving checkpoint at epoch 110 train batch 0 +[ 2023-09-21 09:18:17 ] Completed saving temp checkpoint 456.660 ms, 4.32 s total +[ 2023-09-21 09:18:17 ] Completed replacing temp checkpoint with checkpoint 24.222 ms, 4.35 s total +[ 2023-09-21 09:18:18 ] Completed train batch 1 to device 1,381.370 ms, 5.73 s total +[ 2023-09-21 09:18:18 ] Completed train batch 1 generator forward 22.380 ms, 5.75 s total +[ 2023-09-21 09:18:18 ] Completed train batch 1 generator loss: 1.307 63.467 ms, 5.82 s total +[ 2023-09-21 09:18:19 ] Completed train batch 1 generator backward 1,310.842 ms, 7.13 s total +[ 2023-09-21 09:18:20 ] Completed train batch 1 discriminator loss 0.056 56.198 ms, 7.18 s total +[ 2023-09-21 09:18:20 ] Completed train batch 1 discriminator backward 66.427 ms, 7.25 s total +Epoch [110] Step [1/2], gen_loss: 0.428, disc_loss: 0.019 +[ 2023-09-21 09:18:20 ] Completed train batch 1 metrics update 5.529 ms, 7.26 s total +Saving checkpoint at epoch 110 train batch 1 +[ 2023-09-21 09:18:20 ] Completed saving temp checkpoint 483.696 ms, 7.74 s total +[ 2023-09-21 09:18:20 ] Completed replacing temp checkpoint with checkpoint 31.935 ms, 7.77 s total +Epoch [110] :: gen_loss: 0.425, disc_loss: 0.019 +[ 2023-09-21 09:18:20 ] Completed training generator for epoch 110 57.236 ms, 7.83 s total +[ 2023-09-21 09:18:20 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 110 from eval step 0 + +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 09:29:10 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 09:29:10 ] Completed importing Timer 0.026 ms, 0.00 s total +[ 2023-09-21 09:29:15 ] Completed importing everything else 4,696.381 ms, 4.70 s total +| distributed init (rank 1): env:// +| distributed init (rank 4): env:// +| distributed init (rank 0): env:// +| distributed init (rank 2): env:// +| distributed init (rank 3): env:// +| distributed init (rank 5): env:// +[ 2023-09-21 09:29:23 ] Completed preliminaries 8,133.692 ms, 12.83 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 09:29:23 ] Completed build datasets 11.905 ms, 12.84 s total +[ 2023-09-21 09:29:23 ] Completed build samplers 0.044 ms, 12.84 s total +[ 2023-09-21 09:29:23 ] Completed build dataloaders 0.163 ms, 12.84 s total +[ 2023-09-21 09:29:23 ] Completed generator to device 76.030 ms, 12.92 s total +[ 2023-09-21 09:29:23 ] Completed discriminator to device 7.243 ms, 12.93 s total +[ 2023-09-21 09:29:23 ] Completed loss functions 375.028 ms, 13.30 s total +[ 2023-09-21 09:29:23 ] Completed models prepped for distribution 100.542 ms, 13.40 s total +[ 2023-09-21 09:29:23 ] Completed optimizers 0.689 ms, 13.40 s total +[ 2023-09-21 09:29:23 ] Completed grad scalers 0.024 ms, 13.40 s total +[ 2023-09-21 09:29:24 ] Completed checkpoint retrieval 500.781 ms, 13.90 s total + + +EPOCH :: 110 + + +[ 2023-09-21 09:29:24 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 110 from training step 2 + +Epoch [110] :: gen_loss: 0.425, disc_loss: 0.019 +[ 2023-09-21 09:29:24 ] Completed training generator for epoch 110 116.103 ms, 0.12 s total +[ 2023-09-21 09:29:24 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 110 from eval step 0 + +[ 2023-09-21 09:29:25 ] Completed eval batch 0 to device 1,159.115 ms, 1.16 s total +[ 2023-09-21 09:29:27 ] Completed eval batch 0 forward 1,478.355 ms, 2.64 s total +[ 2023-09-21 09:29:27 ] Completed eval batch 0 recons_loss 0.559 ms, 2.64 s total +[ 2023-09-21 09:29:28 ] Completed eval batch 0 metrics update 995.067 ms, 3.63 s total +Saving checkpoint at epoch 110 val batch 0 +[ 2023-09-21 09:29:28 ] Completed saving temp checkpoint 311.898 ms, 3.94 s total +[ 2023-09-21 09:29:28 ] Completed replacing temp checkpoint with checkpoint 135.004 ms, 4.08 s total +[ 2023-09-21 09:29:28 ] Completed eval batch 1 to device 16.147 ms, 4.10 s total +[ 2023-09-21 09:29:28 ] Completed eval batch 1 forward 27.049 ms, 4.12 s total +[ 2023-09-21 09:29:28 ] Completed eval batch 1 recons_loss 0.403 ms, 4.12 s total +[ 2023-09-21 09:29:28 ] Completed eval batch 1 metrics update 0.334 ms, 4.12 s total +Saving checkpoint at epoch 110 val batch 1 +Epoch 110 val loss: 0.0549 +[ 2023-09-21 09:29:28 ] Completed saving temp checkpoint 409.982 ms, 4.53 s total +[ 2023-09-21 09:29:28 ] Completed replacing temp checkpoint with checkpoint 26.415 ms, 4.56 s total +[ 2023-09-21 09:29:29 ] Completed evaluating generator for epoch 110 37.694 ms, 4.60 s total + + +EPOCH :: 111 + + +[ 2023-09-21 09:29:29 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 111 from training step 0 + +[ 2023-09-21 09:29:32 ] Completed train batch 0 to device 3,067.241 ms, 3.07 s total +[ 2023-09-21 09:29:32 ] Completed train batch 0 generator forward 43.509 ms, 3.11 s total +[ 2023-09-21 09:29:32 ] Completed train batch 0 generator loss: 1.311 80.400 ms, 3.19 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 09:29:33 ] Completed train batch 0 generator backward 827.987 ms, 4.02 s total +[ 2023-09-21 09:29:33 ] Completed train batch 0 discriminator loss 0.054 58.016 ms, 4.08 s total +[ 2023-09-21 09:29:33 ] Completed train batch 0 discriminator backward 95.212 ms, 4.17 s total +Epoch [111] Step [0/2], gen_loss: 0.435, disc_loss: 0.018 +[ 2023-09-21 09:29:33 ] Completed train batch 0 metrics update 6.253 ms, 4.18 s total +Saving checkpoint at epoch 111 train batch 0 +[ 2023-09-21 09:29:33 ] Completed saving temp checkpoint 404.708 ms, 4.58 s total +[ 2023-09-21 09:29:33 ] Completed replacing temp checkpoint with checkpoint 30.282 ms, 4.61 s total +[ 2023-09-21 09:29:34 ] Completed train batch 1 to device 1,298.923 ms, 5.91 s total +[ 2023-09-21 09:29:34 ] Completed train batch 1 generator forward 23.327 ms, 5.94 s total +[ 2023-09-21 09:29:35 ] Completed train batch 1 generator loss: 1.289 63.757 ms, 6.00 s total +[ 2023-09-21 09:29:36 ] Completed train batch 1 generator backward 1,454.202 ms, 7.45 s total +[ 2023-09-21 09:29:36 ] Completed train batch 1 discriminator loss 0.051 56.641 ms, 7.51 s total +[ 2023-09-21 09:29:36 ] Completed train batch 1 discriminator backward 76.873 ms, 7.59 s total +Epoch [111] Step [1/2], gen_loss: 0.437, disc_loss: 0.017 +[ 2023-09-21 09:29:36 ] Completed train batch 1 metrics update 5.185 ms, 7.59 s total +Saving checkpoint at epoch 111 train batch 1 +[ 2023-09-21 09:29:37 ] Completed saving temp checkpoint 477.145 ms, 8.07 s total +[ 2023-09-21 09:29:37 ] Completed replacing temp checkpoint with checkpoint 24.147 ms, 8.09 s total +Epoch [111] :: gen_loss: 0.436, disc_loss: 0.018 +[ 2023-09-21 09:29:37 ] Completed training generator for epoch 111 55.629 ms, 8.15 s total +[ 2023-09-21 09:29:37 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 111 from eval step 0 + +[ 2023-09-21 09:29:38 ] Completed eval batch 0 to device 1,143.203 ms, 1.14 s total +[ 2023-09-21 09:29:38 ] Completed eval batch 0 forward 42.052 ms, 1.19 s total +[ 2023-09-21 09:29:38 ] Completed eval batch 0 recons_loss 0.343 ms, 1.19 s total +[ 2023-09-21 09:29:38 ] Completed eval batch 0 metrics update 0.600 ms, 1.19 s total +Saving checkpoint at epoch 111 val batch 0 +[ 2023-09-21 09:29:38 ] Completed saving temp checkpoint 388.731 ms, 1.57 s total +[ 2023-09-21 09:29:38 ] Completed replacing temp checkpoint with checkpoint 27.608 ms, 1.60 s total +[ 2023-09-21 09:29:39 ] Completed eval batch 1 to device 443.528 ms, 2.05 s total +[ 2023-09-21 09:29:39 ] Completed eval batch 1 forward 39.738 ms, 2.09 s total +[ 2023-09-21 09:29:39 ] Completed eval batch 1 recons_loss 0.557 ms, 2.09 s total +[ 2023-09-21 09:29:39 ] Completed eval batch 1 metrics update 231.386 ms, 2.32 s total +Saving checkpoint at epoch 111 val batch 1 +Epoch 111 val loss: 0.0766 +[ 2023-09-21 09:29:39 ] Completed saving temp checkpoint 403.129 ms, 2.72 s total +[ 2023-09-21 09:29:39 ] Completed replacing temp checkpoint with checkpoint 31.373 ms, 2.75 s total +[ 2023-09-21 09:29:39 ] Completed evaluating generator for epoch 111 56.488 ms, 2.81 s total + + +EPOCH :: 112 + + +[ 2023-09-21 09:29:39 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 112 from training step 0 + +[ 2023-09-21 09:29:43 ] Completed train batch 0 to device 3,129.755 ms, 3.13 s total +[ 2023-09-21 09:29:43 ] Completed train batch 0 generator forward 31.001 ms, 3.16 s total +[ 2023-09-21 09:29:43 ] Completed train batch 0 generator loss: 1.342 67.127 ms, 3.23 s total +[ 2023-09-21 09:29:43 ] Completed train batch 0 generator backward 762.577 ms, 3.99 s total +[ 2023-09-21 09:29:44 ] Completed train batch 0 discriminator loss 0.051 75.656 ms, 4.07 s total +[ 2023-09-21 09:29:44 ] Completed train batch 0 discriminator backward 66.036 ms, 4.13 s total +Epoch [112] Step [0/2], gen_loss: 0.436, disc_loss: 0.017 +[ 2023-09-21 09:29:44 ] Completed train batch 0 metrics update 4.724 ms, 4.14 s total +Saving checkpoint at epoch 112 train batch 0 +[ 2023-09-21 09:29:44 ] Completed saving temp checkpoint 432.910 ms, 4.57 s total +[ 2023-09-21 09:29:44 ] Completed replacing temp checkpoint with checkpoint 31.649 ms, 4.60 s total +[ 2023-09-21 09:29:45 ] Completed train batch 1 to device 1,289.529 ms, 5.89 s total +[ 2023-09-21 09:29:45 ] Completed train batch 1 generator forward 22.437 ms, 5.91 s total +[ 2023-09-21 09:29:45 ] Completed train batch 1 generator loss: 1.318 63.691 ms, 5.98 s total +[ 2023-09-21 09:29:47 ] Completed train batch 1 generator backward 1,458.594 ms, 7.44 s total +[ 2023-09-21 09:29:47 ] Completed train batch 1 discriminator loss 0.055 55.882 ms, 7.49 s total +[ 2023-09-21 09:29:47 ] Completed train batch 1 discriminator backward 71.071 ms, 7.56 s total +Epoch [112] Step [1/2], gen_loss: 0.440, disc_loss: 0.018 +[ 2023-09-21 09:29:47 ] Completed train batch 1 metrics update 5.357 ms, 7.57 s total +Saving checkpoint at epoch 112 train batch 1 +[ 2023-09-21 09:29:47 ] Completed saving temp checkpoint 398.932 ms, 7.97 s total +[ 2023-09-21 09:29:47 ] Completed replacing temp checkpoint with checkpoint 25.443 ms, 7.99 s total +Epoch [112] :: gen_loss: 0.438, disc_loss: 0.017 +[ 2023-09-21 09:29:48 ] Completed training generator for epoch 112 53.334 ms, 8.05 s total +[ 2023-09-21 09:29:48 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 112 from eval step 0 + +[ 2023-09-21 09:29:49 ] Completed eval batch 0 to device 1,109.065 ms, 1.11 s total +[ 2023-09-21 09:29:49 ] Completed eval batch 0 forward 42.095 ms, 1.15 s total +[ 2023-09-21 09:29:49 ] Completed eval batch 0 recons_loss 0.296 ms, 1.15 s total +[ 2023-09-21 09:29:49 ] Completed eval batch 0 metrics update 0.593 ms, 1.15 s total +Saving checkpoint at epoch 112 val batch 0 +[ 2023-09-21 09:29:49 ] Completed saving temp checkpoint 449.074 ms, 1.60 s total +[ 2023-09-21 09:29:49 ] Completed replacing temp checkpoint with checkpoint 35.128 ms, 1.64 s total +[ 2023-09-21 09:29:50 ] Completed eval batch 1 to device 405.268 ms, 2.04 s total +[ 2023-09-21 09:29:50 ] Completed eval batch 1 forward 40.583 ms, 2.08 s total +[ 2023-09-21 09:29:50 ] Completed eval batch 1 recons_loss 0.535 ms, 2.08 s total +[ 2023-09-21 09:29:50 ] Completed eval batch 1 metrics update 234.113 ms, 2.32 s total +Saving checkpoint at epoch 112 val batch 1 +Epoch 112 val loss: 0.0691 +[ 2023-09-21 09:29:50 ] Completed saving temp checkpoint 449.731 ms, 2.77 s total +[ 2023-09-21 09:29:50 ] Completed replacing temp checkpoint with checkpoint 17.203 ms, 2.78 s total +[ 2023-09-21 09:29:50 ] Completed evaluating generator for epoch 112 48.701 ms, 2.83 s total + + +EPOCH :: 113 + + +[ 2023-09-21 09:29:50 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 113 from training step 0 + +[ 2023-09-21 09:29:53 ] Completed train batch 0 to device 2,947.953 ms, 2.95 s total +[ 2023-09-21 09:29:53 ] Completed train batch 0 generator forward 41.328 ms, 2.99 s total +[ 2023-09-21 09:29:53 ] Completed train batch 0 generator loss: 1.230 65.831 ms, 3.06 s total +[ 2023-09-21 09:29:54 ] Completed train batch 0 generator backward 841.883 ms, 3.90 s total +[ 2023-09-21 09:29:54 ] Completed train batch 0 discriminator loss 0.052 76.799 ms, 3.97 s total +[ 2023-09-21 09:29:54 ] Completed train batch 0 discriminator backward 73.219 ms, 4.05 s total +Epoch [113] Step [0/2], gen_loss: 0.421, disc_loss: 0.018 +[ 2023-09-21 09:29:54 ] Completed train batch 0 metrics update 0.874 ms, 4.05 s total +Saving checkpoint at epoch 113 train batch 0 +[ 2023-09-21 09:29:55 ] Completed saving temp checkpoint 426.765 ms, 4.47 s total +[ 2023-09-21 09:29:55 ] Completed replacing temp checkpoint with checkpoint 22.149 ms, 4.50 s total +[ 2023-09-21 09:29:56 ] Completed train batch 1 to device 1,386.722 ms, 5.88 s total +[ 2023-09-21 09:29:56 ] Completed train batch 1 generator forward 38.725 ms, 5.92 s total +[ 2023-09-21 09:29:56 ] Completed train batch 1 generator loss: 1.287 64.720 ms, 5.99 s total +[ 2023-09-21 09:29:58 ] Completed train batch 1 generator backward 1,262.005 ms, 7.25 s total +[ 2023-09-21 09:29:58 ] Completed train batch 1 discriminator loss 0.054 56.503 ms, 7.31 s total +[ 2023-09-21 09:29:58 ] Completed train batch 1 discriminator backward 87.895 ms, 7.39 s total +Epoch [113] Step [1/2], gen_loss: 0.423, disc_loss: 0.019 +[ 2023-09-21 09:29:58 ] Completed train batch 1 metrics update 5.590 ms, 7.40 s total +Saving checkpoint at epoch 113 train batch 1 +[ 2023-09-21 09:29:58 ] Completed saving temp checkpoint 397.665 ms, 7.80 s total +[ 2023-09-21 09:29:58 ] Completed replacing temp checkpoint with checkpoint 15.114 ms, 7.81 s total +Epoch [113] :: gen_loss: 0.422, disc_loss: 0.018 +[ 2023-09-21 09:29:58 ] Completed training generator for epoch 113 61.667 ms, 7.87 s total +[ 2023-09-21 09:29:58 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 113 from eval step 0 + +[ 2023-09-21 09:29:59 ] Completed eval batch 0 to device 1,142.860 ms, 1.14 s total +[ 2023-09-21 09:29:59 ] Completed eval batch 0 forward 24.629 ms, 1.17 s total +[ 2023-09-21 09:29:59 ] Completed eval batch 0 recons_loss 0.304 ms, 1.17 s total +[ 2023-09-21 09:29:59 ] Completed eval batch 0 metrics update 0.561 ms, 1.17 s total +Saving checkpoint at epoch 113 val batch 0 +[ 2023-09-21 09:30:00 ] Completed saving temp checkpoint 331.855 ms, 1.50 s total +[ 2023-09-21 09:30:00 ] Completed replacing temp checkpoint with checkpoint 20.241 ms, 1.52 s total +[ 2023-09-21 09:30:00 ] Completed eval batch 1 to device 553.493 ms, 2.07 s total +[ 2023-09-21 09:30:00 ] Completed eval batch 1 forward 21.471 ms, 2.10 s total +[ 2023-09-21 09:30:00 ] Completed eval batch 1 recons_loss 0.359 ms, 2.10 s total +[ 2023-09-21 09:30:01 ] Completed eval batch 1 metrics update 206.059 ms, 2.30 s total +Saving checkpoint at epoch 113 val batch 1 +Epoch 113 val loss: 0.0568 +[ 2023-09-21 09:30:01 ] Completed saving temp checkpoint 510.531 ms, 2.81 s total +[ 2023-09-21 09:30:01 ] Completed replacing temp checkpoint with checkpoint 24.652 ms, 2.84 s total +[ 2023-09-21 09:30:01 ] Completed evaluating generator for epoch 113 53.658 ms, 2.89 s total + + +EPOCH :: 114 + + +[ 2023-09-21 09:30:01 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 114 from training step 0 + +[ 2023-09-21 09:30:04 ] Completed train batch 0 to device 3,024.969 ms, 3.02 s total +[ 2023-09-21 09:30:04 ] Completed train batch 0 generator forward 26.303 ms, 3.05 s total +[ 2023-09-21 09:30:04 ] Completed train batch 0 generator loss: 1.231 66.007 ms, 3.12 s total +[ 2023-09-21 09:30:05 ] Completed train batch 0 generator backward 658.250 ms, 3.78 s total +[ 2023-09-21 09:30:05 ] Completed train batch 0 discriminator loss 0.053 78.527 ms, 3.85 s total +[ 2023-09-21 09:30:05 ] Completed train batch 0 discriminator backward 65.050 ms, 3.92 s total +Epoch [114] Step [0/2], gen_loss: 0.424, disc_loss: 0.018 +[ 2023-09-21 09:30:05 ] Completed train batch 0 metrics update 5.692 ms, 3.92 s total +Saving checkpoint at epoch 114 train batch 0 +[ 2023-09-21 09:30:05 ] Completed saving temp checkpoint 431.092 ms, 4.36 s total +[ 2023-09-21 09:30:06 ] Completed replacing temp checkpoint with checkpoint 23.732 ms, 4.38 s total +[ 2023-09-21 09:30:07 ] Completed train batch 1 to device 1,471.484 ms, 5.85 s total +[ 2023-09-21 09:30:07 ] Completed train batch 1 generator forward 28.844 ms, 5.88 s total +[ 2023-09-21 09:30:07 ] Completed train batch 1 generator loss: 1.307 65.852 ms, 5.95 s total +[ 2023-09-21 09:30:08 ] Completed train batch 1 generator backward 1,323.229 ms, 7.27 s total +[ 2023-09-21 09:30:08 ] Completed train batch 1 discriminator loss 0.055 55.981 ms, 7.33 s total +[ 2023-09-21 09:30:09 ] Completed train batch 1 discriminator backward 79.360 ms, 7.40 s total +Epoch [114] Step [1/2], gen_loss: 0.440, disc_loss: 0.019 +[ 2023-09-21 09:30:09 ] Completed train batch 1 metrics update 2.170 ms, 7.41 s total +Saving checkpoint at epoch 114 train batch 1 +[ 2023-09-21 09:30:09 ] Completed saving temp checkpoint 478.892 ms, 7.89 s total +[ 2023-09-21 09:30:09 ] Completed replacing temp checkpoint with checkpoint 22.416 ms, 7.91 s total +Epoch [114] :: gen_loss: 0.432, disc_loss: 0.019 +[ 2023-09-21 09:30:09 ] Completed training generator for epoch 114 48.238 ms, 7.96 s total +[ 2023-09-21 09:30:09 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 114 from eval step 0 + +[ 2023-09-21 09:30:10 ] Completed eval batch 0 to device 1,149.237 ms, 1.15 s total +[ 2023-09-21 09:30:10 ] Completed eval batch 0 forward 39.951 ms, 1.19 s total +[ 2023-09-21 09:30:10 ] Completed eval batch 0 recons_loss 0.299 ms, 1.19 s total +[ 2023-09-21 09:30:10 ] Completed eval batch 0 metrics update 0.594 ms, 1.19 s total +Saving checkpoint at epoch 114 val batch 0 +[ 2023-09-21 09:30:11 ] Completed saving temp checkpoint 381.257 ms, 1.57 s total +[ 2023-09-21 09:30:11 ] Completed replacing temp checkpoint with checkpoint 20.949 ms, 1.59 s total +[ 2023-09-21 09:30:11 ] Completed eval batch 1 to device 412.089 ms, 2.00 s total +[ 2023-09-21 09:30:11 ] Completed eval batch 1 forward 38.897 ms, 2.04 s total +[ 2023-09-21 09:30:11 ] Completed eval batch 1 recons_loss 0.329 ms, 2.04 s total +[ 2023-09-21 09:30:11 ] Completed eval batch 1 metrics update 223.763 ms, 2.27 s total +Saving checkpoint at epoch 114 val batch 1 +Epoch 114 val loss: 0.0578 +[ 2023-09-21 09:30:12 ] Completed saving temp checkpoint 402.906 ms, 2.67 s total +[ 2023-09-21 09:30:12 ] Completed replacing temp checkpoint with checkpoint 24.547 ms, 2.69 s total +[ 2023-09-21 09:30:12 ] Completed evaluating generator for epoch 114 47.036 ms, 2.74 s total + + +EPOCH :: 115 + + +[ 2023-09-21 09:30:12 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 115 from training step 0 + +[ 2023-09-21 09:30:15 ] Completed train batch 0 to device 3,309.985 ms, 3.31 s total +[ 2023-09-21 09:30:15 ] Completed train batch 0 generator forward 32.214 ms, 3.34 s total +[ 2023-09-21 09:30:15 ] Completed train batch 0 generator loss: 1.253 66.632 ms, 3.41 s total +[ 2023-09-21 09:30:16 ] Completed train batch 0 generator backward 570.529 ms, 3.98 s total +[ 2023-09-21 09:30:16 ] Completed train batch 0 discriminator loss 0.061 56.856 ms, 4.04 s total +[ 2023-09-21 09:30:16 ] Completed train batch 0 discriminator backward 80.444 ms, 4.12 s total +Epoch [115] Step [0/2], gen_loss: 0.426, disc_loss: 0.020 +[ 2023-09-21 09:30:16 ] Completed train batch 0 metrics update 24.104 ms, 4.14 s total +Saving checkpoint at epoch 115 train batch 0 +[ 2023-09-21 09:30:16 ] Completed saving temp checkpoint 445.688 ms, 4.59 s total +[ 2023-09-21 09:30:16 ] Completed replacing temp checkpoint with checkpoint 31.571 ms, 4.62 s total +[ 2023-09-21 09:30:18 ] Completed train batch 1 to device 1,565.106 ms, 6.18 s total +[ 2023-09-21 09:30:18 ] Completed train batch 1 generator forward 42.553 ms, 6.23 s total +[ 2023-09-21 09:30:18 ] Completed train batch 1 generator loss: 1.252 63.711 ms, 6.29 s total +[ 2023-09-21 09:30:19 ] Completed train batch 1 generator backward 1,326.217 ms, 7.62 s total +[ 2023-09-21 09:30:19 ] Completed train batch 1 discriminator loss 0.057 56.108 ms, 7.67 s total +[ 2023-09-21 09:30:20 ] Completed train batch 1 discriminator backward 69.824 ms, 7.74 s total +Epoch [115] Step [1/2], gen_loss: 0.427, disc_loss: 0.020 +[ 2023-09-21 09:30:20 ] Completed train batch 1 metrics update 1.609 ms, 7.74 s total +Saving checkpoint at epoch 115 train batch 1 +[ 2023-09-21 09:30:20 ] Completed saving temp checkpoint 462.349 ms, 8.21 s total +[ 2023-09-21 09:30:20 ] Completed replacing temp checkpoint with checkpoint 32.442 ms, 8.24 s total +Epoch [115] :: gen_loss: 0.427, disc_loss: 0.020 +[ 2023-09-21 09:30:20 ] Completed training generator for epoch 115 52.448 ms, 8.29 s total +[ 2023-09-21 09:30:20 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 115 from eval step 0 + +[ 2023-09-21 09:30:21 ] Completed eval batch 0 to device 1,078.928 ms, 1.08 s total +[ 2023-09-21 09:30:21 ] Completed eval batch 0 forward 42.658 ms, 1.12 s total +[ 2023-09-21 09:30:21 ] Completed eval batch 0 recons_loss 0.305 ms, 1.12 s total +[ 2023-09-21 09:30:21 ] Completed eval batch 0 metrics update 0.593 ms, 1.12 s total +Saving checkpoint at epoch 115 val batch 0 +[ 2023-09-21 09:30:22 ] Completed saving temp checkpoint 430.575 ms, 1.55 s total +[ 2023-09-21 09:30:22 ] Completed replacing temp checkpoint with checkpoint 26.472 ms, 1.58 s total +[ 2023-09-21 09:30:22 ] Completed eval batch 1 to device 349.128 ms, 1.93 s total +[ 2023-09-21 09:30:22 ] Completed eval batch 1 forward 39.783 ms, 1.97 s total +[ 2023-09-21 09:30:22 ] Completed eval batch 1 recons_loss 0.575 ms, 1.97 s total +[ 2023-09-21 09:30:22 ] Completed eval batch 1 metrics update 359.061 ms, 2.33 s total +Saving checkpoint at epoch 115 val batch 1 +Epoch 115 val loss: 0.0636 +[ 2023-09-21 09:30:23 ] Completed saving temp checkpoint 440.609 ms, 2.77 s total +[ 2023-09-21 09:30:23 ] Completed replacing temp checkpoint with checkpoint 30.283 ms, 2.80 s total +[ 2023-09-21 09:30:23 ] Completed evaluating generator for epoch 115 50.499 ms, 2.85 s total + + +EPOCH :: 116 + + +[ 2023-09-21 09:30:23 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 116 from training step 0 + +[ 2023-09-21 09:30:26 ] Completed train batch 0 to device 3,105.298 ms, 3.11 s total +[ 2023-09-21 09:30:26 ] Completed train batch 0 generator forward 38.900 ms, 3.14 s total +[ 2023-09-21 09:30:26 ] Completed train batch 0 generator loss: 1.263 65.063 ms, 3.21 s total +[ 2023-09-21 09:30:27 ] Completed train batch 0 generator backward 747.558 ms, 3.96 s total +[ 2023-09-21 09:30:27 ] Completed train batch 0 discriminator loss 0.065 55.780 ms, 4.01 s total +[ 2023-09-21 09:30:27 ] Completed train batch 0 discriminator backward 107.789 ms, 4.12 s total +Epoch [116] Step [0/2], gen_loss: 0.428, disc_loss: 0.020 +[ 2023-09-21 09:30:27 ] Completed train batch 0 metrics update 6.341 ms, 4.13 s total +Saving checkpoint at epoch 116 train batch 0 +[ 2023-09-21 09:30:28 ] Completed saving temp checkpoint 437.934 ms, 4.56 s total +[ 2023-09-21 09:30:28 ] Completed replacing temp checkpoint with checkpoint 22.109 ms, 4.59 s total +[ 2023-09-21 09:30:29 ] Completed train batch 1 to device 1,332.801 ms, 5.92 s total +[ 2023-09-21 09:30:29 ] Completed train batch 1 generator forward 39.157 ms, 5.96 s total +[ 2023-09-21 09:30:29 ] Completed train batch 1 generator loss: 1.279 63.477 ms, 6.02 s total +[ 2023-09-21 09:30:31 ] Completed train batch 1 generator backward 1,540.720 ms, 7.56 s total +[ 2023-09-21 09:30:31 ] Completed train batch 1 discriminator loss 0.058 55.708 ms, 7.62 s total +[ 2023-09-21 09:30:31 ] Completed train batch 1 discriminator backward 90.337 ms, 7.71 s total +Epoch [116] Step [1/2], gen_loss: 0.434, disc_loss: 0.019 +[ 2023-09-21 09:30:31 ] Completed train batch 1 metrics update 5.925 ms, 7.71 s total +Saving checkpoint at epoch 116 train batch 1 +[ 2023-09-21 09:30:31 ] Completed saving temp checkpoint 439.608 ms, 8.15 s total +[ 2023-09-21 09:30:31 ] Completed replacing temp checkpoint with checkpoint 29.782 ms, 8.18 s total +Epoch [116] :: gen_loss: 0.431, disc_loss: 0.020 +[ 2023-09-21 09:30:31 ] Completed training generator for epoch 116 58.128 ms, 8.24 s total +[ 2023-09-21 09:30:31 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 116 from eval step 0 + +[ 2023-09-21 09:30:32 ] Completed eval batch 0 to device 1,161.594 ms, 1.16 s total +[ 2023-09-21 09:30:32 ] Completed eval batch 0 forward 26.062 ms, 1.19 s total +[ 2023-09-21 09:30:32 ] Completed eval batch 0 recons_loss 0.294 ms, 1.19 s total +[ 2023-09-21 09:30:32 ] Completed eval batch 0 metrics update 0.571 ms, 1.19 s total +Saving checkpoint at epoch 116 val batch 0 +[ 2023-09-21 09:30:33 ] Completed saving temp checkpoint 410.168 ms, 1.60 s total +[ 2023-09-21 09:30:33 ] Completed replacing temp checkpoint with checkpoint 30.178 ms, 1.63 s total +[ 2023-09-21 09:30:33 ] Completed eval batch 1 to device 427.139 ms, 2.06 s total +[ 2023-09-21 09:30:33 ] Completed eval batch 1 forward 39.811 ms, 2.10 s total +[ 2023-09-21 09:30:33 ] Completed eval batch 1 recons_loss 0.575 ms, 2.10 s total +[ 2023-09-21 09:30:34 ] Completed eval batch 1 metrics update 211.731 ms, 2.31 s total +Saving checkpoint at epoch 116 val batch 1 +Epoch 116 val loss: 0.0621 +[ 2023-09-21 09:30:34 ] Completed saving temp checkpoint 644.811 ms, 2.95 s total +[ 2023-09-21 09:30:34 ] Completed replacing temp checkpoint with checkpoint 34.986 ms, 2.99 s total +[ 2023-09-21 09:30:34 ] Completed evaluating generator for epoch 116 53.591 ms, 3.04 s total + + +EPOCH :: 117 + + +[ 2023-09-21 09:30:34 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 117 from training step 0 + +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 09:41:22 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 09:41:22 ] Completed importing Timer 0.023 ms, 0.00 s total +[ 2023-09-21 09:41:27 ] Completed importing everything else 4,728.866 ms, 4.73 s total +| distributed init (rank 1): env:// +| distributed init (rank 3): env:// +| distributed init (rank 5): env:// +| distributed init (rank 4): env:// +| distributed init (rank 0): env:// +| distributed init (rank 2): env:// +[ 2023-09-21 09:41:35 ] Completed preliminaries 7,930.269 ms, 12.66 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 09:41:35 ] Completed build datasets 11.411 ms, 12.67 s total +[ 2023-09-21 09:41:35 ] Completed build samplers 0.047 ms, 12.67 s total +[ 2023-09-21 09:41:35 ] Completed build dataloaders 0.131 ms, 12.67 s total +[ 2023-09-21 09:41:35 ] Completed generator to device 75.664 ms, 12.75 s total +[ 2023-09-21 09:41:35 ] Completed discriminator to device 9.736 ms, 12.76 s total +[ 2023-09-21 09:41:35 ] Completed loss functions 419.560 ms, 13.18 s total +[ 2023-09-21 09:41:35 ] Completed models prepped for distribution 81.504 ms, 13.26 s total +[ 2023-09-21 09:41:35 ] Completed optimizers 0.673 ms, 13.26 s total +[ 2023-09-21 09:41:35 ] Completed grad scalers 0.030 ms, 13.26 s total +[ 2023-09-21 09:41:36 ] Completed checkpoint retrieval 497.827 ms, 13.76 s total + + +EPOCH :: 116 + + +[ 2023-09-21 09:41:36 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 116 from training step 2 + +Epoch [116] :: gen_loss: 0.431, disc_loss: 0.020 +[ 2023-09-21 09:41:36 ] Completed training generator for epoch 116 145.785 ms, 0.15 s total +[ 2023-09-21 09:41:36 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 116 from eval step 2 + +[ 2023-09-21 09:41:36 ] Completed evaluating generator for epoch 116 127.267 ms, 0.13 s total + + +EPOCH :: 117 + + +[ 2023-09-21 09:41:36 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 117 from training step 0 + +[ 2023-09-21 09:41:40 ] Completed train batch 0 to device 3,998.699 ms, 4.00 s total +[ 2023-09-21 09:41:42 ] Completed train batch 0 generator forward 1,611.274 ms, 5.61 s total +[ 2023-09-21 09:41:42 ] Completed train batch 0 generator loss: 1.239 81.843 ms, 5.69 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 09:41:42 ] Completed train batch 0 generator backward 563.097 ms, 6.25 s total +[ 2023-09-21 09:41:43 ] Completed train batch 0 discriminator loss 0.058 57.417 ms, 6.31 s total +[ 2023-09-21 09:41:43 ] Completed train batch 0 discriminator backward 106.414 ms, 6.42 s total +Epoch [117] Step [0/2], gen_loss: 0.418, disc_loss: 0.019 +[ 2023-09-21 09:41:43 ] Completed train batch 0 metrics update 7.296 ms, 6.43 s total +Saving checkpoint at epoch 117 train batch 0 +[ 2023-09-21 09:41:43 ] Completed saving temp checkpoint 441.549 ms, 6.87 s total +[ 2023-09-21 09:41:43 ] Completed replacing temp checkpoint with checkpoint 129.946 ms, 7.00 s total +[ 2023-09-21 09:41:44 ] Completed train batch 1 to device 595.843 ms, 7.59 s total +[ 2023-09-21 09:41:44 ] Completed train batch 1 generator forward 22.147 ms, 7.62 s total +[ 2023-09-21 09:41:44 ] Completed train batch 1 generator loss: 1.238 61.605 ms, 7.68 s total +[ 2023-09-21 09:41:44 ] Completed train batch 1 generator backward 516.666 ms, 8.19 s total +[ 2023-09-21 09:41:44 ] Completed train batch 1 discriminator loss 0.055 56.108 ms, 8.25 s total +[ 2023-09-21 09:41:45 ] Completed train batch 1 discriminator backward 85.853 ms, 8.34 s total +Epoch [117] Step [1/2], gen_loss: 0.411, disc_loss: 0.018 +[ 2023-09-21 09:41:45 ] Completed train batch 1 metrics update 5.933 ms, 8.34 s total +Saving checkpoint at epoch 117 train batch 1 +[ 2023-09-21 09:41:45 ] Completed saving temp checkpoint 415.934 ms, 8.76 s total +[ 2023-09-21 09:41:45 ] Completed replacing temp checkpoint with checkpoint 24.245 ms, 8.78 s total +Epoch [117] :: gen_loss: 0.415, disc_loss: 0.019 +[ 2023-09-21 09:41:45 ] Completed training generator for epoch 117 43.606 ms, 8.83 s total +[ 2023-09-21 09:41:45 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 117 from eval step 0 + +[ 2023-09-21 09:41:46 ] Completed eval batch 0 to device 1,142.096 ms, 1.14 s total +[ 2023-09-21 09:41:46 ] Completed eval batch 0 forward 25.142 ms, 1.17 s total +[ 2023-09-21 09:41:46 ] Completed eval batch 0 recons_loss 0.304 ms, 1.17 s total +[ 2023-09-21 09:41:46 ] Completed eval batch 0 metrics update 0.619 ms, 1.17 s total +Saving checkpoint at epoch 117 val batch 0 +[ 2023-09-21 09:41:47 ] Completed saving temp checkpoint 391.948 ms, 1.56 s total +[ 2023-09-21 09:41:47 ] Completed replacing temp checkpoint with checkpoint 25.718 ms, 1.59 s total +[ 2023-09-21 09:41:47 ] Completed eval batch 1 to device 521.894 ms, 2.11 s total +[ 2023-09-21 09:41:47 ] Completed eval batch 1 forward 39.687 ms, 2.15 s total +[ 2023-09-21 09:41:47 ] Completed eval batch 1 recons_loss 0.573 ms, 2.15 s total +[ 2023-09-21 09:41:47 ] Completed eval batch 1 metrics update 5.144 ms, 2.15 s total +Saving checkpoint at epoch 117 val batch 1 +Epoch 117 val loss: 0.0627 +[ 2023-09-21 09:41:48 ] Completed saving temp checkpoint 473.784 ms, 2.63 s total +[ 2023-09-21 09:41:48 ] Completed replacing temp checkpoint with checkpoint 30.960 ms, 2.66 s total +[ 2023-09-21 09:41:48 ] Completed evaluating generator for epoch 117 49.776 ms, 2.71 s total + + +EPOCH :: 118 + + +[ 2023-09-21 09:41:48 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 118 from training step 0 + +[ 2023-09-21 09:41:51 ] Completed train batch 0 to device 3,150.309 ms, 3.15 s total +[ 2023-09-21 09:41:51 ] Completed train batch 0 generator forward 31.580 ms, 3.18 s total +[ 2023-09-21 09:41:51 ] Completed train batch 0 generator loss: 1.214 66.743 ms, 3.25 s total +[ 2023-09-21 09:41:51 ] Completed train batch 0 generator backward 288.448 ms, 3.54 s total +[ 2023-09-21 09:41:51 ] Completed train batch 0 discriminator loss 0.055 66.255 ms, 3.60 s total +[ 2023-09-21 09:41:51 ] Completed train batch 0 discriminator backward 85.284 ms, 3.69 s total +Epoch [118] Step [0/2], gen_loss: 0.416, disc_loss: 0.018 +[ 2023-09-21 09:41:51 ] Completed train batch 0 metrics update 6.930 ms, 3.70 s total +Saving checkpoint at epoch 118 train batch 0 +[ 2023-09-21 09:41:52 ] Completed saving temp checkpoint 467.222 ms, 4.16 s total +[ 2023-09-21 09:41:52 ] Completed replacing temp checkpoint with checkpoint 23.350 ms, 4.19 s total +[ 2023-09-21 09:41:54 ] Completed train batch 1 to device 1,985.198 ms, 6.17 s total +[ 2023-09-21 09:41:54 ] Completed train batch 1 generator forward 41.170 ms, 6.21 s total +[ 2023-09-21 09:41:54 ] Completed train batch 1 generator loss: 1.237 65.028 ms, 6.28 s total +[ 2023-09-21 09:41:55 ] Completed train batch 1 generator backward 489.596 ms, 6.77 s total +[ 2023-09-21 09:41:55 ] Completed train batch 1 discriminator loss 0.053 55.602 ms, 6.82 s total +[ 2023-09-21 09:41:55 ] Completed train batch 1 discriminator backward 88.643 ms, 6.91 s total +Epoch [118] Step [1/2], gen_loss: 0.415, disc_loss: 0.018 +[ 2023-09-21 09:41:55 ] Completed train batch 1 metrics update 4.263 ms, 6.92 s total +Saving checkpoint at epoch 118 train batch 1 +[ 2023-09-21 09:41:55 ] Completed saving temp checkpoint 452.248 ms, 7.37 s total +[ 2023-09-21 09:41:55 ] Completed replacing temp checkpoint with checkpoint 27.271 ms, 7.40 s total +Epoch [118] :: gen_loss: 0.416, disc_loss: 0.018 +[ 2023-09-21 09:41:55 ] Completed training generator for epoch 118 58.104 ms, 7.45 s total +[ 2023-09-21 09:41:55 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 118 from eval step 0 + +[ 2023-09-21 09:41:56 ] Completed eval batch 0 to device 1,121.893 ms, 1.12 s total +[ 2023-09-21 09:41:56 ] Completed eval batch 0 forward 40.152 ms, 1.16 s total +[ 2023-09-21 09:41:56 ] Completed eval batch 0 recons_loss 0.304 ms, 1.16 s total +[ 2023-09-21 09:41:56 ] Completed eval batch 0 metrics update 0.669 ms, 1.16 s total +Saving checkpoint at epoch 118 val batch 0 +[ 2023-09-21 09:41:57 ] Completed saving temp checkpoint 426.901 ms, 1.59 s total +[ 2023-09-21 09:41:57 ] Completed replacing temp checkpoint with checkpoint 32.480 ms, 1.62 s total +[ 2023-09-21 09:41:57 ] Completed eval batch 1 to device 422.905 ms, 2.05 s total +[ 2023-09-21 09:41:57 ] Completed eval batch 1 forward 38.696 ms, 2.08 s total +[ 2023-09-21 09:41:57 ] Completed eval batch 1 recons_loss 0.355 ms, 2.08 s total +[ 2023-09-21 09:41:57 ] Completed eval batch 1 metrics update 94.259 ms, 2.18 s total +Saving checkpoint at epoch 118 val batch 1 +Epoch 118 val loss: 0.0611 +[ 2023-09-21 09:41:58 ] Completed saving temp checkpoint 462.384 ms, 2.64 s total +[ 2023-09-21 09:41:58 ] Completed replacing temp checkpoint with checkpoint 29.998 ms, 2.67 s total +[ 2023-09-21 09:41:58 ] Completed evaluating generator for epoch 118 54.379 ms, 2.73 s total + + +EPOCH :: 119 + + +[ 2023-09-21 09:41:58 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 119 from training step 0 + +[ 2023-09-21 09:42:01 ] Completed train batch 0 to device 3,110.731 ms, 3.11 s total +[ 2023-09-21 09:42:01 ] Completed train batch 0 generator forward 30.804 ms, 3.14 s total +[ 2023-09-21 09:42:01 ] Completed train batch 0 generator loss: 1.257 67.162 ms, 3.21 s total +[ 2023-09-21 09:42:02 ] Completed train batch 0 generator backward 422.513 ms, 3.63 s total +[ 2023-09-21 09:42:02 ] Completed train batch 0 discriminator loss 0.056 77.229 ms, 3.71 s total +[ 2023-09-21 09:42:02 ] Completed train batch 0 discriminator backward 79.712 ms, 3.79 s total +Epoch [119] Step [0/2], gen_loss: 0.412, disc_loss: 0.018 +[ 2023-09-21 09:42:02 ] Completed train batch 0 metrics update 0.890 ms, 3.79 s total +Saving checkpoint at epoch 119 train batch 0 +[ 2023-09-21 09:42:02 ] Completed saving temp checkpoint 443.666 ms, 4.23 s total +[ 2023-09-21 09:42:02 ] Completed replacing temp checkpoint with checkpoint 27.612 ms, 4.26 s total +[ 2023-09-21 09:42:04 ] Completed train batch 1 to device 1,527.336 ms, 5.79 s total +[ 2023-09-21 09:42:04 ] Completed train batch 1 generator forward 40.981 ms, 5.83 s total +[ 2023-09-21 09:42:04 ] Completed train batch 1 generator loss: 1.242 63.777 ms, 5.89 s total +[ 2023-09-21 09:42:05 ] Completed train batch 1 generator backward 1,083.763 ms, 6.98 s total +[ 2023-09-21 09:42:05 ] Completed train batch 1 discriminator loss 0.056 55.938 ms, 7.03 s total +[ 2023-09-21 09:42:05 ] Completed train batch 1 discriminator backward 68.607 ms, 7.10 s total +Epoch [119] Step [1/2], gen_loss: 0.414, disc_loss: 0.019 +[ 2023-09-21 09:42:05 ] Completed train batch 1 metrics update 2.343 ms, 7.10 s total +Saving checkpoint at epoch 119 train batch 1 +[ 2023-09-21 09:42:06 ] Completed saving temp checkpoint 463.136 ms, 7.57 s total +[ 2023-09-21 09:42:06 ] Completed replacing temp checkpoint with checkpoint 16.297 ms, 7.58 s total +Epoch [119] :: gen_loss: 0.413, disc_loss: 0.018 +[ 2023-09-21 09:42:06 ] Completed training generator for epoch 119 50.442 ms, 7.63 s total +[ 2023-09-21 09:42:06 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 119 from eval step 0 + +[ 2023-09-21 09:42:07 ] Completed eval batch 0 to device 1,046.052 ms, 1.05 s total +[ 2023-09-21 09:42:07 ] Completed eval batch 0 forward 41.174 ms, 1.09 s total +[ 2023-09-21 09:42:07 ] Completed eval batch 0 recons_loss 0.233 ms, 1.09 s total +[ 2023-09-21 09:42:07 ] Completed eval batch 0 metrics update 0.523 ms, 1.09 s total +Saving checkpoint at epoch 119 val batch 0 +[ 2023-09-21 09:42:07 ] Completed saving temp checkpoint 446.825 ms, 1.53 s total +[ 2023-09-21 09:42:07 ] Completed replacing temp checkpoint with checkpoint 25.971 ms, 1.56 s total +[ 2023-09-21 09:42:07 ] Completed eval batch 1 to device 353.976 ms, 1.91 s total +[ 2023-09-21 09:42:08 ] Completed eval batch 1 forward 40.941 ms, 1.96 s total +[ 2023-09-21 09:42:08 ] Completed eval batch 1 recons_loss 0.528 ms, 1.96 s total +[ 2023-09-21 09:42:08 ] Completed eval batch 1 metrics update 181.165 ms, 2.14 s total +Saving checkpoint at epoch 119 val batch 1 +Epoch 119 val loss: 0.0591 +[ 2023-09-21 09:42:08 ] Completed saving temp checkpoint 440.291 ms, 2.58 s total +[ 2023-09-21 09:42:08 ] Completed replacing temp checkpoint with checkpoint 18.109 ms, 2.60 s total +[ 2023-09-21 09:42:08 ] Completed evaluating generator for epoch 119 46.998 ms, 2.64 s total + + +EPOCH :: 120 + + +[ 2023-09-21 09:42:08 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 120 from training step 0 + +[ 2023-09-21 09:42:11 ] Completed train batch 0 to device 3,099.790 ms, 3.10 s total +[ 2023-09-21 09:42:11 ] Completed train batch 0 generator forward 31.159 ms, 3.13 s total +[ 2023-09-21 09:42:11 ] Completed train batch 0 generator loss: 1.203 65.874 ms, 3.20 s total +[ 2023-09-21 09:42:12 ] Completed train batch 0 generator backward 392.938 ms, 3.59 s total +[ 2023-09-21 09:42:12 ] Completed train batch 0 discriminator loss 0.057 56.638 ms, 3.65 s total +[ 2023-09-21 09:42:12 ] Completed train batch 0 discriminator backward 106.567 ms, 3.75 s total +Epoch [120] Step [0/2], gen_loss: 0.414, disc_loss: 0.019 +[ 2023-09-21 09:42:12 ] Completed train batch 0 metrics update 9.789 ms, 3.76 s total +Saving checkpoint at epoch 120 train batch 0 +[ 2023-09-21 09:42:12 ] Completed saving temp checkpoint 450.921 ms, 4.21 s total +[ 2023-09-21 09:42:12 ] Completed replacing temp checkpoint with checkpoint 26.182 ms, 4.24 s total +[ 2023-09-21 09:42:14 ] Completed train batch 1 to device 1,776.811 ms, 6.02 s total +[ 2023-09-21 09:42:14 ] Completed train batch 1 generator forward 27.238 ms, 6.04 s total +[ 2023-09-21 09:42:14 ] Completed train batch 1 generator loss: 1.244 63.167 ms, 6.11 s total +[ 2023-09-21 09:42:15 ] Completed train batch 1 generator backward 882.468 ms, 6.99 s total +[ 2023-09-21 09:42:15 ] Completed train batch 1 discriminator loss 0.057 55.840 ms, 7.05 s total +[ 2023-09-21 09:42:15 ] Completed train batch 1 discriminator backward 90.307 ms, 7.14 s total +Epoch [120] Step [1/2], gen_loss: 0.417, disc_loss: 0.019 +[ 2023-09-21 09:42:15 ] Completed train batch 1 metrics update 6.269 ms, 7.14 s total +Saving checkpoint at epoch 120 train batch 1 +[ 2023-09-21 09:42:16 ] Completed saving temp checkpoint 431.863 ms, 7.57 s total +[ 2023-09-21 09:42:16 ] Completed replacing temp checkpoint with checkpoint 28.227 ms, 7.60 s total +Epoch [120] :: gen_loss: 0.415, disc_loss: 0.019 +[ 2023-09-21 09:42:16 ] Completed training generator for epoch 120 52.915 ms, 7.65 s total +[ 2023-09-21 09:42:16 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 120 from eval step 0 + +[ 2023-09-21 09:42:17 ] Completed eval batch 0 to device 1,102.010 ms, 1.10 s total +[ 2023-09-21 09:42:17 ] Completed eval batch 0 forward 43.501 ms, 1.15 s total +[ 2023-09-21 09:42:17 ] Completed eval batch 0 recons_loss 0.296 ms, 1.15 s total +[ 2023-09-21 09:42:17 ] Completed eval batch 0 metrics update 0.611 ms, 1.15 s total +Saving checkpoint at epoch 120 val batch 0 +[ 2023-09-21 09:42:17 ] Completed saving temp checkpoint 444.201 ms, 1.59 s total +[ 2023-09-21 09:42:17 ] Completed replacing temp checkpoint with checkpoint 32.815 ms, 1.62 s total +[ 2023-09-21 09:42:18 ] Completed eval batch 1 to device 345.716 ms, 1.97 s total +[ 2023-09-21 09:42:18 ] Completed eval batch 1 forward 39.821 ms, 2.01 s total +[ 2023-09-21 09:42:18 ] Completed eval batch 1 recons_loss 0.553 ms, 2.01 s total +[ 2023-09-21 09:42:18 ] Completed eval batch 1 metrics update 190.525 ms, 2.20 s total +Saving checkpoint at epoch 120 val batch 1 +Epoch 120 val loss: 0.0640 +[ 2023-09-21 09:42:19 ] Completed saving temp checkpoint 448.037 ms, 2.65 s total +[ 2023-09-21 09:42:19 ] Completed replacing temp checkpoint with checkpoint 15.329 ms, 2.66 s total +[ 2023-09-21 09:42:19 ] Completed evaluating generator for epoch 120 49.480 ms, 2.71 s total + + +EPOCH :: 121 + + +[ 2023-09-21 09:42:19 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 121 from training step 0 + +[ 2023-09-21 09:42:22 ] Completed train batch 0 to device 3,104.702 ms, 3.10 s total +[ 2023-09-21 09:42:22 ] Completed train batch 0 generator forward 44.872 ms, 3.15 s total +[ 2023-09-21 09:42:22 ] Completed train batch 0 generator loss: 1.279 66.384 ms, 3.22 s total +[ 2023-09-21 09:42:22 ] Completed train batch 0 generator backward 397.272 ms, 3.61 s total +[ 2023-09-21 09:42:22 ] Completed train batch 0 discriminator loss 0.055 55.774 ms, 3.67 s total +[ 2023-09-21 09:42:22 ] Completed train batch 0 discriminator backward 103.723 ms, 3.77 s total +Epoch [121] Step [0/2], gen_loss: 0.416, disc_loss: 0.019 +[ 2023-09-21 09:42:22 ] Completed train batch 0 metrics update 16.299 ms, 3.79 s total +Saving checkpoint at epoch 121 train batch 0 +[ 2023-09-21 09:42:23 ] Completed saving temp checkpoint 475.651 ms, 4.26 s total +[ 2023-09-21 09:42:23 ] Completed replacing temp checkpoint with checkpoint 35.972 ms, 4.30 s total +[ 2023-09-21 09:42:25 ] Completed train batch 1 to device 1,620.723 ms, 5.92 s total +[ 2023-09-21 09:42:25 ] Completed train batch 1 generator forward 22.269 ms, 5.94 s total +[ 2023-09-21 09:42:25 ] Completed train batch 1 generator loss: 1.257 64.389 ms, 6.01 s total +[ 2023-09-21 09:42:26 ] Completed train batch 1 generator backward 1,051.652 ms, 7.06 s total +[ 2023-09-21 09:42:26 ] Completed train batch 1 discriminator loss 0.056 55.965 ms, 7.12 s total +[ 2023-09-21 09:42:26 ] Completed train batch 1 discriminator backward 72.346 ms, 7.19 s total +Epoch [121] Step [1/2], gen_loss: 0.422, disc_loss: 0.018 +[ 2023-09-21 09:42:26 ] Completed train batch 1 metrics update 1.197 ms, 7.19 s total +Saving checkpoint at epoch 121 train batch 1 +[ 2023-09-21 09:42:26 ] Completed saving temp checkpoint 467.262 ms, 7.66 s total +[ 2023-09-21 09:42:26 ] Completed replacing temp checkpoint with checkpoint 22.260 ms, 7.68 s total +Epoch [121] :: gen_loss: 0.419, disc_loss: 0.019 +[ 2023-09-21 09:42:26 ] Completed training generator for epoch 121 55.781 ms, 7.73 s total +[ 2023-09-21 09:42:26 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 121 from eval step 0 + +[ 2023-09-21 09:42:27 ] Completed eval batch 0 to device 1,139.959 ms, 1.14 s total +[ 2023-09-21 09:42:27 ] Completed eval batch 0 forward 28.825 ms, 1.17 s total +[ 2023-09-21 09:42:27 ] Completed eval batch 0 recons_loss 0.316 ms, 1.17 s total +[ 2023-09-21 09:42:27 ] Completed eval batch 0 metrics update 0.647 ms, 1.17 s total +Saving checkpoint at epoch 121 val batch 0 +[ 2023-09-21 09:42:28 ] Completed saving temp checkpoint 442.430 ms, 1.61 s total +[ 2023-09-21 09:42:28 ] Completed replacing temp checkpoint with checkpoint 25.393 ms, 1.64 s total +[ 2023-09-21 09:42:28 ] Completed eval batch 1 to device 394.040 ms, 2.03 s total +[ 2023-09-21 09:42:28 ] Completed eval batch 1 forward 39.568 ms, 2.07 s total +[ 2023-09-21 09:42:28 ] Completed eval batch 1 recons_loss 0.511 ms, 2.07 s total +[ 2023-09-21 09:42:28 ] Completed eval batch 1 metrics update 52.854 ms, 2.12 s total +Saving checkpoint at epoch 121 val batch 1 +Epoch 121 val loss: 0.0662 +[ 2023-09-21 09:42:29 ] Completed saving temp checkpoint 471.384 ms, 2.60 s total +[ 2023-09-21 09:42:29 ] Completed replacing temp checkpoint with checkpoint 22.705 ms, 2.62 s total +[ 2023-09-21 09:42:29 ] Completed evaluating generator for epoch 121 48.796 ms, 2.67 s total + + +EPOCH :: 122 + + +[ 2023-09-21 09:42:29 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 122 from training step 0 + +[ 2023-09-21 09:42:32 ] Completed train batch 0 to device 3,009.339 ms, 3.01 s total +[ 2023-09-21 09:42:32 ] Completed train batch 0 generator forward 35.160 ms, 3.04 s total +[ 2023-09-21 09:42:32 ] Completed train batch 0 generator loss: 1.214 66.214 ms, 3.11 s total +[ 2023-09-21 09:42:33 ] Completed train batch 0 generator backward 515.705 ms, 3.63 s total +[ 2023-09-21 09:42:33 ] Completed train batch 0 discriminator loss 0.056 57.198 ms, 3.68 s total +[ 2023-09-21 09:42:33 ] Completed train batch 0 discriminator backward 94.805 ms, 3.78 s total +Epoch [122] Step [0/2], gen_loss: 0.411, disc_loss: 0.018 +[ 2023-09-21 09:42:33 ] Completed train batch 0 metrics update 14.376 ms, 3.79 s total +Saving checkpoint at epoch 122 train batch 0 +[ 2023-09-21 09:42:33 ] Completed saving temp checkpoint 411.546 ms, 4.20 s total +[ 2023-09-21 09:42:33 ] Completed replacing temp checkpoint with checkpoint 24.044 ms, 4.23 s total +[ 2023-09-21 09:42:35 ] Completed train batch 1 to device 1,576.178 ms, 5.80 s total +[ 2023-09-21 09:42:35 ] Completed train batch 1 generator forward 23.123 ms, 5.83 s total +[ 2023-09-21 09:42:35 ] Completed train batch 1 generator loss: 1.216 64.955 ms, 5.89 s total +[ 2023-09-21 09:42:36 ] Completed train batch 1 generator backward 1,104.728 ms, 7.00 s total +[ 2023-09-21 09:42:36 ] Completed train batch 1 discriminator loss 0.055 56.585 ms, 7.05 s total +[ 2023-09-21 09:42:36 ] Completed train batch 1 discriminator backward 88.883 ms, 7.14 s total +Epoch [122] Step [1/2], gen_loss: 0.421, disc_loss: 0.018 +[ 2023-09-21 09:42:36 ] Completed train batch 1 metrics update 5.167 ms, 7.15 s total +Saving checkpoint at epoch 122 train batch 1 +[ 2023-09-21 09:42:36 ] Completed saving temp checkpoint 356.218 ms, 7.50 s total +[ 2023-09-21 09:42:37 ] Completed replacing temp checkpoint with checkpoint 18.578 ms, 7.52 s total +Epoch [122] :: gen_loss: 0.416, disc_loss: 0.018 +[ 2023-09-21 09:42:37 ] Completed training generator for epoch 122 55.207 ms, 7.58 s total +[ 2023-09-21 09:42:37 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 122 from eval step 0 + +[ 2023-09-21 09:42:38 ] Completed eval batch 0 to device 1,175.167 ms, 1.18 s total +[ 2023-09-21 09:42:38 ] Completed eval batch 0 forward 25.958 ms, 1.20 s total +[ 2023-09-21 09:42:38 ] Completed eval batch 0 recons_loss 0.225 ms, 1.20 s total +[ 2023-09-21 09:42:38 ] Completed eval batch 0 metrics update 0.489 ms, 1.20 s total +Saving checkpoint at epoch 122 val batch 0 +[ 2023-09-21 09:42:38 ] Completed saving temp checkpoint 348.221 ms, 1.55 s total +[ 2023-09-21 09:42:38 ] Completed replacing temp checkpoint with checkpoint 21.901 ms, 1.57 s total +[ 2023-09-21 09:42:39 ] Completed eval batch 1 to device 792.135 ms, 2.36 s total +[ 2023-09-21 09:42:39 ] Completed eval batch 1 forward 21.678 ms, 2.39 s total +[ 2023-09-21 09:42:39 ] Completed eval batch 1 recons_loss 0.371 ms, 2.39 s total +[ 2023-09-21 09:42:39 ] Completed eval batch 1 metrics update 0.340 ms, 2.39 s total +Saving checkpoint at epoch 122 val batch 1 +Epoch 122 val loss: 0.0668 +[ 2023-09-21 09:42:39 ] Completed saving temp checkpoint 354.305 ms, 2.74 s total +[ 2023-09-21 09:42:39 ] Completed replacing temp checkpoint with checkpoint 17.679 ms, 2.76 s total +[ 2023-09-21 09:42:39 ] Completed evaluating generator for epoch 122 53.271 ms, 2.81 s total + + +EPOCH :: 123 + + +[ 2023-09-21 09:42:39 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 123 from training step 0 + +[ 2023-09-21 09:42:42 ] Completed train batch 0 to device 3,105.105 ms, 3.11 s total +[ 2023-09-21 09:42:43 ] Completed train batch 0 generator forward 25.168 ms, 3.13 s total +[ 2023-09-21 09:42:43 ] Completed train batch 0 generator loss: 1.235 64.349 ms, 3.19 s total +[ 2023-09-21 09:42:43 ] Completed train batch 0 generator backward 570.173 ms, 3.76 s total +[ 2023-09-21 09:42:43 ] Completed train batch 0 discriminator loss 0.054 58.643 ms, 3.82 s total +[ 2023-09-21 09:42:43 ] Completed train batch 0 discriminator backward 96.361 ms, 3.92 s total +Epoch [123] Step [0/2], gen_loss: 0.415, disc_loss: 0.018 +[ 2023-09-21 09:42:43 ] Completed train batch 0 metrics update 3.335 ms, 3.92 s total +Saving checkpoint at epoch 123 train batch 0 +[ 2023-09-21 09:42:44 ] Completed saving temp checkpoint 420.114 ms, 4.34 s total +[ 2023-09-21 09:42:44 ] Completed replacing temp checkpoint with checkpoint 33.943 ms, 4.38 s total +[ 2023-09-21 09:42:45 ] Completed train batch 1 to device 1,460.592 ms, 5.84 s total +[ 2023-09-21 09:42:45 ] Completed train batch 1 generator forward 23.228 ms, 5.86 s total +[ 2023-09-21 09:42:45 ] Completed train batch 1 generator loss: 1.273 65.242 ms, 5.93 s total +[ 2023-09-21 09:42:47 ] Completed train batch 1 generator backward 1,222.177 ms, 7.15 s total +[ 2023-09-21 09:42:47 ] Completed train batch 1 discriminator loss 0.056 55.688 ms, 7.20 s total +[ 2023-09-21 09:42:47 ] Completed train batch 1 discriminator backward 88.689 ms, 7.29 s total +Epoch [123] Step [1/2], gen_loss: 0.417, disc_loss: 0.018 +[ 2023-09-21 09:42:47 ] Completed train batch 1 metrics update 1.555 ms, 7.29 s total +Saving checkpoint at epoch 123 train batch 1 +[ 2023-09-21 09:42:47 ] Completed saving temp checkpoint 448.740 ms, 7.74 s total +[ 2023-09-21 09:42:47 ] Completed replacing temp checkpoint with checkpoint 20.248 ms, 7.76 s total +Epoch [123] :: gen_loss: 0.416, disc_loss: 0.018 +[ 2023-09-21 09:42:47 ] Completed training generator for epoch 123 56.021 ms, 7.82 s total +[ 2023-09-21 09:42:47 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 123 from eval step 0 + +[ 2023-09-21 09:42:48 ] Completed eval batch 0 to device 1,131.250 ms, 1.13 s total +[ 2023-09-21 09:42:48 ] Completed eval batch 0 forward 27.024 ms, 1.16 s total +[ 2023-09-21 09:42:48 ] Completed eval batch 0 recons_loss 0.246 ms, 1.16 s total +[ 2023-09-21 09:42:48 ] Completed eval batch 0 metrics update 0.479 ms, 1.16 s total +Saving checkpoint at epoch 123 val batch 0 +[ 2023-09-21 09:42:49 ] Completed saving temp checkpoint 416.929 ms, 1.58 s total +[ 2023-09-21 09:42:49 ] Completed replacing temp checkpoint with checkpoint 27.148 ms, 1.60 s total +[ 2023-09-21 09:42:49 ] Completed eval batch 1 to device 449.488 ms, 2.05 s total +[ 2023-09-21 09:42:49 ] Completed eval batch 1 forward 37.178 ms, 2.09 s total +[ 2023-09-21 09:42:49 ] Completed eval batch 1 recons_loss 0.357 ms, 2.09 s total +[ 2023-09-21 09:42:49 ] Completed eval batch 1 metrics update 97.085 ms, 2.19 s total +Saving checkpoint at epoch 123 val batch 1 +Epoch 123 val loss: 0.0676 +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 09:55:05 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 09:55:05 ] Completed importing Timer 0.023 ms, 0.00 s total +[ 2023-09-21 09:55:10 ] Completed importing everything else 4,731.080 ms, 4.73 s total +| distributed init (rank 4): env:// +| distributed init (rank 5): env:// +| distributed init (rank 1): env:// +| distributed init (rank 0): env:// +| distributed init (rank 3): env:// +| distributed init (rank 2): env:// +[ 2023-09-21 09:55:18 ] Completed preliminaries 7,863.634 ms, 12.59 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 09:55:18 ] Completed build datasets 11.538 ms, 12.61 s total +[ 2023-09-21 09:55:18 ] Completed build samplers 0.045 ms, 12.61 s total +[ 2023-09-21 09:55:18 ] Completed build dataloaders 0.135 ms, 12.61 s total +[ 2023-09-21 09:55:18 ] Completed generator to device 94.229 ms, 12.70 s total +[ 2023-09-21 09:55:18 ] Completed discriminator to device 7.300 ms, 12.71 s total +[ 2023-09-21 09:55:18 ] Completed loss functions 393.820 ms, 13.10 s total +[ 2023-09-21 09:55:18 ] Completed models prepped for distribution 64.785 ms, 13.17 s total +[ 2023-09-21 09:55:18 ] Completed optimizers 0.671 ms, 13.17 s total +[ 2023-09-21 09:55:18 ] Completed grad scalers 0.028 ms, 13.17 s total +[ 2023-09-21 09:55:19 ] Completed checkpoint retrieval 486.603 ms, 13.65 s total + + +EPOCH :: 123 + + +[ 2023-09-21 09:55:19 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 123 from training step 2 + +Epoch [123] :: gen_loss: 0.416, disc_loss: 0.018 +[ 2023-09-21 09:55:19 ] Completed training generator for epoch 123 124.304 ms, 0.12 s total +[ 2023-09-21 09:55:19 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 123 from eval step 1 + +[ 2023-09-21 09:55:20 ] Completed eval batch 1 to device 1,350.797 ms, 1.35 s total +[ 2023-09-21 09:55:22 ] Completed eval batch 1 forward 1,128.760 ms, 2.48 s total +[ 2023-09-21 09:55:22 ] Completed eval batch 1 recons_loss 0.282 ms, 2.48 s total +[ 2023-09-21 09:55:22 ] Completed eval batch 1 metrics update 347.300 ms, 2.83 s total +Saving checkpoint at epoch 123 val batch 1 +Epoch 123 val loss: 0.0662 +[ 2023-09-21 09:55:22 ] Completed saving temp checkpoint 444.682 ms, 3.27 s total +[ 2023-09-21 09:55:22 ] Completed replacing temp checkpoint with checkpoint 138.575 ms, 3.41 s total +[ 2023-09-21 09:55:22 ] Completed evaluating generator for epoch 123 39.736 ms, 3.45 s total + + +EPOCH :: 124 + + +[ 2023-09-21 09:55:22 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 124 from training step 0 + +[ 2023-09-21 09:55:26 ] Completed train batch 0 to device 3,034.987 ms, 3.03 s total +[ 2023-09-21 09:55:26 ] Completed train batch 0 generator forward 43.988 ms, 3.08 s total +[ 2023-09-21 09:55:26 ] Completed train batch 0 generator loss: 1.308 79.120 ms, 3.16 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 09:55:26 ] Completed train batch 0 generator backward 645.645 ms, 3.80 s total +[ 2023-09-21 09:55:26 ] Completed train batch 0 discriminator loss 0.055 57.439 ms, 3.86 s total +[ 2023-09-21 09:55:26 ] Completed train batch 0 discriminator backward 105.543 ms, 3.97 s total +Epoch [124] Step [0/2], gen_loss: 0.416, disc_loss: 0.019 +[ 2023-09-21 09:55:26 ] Completed train batch 0 metrics update 8.310 ms, 3.98 s total +Saving checkpoint at epoch 124 train batch 0 +[ 2023-09-21 09:55:27 ] Completed saving temp checkpoint 445.434 ms, 4.42 s total +[ 2023-09-21 09:55:27 ] Completed replacing temp checkpoint with checkpoint 31.335 ms, 4.45 s total +[ 2023-09-21 09:55:28 ] Completed train batch 1 to device 1,367.709 ms, 5.82 s total +[ 2023-09-21 09:55:28 ] Completed train batch 1 generator forward 40.946 ms, 5.86 s total +[ 2023-09-21 09:55:28 ] Completed train batch 1 generator loss: 1.211 63.407 ms, 5.92 s total +[ 2023-09-21 09:55:30 ] Completed train batch 1 generator backward 1,407.246 ms, 7.33 s total +[ 2023-09-21 09:55:30 ] Completed train batch 1 discriminator loss 0.056 56.424 ms, 7.39 s total +[ 2023-09-21 09:55:30 ] Completed train batch 1 discriminator backward 73.683 ms, 7.46 s total +Epoch [124] Step [1/2], gen_loss: 0.418, disc_loss: 0.019 +[ 2023-09-21 09:55:30 ] Completed train batch 1 metrics update 0.795 ms, 7.46 s total +Saving checkpoint at epoch 124 train batch 1 +[ 2023-09-21 09:55:30 ] Completed saving temp checkpoint 435.254 ms, 7.90 s total +[ 2023-09-21 09:55:30 ] Completed replacing temp checkpoint with checkpoint 27.209 ms, 7.92 s total +Epoch [124] :: gen_loss: 0.417, disc_loss: 0.019 +[ 2023-09-21 09:55:30 ] Completed training generator for epoch 124 63.480 ms, 7.99 s total +[ 2023-09-21 09:55:30 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 124 from eval step 0 + +[ 2023-09-21 09:55:32 ] Completed eval batch 0 to device 1,196.444 ms, 1.20 s total +[ 2023-09-21 09:55:32 ] Completed eval batch 0 forward 25.119 ms, 1.22 s total +[ 2023-09-21 09:55:32 ] Completed eval batch 0 recons_loss 0.322 ms, 1.22 s total +[ 2023-09-21 09:55:32 ] Completed eval batch 0 metrics update 0.668 ms, 1.22 s total +Saving checkpoint at epoch 124 val batch 0 +[ 2023-09-21 09:55:32 ] Completed saving temp checkpoint 398.536 ms, 1.62 s total +[ 2023-09-21 09:55:32 ] Completed replacing temp checkpoint with checkpoint 29.409 ms, 1.65 s total +[ 2023-09-21 09:55:33 ] Completed eval batch 1 to device 441.231 ms, 2.09 s total +[ 2023-09-21 09:55:33 ] Completed eval batch 1 forward 28.997 ms, 2.12 s total +[ 2023-09-21 09:55:33 ] Completed eval batch 1 recons_loss 0.358 ms, 2.12 s total +[ 2023-09-21 09:55:33 ] Completed eval batch 1 metrics update 136.665 ms, 2.26 s total +Saving checkpoint at epoch 124 val batch 1 +Epoch 124 val loss: 0.0642 +[ 2023-09-21 09:55:33 ] Completed saving temp checkpoint 463.719 ms, 2.72 s total +[ 2023-09-21 09:55:33 ] Completed replacing temp checkpoint with checkpoint 22.223 ms, 2.74 s total +[ 2023-09-21 09:55:33 ] Completed evaluating generator for epoch 124 55.286 ms, 2.80 s total + + +EPOCH :: 125 + + +[ 2023-09-21 09:55:33 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 125 from training step 0 + +[ 2023-09-21 09:55:37 ] Completed train batch 0 to device 3,231.130 ms, 3.23 s total +[ 2023-09-21 09:55:37 ] Completed train batch 0 generator forward 27.254 ms, 3.26 s total +[ 2023-09-21 09:55:37 ] Completed train batch 0 generator loss: 1.222 66.817 ms, 3.33 s total +[ 2023-09-21 09:55:37 ] Completed train batch 0 generator backward 316.012 ms, 3.64 s total +[ 2023-09-21 09:55:37 ] Completed train batch 0 discriminator loss 0.053 79.072 ms, 3.72 s total +[ 2023-09-21 09:55:37 ] Completed train batch 0 discriminator backward 67.387 ms, 3.79 s total +Epoch [125] Step [0/2], gen_loss: 0.411, disc_loss: 0.018 +[ 2023-09-21 09:55:37 ] Completed train batch 0 metrics update 1.740 ms, 3.79 s total +Saving checkpoint at epoch 125 train batch 0 +[ 2023-09-21 09:55:37 ] Completed saving temp checkpoint 411.615 ms, 4.20 s total +[ 2023-09-21 09:55:38 ] Completed replacing temp checkpoint with checkpoint 23.300 ms, 4.22 s total +[ 2023-09-21 09:55:40 ] Completed train batch 1 to device 2,007.170 ms, 6.23 s total +[ 2023-09-21 09:55:40 ] Completed train batch 1 generator forward 41.872 ms, 6.27 s total +[ 2023-09-21 09:55:40 ] Completed train batch 1 generator loss: 1.224 64.070 ms, 6.34 s total +[ 2023-09-21 09:55:40 ] Completed train batch 1 generator backward 674.821 ms, 7.01 s total +[ 2023-09-21 09:55:40 ] Completed train batch 1 discriminator loss 0.054 56.633 ms, 7.07 s total +[ 2023-09-21 09:55:40 ] Completed train batch 1 discriminator backward 59.327 ms, 7.13 s total +Epoch [125] Step [1/2], gen_loss: 0.414, disc_loss: 0.018 +[ 2023-09-21 09:55:40 ] Completed train batch 1 metrics update 2.278 ms, 7.13 s total +Saving checkpoint at epoch 125 train batch 1 +[ 2023-09-21 09:55:41 ] Completed saving temp checkpoint 454.221 ms, 7.58 s total +[ 2023-09-21 09:55:41 ] Completed replacing temp checkpoint with checkpoint 20.753 ms, 7.61 s total +Epoch [125] :: gen_loss: 0.413, disc_loss: 0.018 +[ 2023-09-21 09:55:41 ] Completed training generator for epoch 125 49.884 ms, 7.66 s total +[ 2023-09-21 09:55:41 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 125 from eval step 0 + +[ 2023-09-21 09:55:42 ] Completed eval batch 0 to device 1,127.027 ms, 1.13 s total +[ 2023-09-21 09:55:42 ] Completed eval batch 0 forward 36.817 ms, 1.16 s total +[ 2023-09-21 09:55:42 ] Completed eval batch 0 recons_loss 0.308 ms, 1.16 s total +[ 2023-09-21 09:55:42 ] Completed eval batch 0 metrics update 0.570 ms, 1.16 s total +Saving checkpoint at epoch 125 val batch 0 +[ 2023-09-21 09:55:43 ] Completed saving temp checkpoint 416.620 ms, 1.58 s total +[ 2023-09-21 09:55:43 ] Completed replacing temp checkpoint with checkpoint 35.542 ms, 1.62 s total +[ 2023-09-21 09:55:43 ] Completed eval batch 1 to device 478.553 ms, 2.10 s total +[ 2023-09-21 09:55:43 ] Completed eval batch 1 forward 39.170 ms, 2.13 s total +[ 2023-09-21 09:55:43 ] Completed eval batch 1 recons_loss 0.336 ms, 2.13 s total +[ 2023-09-21 09:55:43 ] Completed eval batch 1 metrics update 119.637 ms, 2.25 s total +Saving checkpoint at epoch 125 val batch 1 +Epoch 125 val loss: 0.0647 +[ 2023-09-21 09:55:44 ] Completed saving temp checkpoint 436.897 ms, 2.69 s total +[ 2023-09-21 09:55:44 ] Completed replacing temp checkpoint with checkpoint 23.969 ms, 2.72 s total +[ 2023-09-21 09:55:44 ] Completed evaluating generator for epoch 125 48.144 ms, 2.76 s total + + +EPOCH :: 126 + + +[ 2023-09-21 09:55:44 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 126 from training step 0 + +[ 2023-09-21 09:55:47 ] Completed train batch 0 to device 3,149.913 ms, 3.15 s total +[ 2023-09-21 09:55:47 ] Completed train batch 0 generator forward 44.430 ms, 3.19 s total +[ 2023-09-21 09:55:47 ] Completed train batch 0 generator loss: 1.265 66.538 ms, 3.26 s total +[ 2023-09-21 09:55:48 ] Completed train batch 0 generator backward 605.451 ms, 3.87 s total +[ 2023-09-21 09:55:48 ] Completed train batch 0 discriminator loss 0.054 59.241 ms, 3.93 s total +[ 2023-09-21 09:55:48 ] Completed train batch 0 discriminator backward 93.791 ms, 4.02 s total +Epoch [126] Step [0/2], gen_loss: 0.411, disc_loss: 0.018 +[ 2023-09-21 09:55:48 ] Completed train batch 0 metrics update 9.566 ms, 4.03 s total +Saving checkpoint at epoch 126 train batch 0 +[ 2023-09-21 09:55:48 ] Completed saving temp checkpoint 406.955 ms, 4.44 s total +[ 2023-09-21 09:55:48 ] Completed replacing temp checkpoint with checkpoint 23.871 ms, 4.46 s total +[ 2023-09-21 09:55:50 ] Completed train batch 1 to device 1,482.626 ms, 5.94 s total +[ 2023-09-21 09:55:50 ] Completed train batch 1 generator forward 41.397 ms, 5.98 s total +[ 2023-09-21 09:55:50 ] Completed train batch 1 generator loss: 1.230 64.364 ms, 6.05 s total +[ 2023-09-21 09:55:51 ] Completed train batch 1 generator backward 1,270.647 ms, 7.32 s total +[ 2023-09-21 09:55:51 ] Completed train batch 1 discriminator loss 0.052 56.334 ms, 7.38 s total +[ 2023-09-21 09:55:51 ] Completed train batch 1 discriminator backward 75.701 ms, 7.45 s total +Epoch [126] Step [1/2], gen_loss: 0.418, disc_loss: 0.017 +[ 2023-09-21 09:55:51 ] Completed train batch 1 metrics update 1.591 ms, 7.45 s total +Saving checkpoint at epoch 126 train batch 1 +[ 2023-09-21 09:55:52 ] Completed saving temp checkpoint 466.438 ms, 7.92 s total +[ 2023-09-21 09:55:52 ] Completed replacing temp checkpoint with checkpoint 16.924 ms, 7.94 s total +Epoch [126] :: gen_loss: 0.415, disc_loss: 0.018 +[ 2023-09-21 09:55:52 ] Completed training generator for epoch 126 48.697 ms, 7.98 s total +[ 2023-09-21 09:55:52 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 126 from eval step 0 + +[ 2023-09-21 09:55:53 ] Completed eval batch 0 to device 1,128.666 ms, 1.13 s total +[ 2023-09-21 09:55:53 ] Completed eval batch 0 forward 25.553 ms, 1.15 s total +[ 2023-09-21 09:55:53 ] Completed eval batch 0 recons_loss 0.293 ms, 1.15 s total +[ 2023-09-21 09:55:53 ] Completed eval batch 0 metrics update 0.608 ms, 1.16 s total +Saving checkpoint at epoch 126 val batch 0 +[ 2023-09-21 09:55:53 ] Completed saving temp checkpoint 450.706 ms, 1.61 s total +[ 2023-09-21 09:55:53 ] Completed replacing temp checkpoint with checkpoint 36.316 ms, 1.64 s total +[ 2023-09-21 09:55:54 ] Completed eval batch 1 to device 411.116 ms, 2.05 s total +[ 2023-09-21 09:55:54 ] Completed eval batch 1 forward 22.177 ms, 2.08 s total +[ 2023-09-21 09:55:54 ] Completed eval batch 1 recons_loss 0.352 ms, 2.08 s total +[ 2023-09-21 09:55:54 ] Completed eval batch 1 metrics update 217.471 ms, 2.29 s total +Saving checkpoint at epoch 126 val batch 1 +Epoch 126 val loss: 0.0652 +[ 2023-09-21 09:55:55 ] Completed saving temp checkpoint 534.973 ms, 2.83 s total +[ 2023-09-21 09:55:55 ] Completed replacing temp checkpoint with checkpoint 35.442 ms, 2.86 s total +[ 2023-09-21 09:55:55 ] Completed evaluating generator for epoch 126 57.375 ms, 2.92 s total + + +EPOCH :: 127 + + +[ 2023-09-21 09:55:55 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 127 from training step 0 + +[ 2023-09-21 09:55:58 ] Completed train batch 0 to device 3,197.533 ms, 3.20 s total +[ 2023-09-21 09:55:58 ] Completed train batch 0 generator forward 30.449 ms, 3.23 s total +[ 2023-09-21 09:55:58 ] Completed train batch 0 generator loss: 1.278 66.463 ms, 3.29 s total +[ 2023-09-21 09:55:58 ] Completed train batch 0 generator backward 267.470 ms, 3.56 s total +[ 2023-09-21 09:55:58 ] Completed train batch 0 discriminator loss 0.051 67.319 ms, 3.63 s total +[ 2023-09-21 09:55:58 ] Completed train batch 0 discriminator backward 84.362 ms, 3.71 s total +Epoch [127] Step [0/2], gen_loss: 0.416, disc_loss: 0.017 +[ 2023-09-21 09:55:58 ] Completed train batch 0 metrics update 3.903 ms, 3.72 s total +Saving checkpoint at epoch 127 train batch 0 +[ 2023-09-21 09:55:59 ] Completed saving temp checkpoint 413.844 ms, 4.13 s total +[ 2023-09-21 09:55:59 ] Completed replacing temp checkpoint with checkpoint 31.304 ms, 4.16 s total +[ 2023-09-21 09:56:01 ] Completed train batch 1 to device 1,798.400 ms, 5.96 s total +[ 2023-09-21 09:56:01 ] Completed train batch 1 generator forward 41.579 ms, 6.00 s total +[ 2023-09-21 09:56:01 ] Completed train batch 1 generator loss: 1.301 63.885 ms, 6.07 s total +[ 2023-09-21 09:56:01 ] Completed train batch 1 generator backward 809.457 ms, 6.88 s total +[ 2023-09-21 09:56:02 ] Completed train batch 1 discriminator loss 0.050 56.398 ms, 6.93 s total +[ 2023-09-21 09:56:02 ] Completed train batch 1 discriminator backward 89.802 ms, 7.02 s total +Epoch [127] Step [1/2], gen_loss: 0.423, disc_loss: 0.017 +[ 2023-09-21 09:56:02 ] Completed train batch 1 metrics update 2.067 ms, 7.02 s total +Saving checkpoint at epoch 127 train batch 1 +[ 2023-09-21 09:56:02 ] Completed saving temp checkpoint 425.376 ms, 7.45 s total +[ 2023-09-21 09:56:02 ] Completed replacing temp checkpoint with checkpoint 17.846 ms, 7.47 s total +Epoch [127] :: gen_loss: 0.420, disc_loss: 0.017 +[ 2023-09-21 09:56:02 ] Completed training generator for epoch 127 60.423 ms, 7.53 s total +[ 2023-09-21 09:56:02 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 127 from eval step 0 + +[ 2023-09-21 09:56:03 ] Completed eval batch 0 to device 1,131.378 ms, 1.13 s total +[ 2023-09-21 09:56:03 ] Completed eval batch 0 forward 25.707 ms, 1.16 s total +[ 2023-09-21 09:56:03 ] Completed eval batch 0 recons_loss 0.294 ms, 1.16 s total +[ 2023-09-21 09:56:03 ] Completed eval batch 0 metrics update 0.558 ms, 1.16 s total +Saving checkpoint at epoch 127 val batch 0 +[ 2023-09-21 09:56:04 ] Completed saving temp checkpoint 383.276 ms, 1.54 s total +[ 2023-09-21 09:56:04 ] Completed replacing temp checkpoint with checkpoint 30.145 ms, 1.57 s total +[ 2023-09-21 09:56:04 ] Completed eval batch 1 to device 484.643 ms, 2.06 s total +[ 2023-09-21 09:56:04 ] Completed eval batch 1 forward 21.708 ms, 2.08 s total +[ 2023-09-21 09:56:04 ] Completed eval batch 1 recons_loss 0.345 ms, 2.08 s total +[ 2023-09-21 09:56:04 ] Completed eval batch 1 metrics update 124.556 ms, 2.20 s total +Saving checkpoint at epoch 127 val batch 1 +Epoch 127 val loss: 0.0647 +[ 2023-09-21 09:56:05 ] Completed saving temp checkpoint 407.069 ms, 2.61 s total +[ 2023-09-21 09:56:05 ] Completed replacing temp checkpoint with checkpoint 17.925 ms, 2.63 s total +[ 2023-09-21 09:56:05 ] Completed evaluating generator for epoch 127 48.764 ms, 2.68 s total + + +EPOCH :: 128 + + +[ 2023-09-21 09:56:05 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 128 from training step 0 + +[ 2023-09-21 09:56:08 ] Completed train batch 0 to device 3,096.259 ms, 3.10 s total +[ 2023-09-21 09:56:08 ] Completed train batch 0 generator forward 43.857 ms, 3.14 s total +[ 2023-09-21 09:56:08 ] Completed train batch 0 generator loss: 1.321 66.214 ms, 3.21 s total +[ 2023-09-21 09:56:09 ] Completed train batch 0 generator backward 577.830 ms, 3.78 s total +[ 2023-09-21 09:56:09 ] Completed train batch 0 discriminator loss 0.051 56.866 ms, 3.84 s total +[ 2023-09-21 09:56:09 ] Completed train batch 0 discriminator backward 116.872 ms, 3.96 s total +Epoch [128] Step [0/2], gen_loss: 0.414, disc_loss: 0.017 +[ 2023-09-21 09:56:09 ] Completed train batch 0 metrics update 5.775 ms, 3.96 s total +Saving checkpoint at epoch 128 train batch 0 +[ 2023-09-21 09:56:09 ] Completed saving temp checkpoint 406.076 ms, 4.37 s total +[ 2023-09-21 09:56:09 ] Completed replacing temp checkpoint with checkpoint 20.639 ms, 4.39 s total +[ 2023-09-21 09:56:11 ] Completed train batch 1 to device 1,641.592 ms, 6.03 s total +[ 2023-09-21 09:56:11 ] Completed train batch 1 generator forward 42.914 ms, 6.07 s total +[ 2023-09-21 09:56:11 ] Completed train batch 1 generator loss: 1.322 65.607 ms, 6.14 s total +[ 2023-09-21 09:56:12 ] Completed train batch 1 generator backward 944.152 ms, 7.08 s total +[ 2023-09-21 09:56:12 ] Completed train batch 1 discriminator loss 0.050 57.775 ms, 7.14 s total +[ 2023-09-21 09:56:12 ] Completed train batch 1 discriminator backward 59.706 ms, 7.20 s total +Epoch [128] Step [1/2], gen_loss: 0.423, disc_loss: 0.017 +[ 2023-09-21 09:56:12 ] Completed train batch 1 metrics update 1.580 ms, 7.20 s total +Saving checkpoint at epoch 128 train batch 1 +[ 2023-09-21 09:56:13 ] Completed saving temp checkpoint 707.811 ms, 7.91 s total +[ 2023-09-21 09:56:13 ] Completed replacing temp checkpoint with checkpoint 40.229 ms, 7.95 s total +Epoch [128] :: gen_loss: 0.419, disc_loss: 0.017 +[ 2023-09-21 09:56:13 ] Completed training generator for epoch 128 58.350 ms, 8.01 s total +[ 2023-09-21 09:56:13 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 128 from eval step 0 + +[ 2023-09-21 09:56:14 ] Completed eval batch 0 to device 1,136.050 ms, 1.14 s total +[ 2023-09-21 09:56:14 ] Completed eval batch 0 forward 30.697 ms, 1.17 s total +[ 2023-09-21 09:56:14 ] Completed eval batch 0 recons_loss 0.322 ms, 1.17 s total +[ 2023-09-21 09:56:14 ] Completed eval batch 0 metrics update 0.609 ms, 1.17 s total +Saving checkpoint at epoch 128 val batch 0 +[ 2023-09-21 09:56:14 ] Completed saving temp checkpoint 411.503 ms, 1.58 s total +[ 2023-09-21 09:56:14 ] Completed replacing temp checkpoint with checkpoint 31.088 ms, 1.61 s total +[ 2023-09-21 09:56:15 ] Completed eval batch 1 to device 351.731 ms, 1.96 s total +[ 2023-09-21 09:56:15 ] Completed eval batch 1 forward 22.107 ms, 1.98 s total +[ 2023-09-21 09:56:15 ] Completed eval batch 1 recons_loss 0.385 ms, 1.98 s total +[ 2023-09-21 09:56:15 ] Completed eval batch 1 metrics update 0.327 ms, 1.98 s total +Saving checkpoint at epoch 128 val batch 1 +Epoch 128 val loss: 0.0612 +[ 2023-09-21 09:56:15 ] Completed saving temp checkpoint 423.335 ms, 2.41 s total +[ 2023-09-21 09:56:15 ] Completed replacing temp checkpoint with checkpoint 31.935 ms, 2.44 s total +[ 2023-09-21 09:56:15 ] Completed evaluating generator for epoch 128 56.841 ms, 2.50 s total + + +EPOCH :: 129 + + +[ 2023-09-21 09:56:15 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 129 from training step 0 + +[ 2023-09-21 09:56:18 ] Completed train batch 0 to device 3,099.898 ms, 3.10 s total +[ 2023-09-21 09:56:18 ] Completed train batch 0 generator forward 44.432 ms, 3.14 s total +[ 2023-09-21 09:56:19 ] Completed train batch 0 generator loss: 1.172 66.718 ms, 3.21 s total +[ 2023-09-21 09:56:19 ] Completed train batch 0 generator backward 590.672 ms, 3.80 s total +[ 2023-09-21 09:56:19 ] Completed train batch 0 discriminator loss 0.048 57.235 ms, 3.86 s total +[ 2023-09-21 09:56:19 ] Completed train batch 0 discriminator backward 81.033 ms, 3.94 s total +Epoch [129] Step [0/2], gen_loss: 0.411, disc_loss: 0.016 +[ 2023-09-21 09:56:19 ] Completed train batch 0 metrics update 22.213 ms, 3.96 s total +Saving checkpoint at epoch 129 train batch 0 +[ 2023-09-21 09:56:20 ] Completed saving temp checkpoint 426.501 ms, 4.39 s total +[ 2023-09-21 09:56:20 ] Completed replacing temp checkpoint with checkpoint 28.303 ms, 4.42 s total +[ 2023-09-21 09:56:21 ] Completed train batch 1 to device 1,675.900 ms, 6.09 s total +[ 2023-09-21 09:56:21 ] Completed train batch 1 generator forward 22.537 ms, 6.12 s total +[ 2023-09-21 09:56:21 ] Completed train batch 1 generator loss: 1.238 63.444 ms, 6.18 s total +[ 2023-09-21 09:56:23 ] Completed train batch 1 generator backward 1,096.348 ms, 7.28 s total +[ 2023-09-21 09:56:23 ] Completed train batch 1 discriminator loss 0.048 56.828 ms, 7.33 s total +[ 2023-09-21 09:56:23 ] Completed train batch 1 discriminator backward 77.028 ms, 7.41 s total +Epoch [129] Step [1/2], gen_loss: 0.417, disc_loss: 0.016 +[ 2023-09-21 09:56:23 ] Completed train batch 1 metrics update 4.991 ms, 7.41 s total +Saving checkpoint at epoch 129 train batch 1 +[ 2023-09-21 09:56:23 ] Completed saving temp checkpoint 428.368 ms, 7.84 s total +[ 2023-09-21 09:56:23 ] Completed replacing temp checkpoint with checkpoint 25.891 ms, 7.87 s total +Epoch [129] :: gen_loss: 0.414, disc_loss: 0.016 +[ 2023-09-21 09:56:23 ] Completed training generator for epoch 129 55.755 ms, 7.92 s total +[ 2023-09-21 09:56:23 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 129 from eval step 0 + +[ 2023-09-21 09:56:24 ] Completed eval batch 0 to device 1,168.545 ms, 1.17 s total +[ 2023-09-21 09:56:24 ] Completed eval batch 0 forward 25.004 ms, 1.19 s total +[ 2023-09-21 09:56:24 ] Completed eval batch 0 recons_loss 0.297 ms, 1.19 s total +[ 2023-09-21 09:56:24 ] Completed eval batch 0 metrics update 0.591 ms, 1.19 s total +Saving checkpoint at epoch 129 val batch 0 +[ 2023-09-21 09:56:25 ] Completed saving temp checkpoint 368.532 ms, 1.56 s total +[ 2023-09-21 09:56:25 ] Completed replacing temp checkpoint with checkpoint 26.831 ms, 1.59 s total +[ 2023-09-21 09:56:25 ] Completed eval batch 1 to device 482.123 ms, 2.07 s total +[ 2023-09-21 09:56:25 ] Completed eval batch 1 forward 39.958 ms, 2.11 s total +[ 2023-09-21 09:56:25 ] Completed eval batch 1 recons_loss 0.368 ms, 2.11 s total +[ 2023-09-21 09:56:25 ] Completed eval batch 1 metrics update 86.107 ms, 2.20 s total +Saving checkpoint at epoch 129 val batch 1 +Epoch 129 val loss: 0.0557 +[ 2023-09-21 09:56:26 ] Completed saving temp checkpoint 394.928 ms, 2.59 s total +[ 2023-09-21 09:56:26 ] Completed replacing temp checkpoint with checkpoint 31.003 ms, 2.62 s total +[ 2023-09-21 09:56:26 ] Completed evaluating generator for epoch 129 50.587 ms, 2.67 s total + + +EPOCH :: 130 + + +[ 2023-09-21 09:56:26 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 130 from training step 0 + +[ 2023-09-21 09:56:29 ] Completed train batch 0 to device 2,913.267 ms, 2.91 s total +[ 2023-09-21 09:56:29 ] Completed train batch 0 generator forward 44.340 ms, 2.96 s total +[ 2023-09-21 09:56:29 ] Completed train batch 0 generator loss: 1.217 66.110 ms, 3.02 s total +[ 2023-09-21 09:56:30 ] Completed train batch 0 generator backward 779.375 ms, 3.80 s total +[ 2023-09-21 09:56:30 ] Completed train batch 0 discriminator loss 0.049 58.615 ms, 3.86 s total +[ 2023-09-21 09:56:30 ] Completed train batch 0 discriminator backward 99.349 ms, 3.96 s total +Epoch [130] Step [0/2], gen_loss: 0.413, disc_loss: 0.016 +[ 2023-09-21 09:56:30 ] Completed train batch 0 metrics update 4.978 ms, 3.97 s total +Saving checkpoint at epoch 130 train batch 0 +[ 2023-09-21 09:56:30 ] Completed saving temp checkpoint 385.588 ms, 4.35 s total +[ 2023-09-21 09:56:30 ] Completed replacing temp checkpoint with checkpoint 29.772 ms, 4.38 s total +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 10:08:35 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 10:08:35 ] Completed importing Timer 0.023 ms, 0.00 s total +[ 2023-09-21 10:08:40 ] Completed importing everything else 4,672.730 ms, 4.67 s total +| distributed init (rank 0): env:// +| distributed init (rank 3): env:// +| distributed init (rank 1): env:// +| distributed init (rank 4): env:// +| distributed init (rank 2): env:// +| distributed init (rank 5): env:// +[ 2023-09-21 10:08:49 ] Completed preliminaries 8,675.519 ms, 13.35 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 10:08:49 ] Completed build datasets 12.574 ms, 13.36 s total +[ 2023-09-21 10:08:49 ] Completed build samplers 0.047 ms, 13.36 s total +[ 2023-09-21 10:08:49 ] Completed build dataloaders 0.131 ms, 13.36 s total +[ 2023-09-21 10:08:49 ] Completed generator to device 73.679 ms, 13.43 s total +[ 2023-09-21 10:08:49 ] Completed discriminator to device 7.225 ms, 13.44 s total +[ 2023-09-21 10:08:49 ] Completed loss functions 394.300 ms, 13.84 s total +[ 2023-09-21 10:08:49 ] Completed models prepped for distribution 73.495 ms, 13.91 s total +[ 2023-09-21 10:08:49 ] Completed optimizers 0.663 ms, 13.91 s total +[ 2023-09-21 10:08:49 ] Completed grad scalers 0.021 ms, 13.91 s total +[ 2023-09-21 10:08:50 ] Completed checkpoint retrieval 469.713 ms, 14.38 s total + + +EPOCH :: 130 + + +[ 2023-09-21 10:08:50 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 130 from training step 1 + +[ 2023-09-21 10:08:53 ] Completed train batch 1 to device 3,464.454 ms, 3.46 s total +[ 2023-09-21 10:08:54 ] Completed train batch 1 generator forward 1,098.096 ms, 4.56 s total +[ 2023-09-21 10:08:54 ] Completed train batch 1 generator loss: 1.225 72.154 ms, 4.63 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 10:08:55 ] Completed train batch 1 generator backward 888.534 ms, 5.52 s total +[ 2023-09-21 10:08:55 ] Completed train batch 1 discriminator loss 0.051 65.729 ms, 5.59 s total +[ 2023-09-21 10:08:55 ] Completed train batch 1 discriminator backward 71.616 ms, 5.66 s total +Epoch [130] Step [1/2], gen_loss: 0.418, disc_loss: 0.017 +[ 2023-09-21 10:08:55 ] Completed train batch 1 metrics update 4.658 ms, 5.67 s total +Saving checkpoint at epoch 130 train batch 1 +[ 2023-09-21 10:08:56 ] Completed saving temp checkpoint 413.657 ms, 6.08 s total +[ 2023-09-21 10:08:56 ] Completed replacing temp checkpoint with checkpoint 131.643 ms, 6.21 s total +Epoch [130] :: gen_loss: 0.415, disc_loss: 0.017 +[ 2023-09-21 10:08:56 ] Completed training generator for epoch 130 50.429 ms, 6.26 s total +[ 2023-09-21 10:08:56 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 130 from eval step 0 + +[ 2023-09-21 10:08:57 ] Completed eval batch 0 to device 1,092.074 ms, 1.09 s total +[ 2023-09-21 10:08:57 ] Completed eval batch 0 forward 24.941 ms, 1.12 s total +[ 2023-09-21 10:08:57 ] Completed eval batch 0 recons_loss 0.301 ms, 1.12 s total +[ 2023-09-21 10:08:57 ] Completed eval batch 0 metrics update 0.606 ms, 1.12 s total +Saving checkpoint at epoch 130 val batch 0 +[ 2023-09-21 10:08:57 ] Completed saving temp checkpoint 394.198 ms, 1.51 s total +[ 2023-09-21 10:08:57 ] Completed replacing temp checkpoint with checkpoint 18.653 ms, 1.53 s total +[ 2023-09-21 10:08:58 ] Completed eval batch 1 to device 411.682 ms, 1.94 s total +[ 2023-09-21 10:08:58 ] Completed eval batch 1 forward 39.716 ms, 1.98 s total +[ 2023-09-21 10:08:58 ] Completed eval batch 1 recons_loss 0.551 ms, 1.98 s total +[ 2023-09-21 10:08:58 ] Completed eval batch 1 metrics update 166.134 ms, 2.15 s total +Saving checkpoint at epoch 130 val batch 1 +Epoch 130 val loss: 0.0559 +[ 2023-09-21 10:08:58 ] Completed saving temp checkpoint 402.095 ms, 2.55 s total +[ 2023-09-21 10:08:58 ] Completed replacing temp checkpoint with checkpoint 25.718 ms, 2.58 s total +[ 2023-09-21 10:08:58 ] Completed evaluating generator for epoch 130 55.605 ms, 2.63 s total + + +EPOCH :: 131 + + +[ 2023-09-21 10:08:58 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 131 from training step 0 + +[ 2023-09-21 10:09:02 ] Completed train batch 0 to device 3,104.007 ms, 3.10 s total +[ 2023-09-21 10:09:02 ] Completed train batch 0 generator forward 41.964 ms, 3.15 s total +[ 2023-09-21 10:09:02 ] Completed train batch 0 generator loss: 1.259 66.184 ms, 3.21 s total +[ 2023-09-21 10:09:02 ] Completed train batch 0 generator backward 503.805 ms, 3.72 s total +[ 2023-09-21 10:09:02 ] Completed train batch 0 discriminator loss 0.054 57.413 ms, 3.77 s total +[ 2023-09-21 10:09:02 ] Completed train batch 0 discriminator backward 113.193 ms, 3.89 s total +Epoch [131] Step [0/2], gen_loss: 0.412, disc_loss: 0.017 +[ 2023-09-21 10:09:02 ] Completed train batch 0 metrics update 23.786 ms, 3.91 s total +Saving checkpoint at epoch 131 train batch 0 +[ 2023-09-21 10:09:03 ] Completed saving temp checkpoint 408.243 ms, 4.32 s total +[ 2023-09-21 10:09:03 ] Completed replacing temp checkpoint with checkpoint 28.782 ms, 4.35 s total +[ 2023-09-21 10:09:04 ] Completed train batch 1 to device 1,573.176 ms, 5.92 s total +[ 2023-09-21 10:09:04 ] Completed train batch 1 generator forward 24.096 ms, 5.94 s total +[ 2023-09-21 10:09:04 ] Completed train batch 1 generator loss: 1.283 63.137 ms, 6.01 s total +[ 2023-09-21 10:09:06 ] Completed train batch 1 generator backward 1,099.368 ms, 7.11 s total +[ 2023-09-21 10:09:06 ] Completed train batch 1 discriminator loss 0.052 56.600 ms, 7.16 s total +[ 2023-09-21 10:09:06 ] Completed train batch 1 discriminator backward 72.646 ms, 7.24 s total +Epoch [131] Step [1/2], gen_loss: 0.412, disc_loss: 0.017 +[ 2023-09-21 10:09:06 ] Completed train batch 1 metrics update 5.065 ms, 7.24 s total +Saving checkpoint at epoch 131 train batch 1 +[ 2023-09-21 10:09:06 ] Completed saving temp checkpoint 425.724 ms, 7.67 s total +[ 2023-09-21 10:09:06 ] Completed replacing temp checkpoint with checkpoint 28.199 ms, 7.70 s total +Epoch [131] :: gen_loss: 0.412, disc_loss: 0.017 +[ 2023-09-21 10:09:06 ] Completed training generator for epoch 131 53.083 ms, 7.75 s total +[ 2023-09-21 10:09:06 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 131 from eval step 0 + +[ 2023-09-21 10:09:07 ] Completed eval batch 0 to device 1,075.652 ms, 1.08 s total +[ 2023-09-21 10:09:07 ] Completed eval batch 0 forward 23.732 ms, 1.10 s total +[ 2023-09-21 10:09:07 ] Completed eval batch 0 recons_loss 0.226 ms, 1.10 s total +[ 2023-09-21 10:09:07 ] Completed eval batch 0 metrics update 0.496 ms, 1.10 s total +Saving checkpoint at epoch 131 val batch 0 +[ 2023-09-21 10:09:08 ] Completed saving temp checkpoint 378.312 ms, 1.48 s total +[ 2023-09-21 10:09:08 ] Completed replacing temp checkpoint with checkpoint 23.204 ms, 1.50 s total +[ 2023-09-21 10:09:08 ] Completed eval batch 1 to device 477.528 ms, 1.98 s total +[ 2023-09-21 10:09:08 ] Completed eval batch 1 forward 39.911 ms, 2.02 s total +[ 2023-09-21 10:09:08 ] Completed eval batch 1 recons_loss 0.554 ms, 2.02 s total +[ 2023-09-21 10:09:08 ] Completed eval batch 1 metrics update 117.433 ms, 2.14 s total +Saving checkpoint at epoch 131 val batch 1 +Epoch 131 val loss: 0.0595 +[ 2023-09-21 10:09:09 ] Completed saving temp checkpoint 486.625 ms, 2.62 s total +[ 2023-09-21 10:09:09 ] Completed replacing temp checkpoint with checkpoint 24.007 ms, 2.65 s total +[ 2023-09-21 10:09:09 ] Completed evaluating generator for epoch 131 64.320 ms, 2.71 s total + + +EPOCH :: 132 + + +[ 2023-09-21 10:09:09 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 132 from training step 0 + +[ 2023-09-21 10:09:12 ] Completed train batch 0 to device 2,969.579 ms, 2.97 s total +[ 2023-09-21 10:09:12 ] Completed train batch 0 generator forward 27.441 ms, 3.00 s total +[ 2023-09-21 10:09:12 ] Completed train batch 0 generator loss: 1.279 75.189 ms, 3.07 s total +[ 2023-09-21 10:09:13 ] Completed train batch 0 generator backward 605.966 ms, 3.68 s total +[ 2023-09-21 10:09:13 ] Completed train batch 0 discriminator loss 0.050 57.210 ms, 3.74 s total +[ 2023-09-21 10:09:13 ] Completed train batch 0 discriminator backward 106.258 ms, 3.84 s total +Epoch [132] Step [0/2], gen_loss: 0.416, disc_loss: 0.017 +[ 2023-09-21 10:09:13 ] Completed train batch 0 metrics update 5.468 ms, 3.85 s total +Saving checkpoint at epoch 132 train batch 0 +[ 2023-09-21 10:09:13 ] Completed saving temp checkpoint 443.045 ms, 4.29 s total +[ 2023-09-21 10:09:13 ] Completed replacing temp checkpoint with checkpoint 23.555 ms, 4.31 s total +[ 2023-09-21 10:09:15 ] Completed train batch 1 to device 1,442.636 ms, 5.76 s total +[ 2023-09-21 10:09:15 ] Completed train batch 1 generator forward 39.882 ms, 5.80 s total +[ 2023-09-21 10:09:15 ] Completed train batch 1 generator loss: 1.276 63.512 ms, 5.86 s total +[ 2023-09-21 10:09:16 ] Completed train batch 1 generator backward 1,409.417 ms, 7.27 s total +[ 2023-09-21 10:09:16 ] Completed train batch 1 discriminator loss 0.049 56.689 ms, 7.33 s total +[ 2023-09-21 10:09:16 ] Completed train batch 1 discriminator backward 74.761 ms, 7.40 s total +Epoch [132] Step [1/2], gen_loss: 0.415, disc_loss: 0.016 +[ 2023-09-21 10:09:16 ] Completed train batch 1 metrics update 2.243 ms, 7.40 s total +Saving checkpoint at epoch 132 train batch 1 +[ 2023-09-21 10:09:17 ] Completed saving temp checkpoint 453.911 ms, 7.86 s total +[ 2023-09-21 10:09:17 ] Completed replacing temp checkpoint with checkpoint 30.495 ms, 7.89 s total +Epoch [132] :: gen_loss: 0.415, disc_loss: 0.017 +[ 2023-09-21 10:09:17 ] Completed training generator for epoch 132 56.447 ms, 7.94 s total +[ 2023-09-21 10:09:17 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 132 from eval step 0 + +[ 2023-09-21 10:09:18 ] Completed eval batch 0 to device 1,131.609 ms, 1.13 s total +[ 2023-09-21 10:09:18 ] Completed eval batch 0 forward 42.609 ms, 1.17 s total +[ 2023-09-21 10:09:18 ] Completed eval batch 0 recons_loss 0.312 ms, 1.17 s total +[ 2023-09-21 10:09:18 ] Completed eval batch 0 metrics update 0.591 ms, 1.18 s total +Saving checkpoint at epoch 132 val batch 0 +[ 2023-09-21 10:09:18 ] Completed saving temp checkpoint 380.509 ms, 1.56 s total +[ 2023-09-21 10:09:18 ] Completed replacing temp checkpoint with checkpoint 29.447 ms, 1.59 s total +[ 2023-09-21 10:09:19 ] Completed eval batch 1 to device 447.357 ms, 2.03 s total +[ 2023-09-21 10:09:19 ] Completed eval batch 1 forward 40.024 ms, 2.07 s total +[ 2023-09-21 10:09:19 ] Completed eval batch 1 recons_loss 0.552 ms, 2.07 s total +[ 2023-09-21 10:09:19 ] Completed eval batch 1 metrics update 180.271 ms, 2.25 s total +Saving checkpoint at epoch 132 val batch 1 +Epoch 132 val loss: 0.0610 +[ 2023-09-21 10:09:20 ] Completed saving temp checkpoint 397.280 ms, 2.65 s total +[ 2023-09-21 10:09:20 ] Completed replacing temp checkpoint with checkpoint 20.420 ms, 2.67 s total +[ 2023-09-21 10:09:20 ] Completed evaluating generator for epoch 132 46.540 ms, 2.72 s total + + +EPOCH :: 133 + + +[ 2023-09-21 10:09:20 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 133 from training step 0 + +[ 2023-09-21 10:09:23 ] Completed train batch 0 to device 3,110.791 ms, 3.11 s total +[ 2023-09-21 10:09:23 ] Completed train batch 0 generator forward 42.951 ms, 3.15 s total +[ 2023-09-21 10:09:23 ] Completed train batch 0 generator loss: 1.276 64.889 ms, 3.22 s total +[ 2023-09-21 10:09:23 ] Completed train batch 0 generator backward 633.321 ms, 3.85 s total +[ 2023-09-21 10:09:24 ] Completed train batch 0 discriminator loss 0.048 56.410 ms, 3.91 s total +[ 2023-09-21 10:09:24 ] Completed train batch 0 discriminator backward 95.081 ms, 4.00 s total +Epoch [133] Step [0/2], gen_loss: 0.422, disc_loss: 0.016 +[ 2023-09-21 10:09:24 ] Completed train batch 0 metrics update 4.936 ms, 4.01 s total +Saving checkpoint at epoch 133 train batch 0 +[ 2023-09-21 10:09:24 ] Completed saving temp checkpoint 451.106 ms, 4.46 s total +[ 2023-09-21 10:09:24 ] Completed replacing temp checkpoint with checkpoint 19.597 ms, 4.48 s total +[ 2023-09-21 10:09:25 ] Completed train batch 1 to device 1,406.301 ms, 5.89 s total +[ 2023-09-21 10:09:26 ] Completed train batch 1 generator forward 22.485 ms, 5.91 s total +[ 2023-09-21 10:09:26 ] Completed train batch 1 generator loss: 1.256 63.816 ms, 5.97 s total +[ 2023-09-21 10:09:27 ] Completed train batch 1 generator backward 1,245.284 ms, 7.22 s total +[ 2023-09-21 10:09:27 ] Completed train batch 1 discriminator loss 0.049 56.724 ms, 7.27 s total +[ 2023-09-21 10:09:27 ] Completed train batch 1 discriminator backward 71.427 ms, 7.35 s total +Epoch [133] Step [1/2], gen_loss: 0.416, disc_loss: 0.016 +[ 2023-09-21 10:09:27 ] Completed train batch 1 metrics update 0.596 ms, 7.35 s total +Saving checkpoint at epoch 133 train batch 1 +[ 2023-09-21 10:09:27 ] Completed saving temp checkpoint 484.990 ms, 7.83 s total +[ 2023-09-21 10:09:27 ] Completed replacing temp checkpoint with checkpoint 17.338 ms, 7.85 s total +Epoch [133] :: gen_loss: 0.419, disc_loss: 0.016 +[ 2023-09-21 10:09:27 ] Completed training generator for epoch 133 52.161 ms, 7.90 s total +[ 2023-09-21 10:09:27 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 133 from eval step 0 + +[ 2023-09-21 10:09:29 ] Completed eval batch 0 to device 1,070.066 ms, 1.07 s total +[ 2023-09-21 10:09:29 ] Completed eval batch 0 forward 23.945 ms, 1.09 s total +[ 2023-09-21 10:09:29 ] Completed eval batch 0 recons_loss 0.226 ms, 1.09 s total +[ 2023-09-21 10:09:29 ] Completed eval batch 0 metrics update 0.514 ms, 1.09 s total +Saving checkpoint at epoch 133 val batch 0 +[ 2023-09-21 10:09:29 ] Completed saving temp checkpoint 418.723 ms, 1.51 s total +[ 2023-09-21 10:09:29 ] Completed replacing temp checkpoint with checkpoint 27.518 ms, 1.54 s total +[ 2023-09-21 10:09:29 ] Completed eval batch 1 to device 422.612 ms, 1.96 s total +[ 2023-09-21 10:09:29 ] Completed eval batch 1 forward 21.683 ms, 1.99 s total +[ 2023-09-21 10:09:29 ] Completed eval batch 1 recons_loss 0.347 ms, 1.99 s total +[ 2023-09-21 10:09:30 ] Completed eval batch 1 metrics update 256.914 ms, 2.24 s total +Saving checkpoint at epoch 133 val batch 1 +Epoch 133 val loss: 0.0566 +[ 2023-09-21 10:09:30 ] Completed saving temp checkpoint 417.784 ms, 2.66 s total +[ 2023-09-21 10:09:30 ] Completed replacing temp checkpoint with checkpoint 21.846 ms, 2.68 s total +[ 2023-09-21 10:09:30 ] Completed evaluating generator for epoch 133 46.285 ms, 2.73 s total + + +EPOCH :: 134 + + +[ 2023-09-21 10:09:30 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 134 from training step 0 + +[ 2023-09-21 10:09:33 ] Completed train batch 0 to device 2,803.099 ms, 2.80 s total +[ 2023-09-21 10:09:33 ] Completed train batch 0 generator forward 24.906 ms, 2.83 s total +[ 2023-09-21 10:09:33 ] Completed train batch 0 generator loss: 1.285 64.415 ms, 2.89 s total +[ 2023-09-21 10:09:34 ] Completed train batch 0 generator backward 913.025 ms, 3.81 s total +[ 2023-09-21 10:09:34 ] Completed train batch 0 discriminator loss 0.050 56.435 ms, 3.86 s total +[ 2023-09-21 10:09:34 ] Completed train batch 0 discriminator backward 109.698 ms, 3.97 s total +Epoch [134] Step [0/2], gen_loss: 0.415, disc_loss: 0.017 +[ 2023-09-21 10:09:34 ] Completed train batch 0 metrics update 8.834 ms, 3.98 s total +Saving checkpoint at epoch 134 train batch 0 +[ 2023-09-21 10:09:35 ] Completed saving temp checkpoint 442.805 ms, 4.42 s total +[ 2023-09-21 10:09:35 ] Completed replacing temp checkpoint with checkpoint 30.289 ms, 4.45 s total +[ 2023-09-21 10:09:36 ] Completed train batch 1 to device 1,018.144 ms, 5.47 s total +[ 2023-09-21 10:09:36 ] Completed train batch 1 generator forward 39.220 ms, 5.51 s total +[ 2023-09-21 10:09:36 ] Completed train batch 1 generator loss: 1.237 62.417 ms, 5.57 s total +[ 2023-09-21 10:09:37 ] Completed train batch 1 generator backward 1,646.945 ms, 7.22 s total +[ 2023-09-21 10:09:38 ] Completed train batch 1 discriminator loss 0.052 56.599 ms, 7.28 s total +[ 2023-09-21 10:09:38 ] Completed train batch 1 discriminator backward 72.885 ms, 7.35 s total +Epoch [134] Step [1/2], gen_loss: 0.414, disc_loss: 0.017 +[ 2023-09-21 10:09:38 ] Completed train batch 1 metrics update 0.664 ms, 7.35 s total +Saving checkpoint at epoch 134 train batch 1 +[ 2023-09-21 10:09:38 ] Completed saving temp checkpoint 516.944 ms, 7.87 s total +[ 2023-09-21 10:09:38 ] Completed replacing temp checkpoint with checkpoint 24.262 ms, 7.89 s total +Epoch [134] :: gen_loss: 0.414, disc_loss: 0.017 +[ 2023-09-21 10:09:38 ] Completed training generator for epoch 134 61.644 ms, 7.95 s total +[ 2023-09-21 10:09:38 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 134 from eval step 0 + +[ 2023-09-21 10:09:39 ] Completed eval batch 0 to device 1,055.533 ms, 1.06 s total +[ 2023-09-21 10:09:39 ] Completed eval batch 0 forward 25.016 ms, 1.08 s total +[ 2023-09-21 10:09:39 ] Completed eval batch 0 recons_loss 0.297 ms, 1.08 s total +[ 2023-09-21 10:09:39 ] Completed eval batch 0 metrics update 0.601 ms, 1.08 s total +Saving checkpoint at epoch 134 val batch 0 +[ 2023-09-21 10:09:40 ] Completed saving temp checkpoint 449.658 ms, 1.53 s total +[ 2023-09-21 10:09:40 ] Completed replacing temp checkpoint with checkpoint 36.129 ms, 1.57 s total +[ 2023-09-21 10:09:40 ] Completed eval batch 1 to device 333.659 ms, 1.90 s total +[ 2023-09-21 10:09:40 ] Completed eval batch 1 forward 22.269 ms, 1.92 s total +[ 2023-09-21 10:09:40 ] Completed eval batch 1 recons_loss 0.348 ms, 1.92 s total +[ 2023-09-21 10:09:40 ] Completed eval batch 1 metrics update 262.210 ms, 2.19 s total +Saving checkpoint at epoch 134 val batch 1 +Epoch 134 val loss: 0.0512 +[ 2023-09-21 10:09:41 ] Completed saving temp checkpoint 446.764 ms, 2.63 s total +[ 2023-09-21 10:09:41 ] Completed replacing temp checkpoint with checkpoint 26.619 ms, 2.66 s total +[ 2023-09-21 10:09:41 ] Completed evaluating generator for epoch 134 51.425 ms, 2.71 s total + + +EPOCH :: 135 + + +[ 2023-09-21 10:09:41 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 135 from training step 0 + +[ 2023-09-21 10:09:44 ] Completed train batch 0 to device 2,977.969 ms, 2.98 s total +[ 2023-09-21 10:09:44 ] Completed train batch 0 generator forward 34.116 ms, 3.01 s total +[ 2023-09-21 10:09:44 ] Completed train batch 0 generator loss: 1.282 76.628 ms, 3.09 s total +[ 2023-09-21 10:09:45 ] Completed train batch 0 generator backward 677.238 ms, 3.77 s total +[ 2023-09-21 10:09:45 ] Completed train batch 0 discriminator loss 0.053 57.183 ms, 3.82 s total +[ 2023-09-21 10:09:45 ] Completed train batch 0 discriminator backward 84.301 ms, 3.91 s total +Epoch [135] Step [0/2], gen_loss: 0.415, disc_loss: 0.017 +[ 2023-09-21 10:09:45 ] Completed train batch 0 metrics update 20.943 ms, 3.93 s total +Saving checkpoint at epoch 135 train batch 0 +[ 2023-09-21 10:09:45 ] Completed saving temp checkpoint 401.671 ms, 4.33 s total +[ 2023-09-21 10:09:45 ] Completed replacing temp checkpoint with checkpoint 20.985 ms, 4.35 s total +[ 2023-09-21 10:09:47 ] Completed train batch 1 to device 1,731.447 ms, 6.08 s total +[ 2023-09-21 10:09:47 ] Completed train batch 1 generator forward 40.359 ms, 6.12 s total +[ 2023-09-21 10:09:47 ] Completed train batch 1 generator loss: 1.133 63.374 ms, 6.19 s total +[ 2023-09-21 10:09:48 ] Completed train batch 1 generator backward 671.246 ms, 6.86 s total +[ 2023-09-21 10:09:48 ] Completed train batch 1 discriminator loss 0.050 56.583 ms, 6.91 s total +[ 2023-09-21 10:09:48 ] Completed train batch 1 discriminator backward 69.060 ms, 6.98 s total +Epoch [135] Step [1/2], gen_loss: 0.407, disc_loss: 0.017 +[ 2023-09-21 10:09:48 ] Completed train batch 1 metrics update 0.698 ms, 6.98 s total +Saving checkpoint at epoch 135 train batch 1 +[ 2023-09-21 10:09:48 ] Completed saving temp checkpoint 464.251 ms, 7.45 s total +[ 2023-09-21 10:09:48 ] Completed replacing temp checkpoint with checkpoint 28.430 ms, 7.48 s total +Epoch [135] :: gen_loss: 0.411, disc_loss: 0.017 +[ 2023-09-21 10:09:48 ] Completed training generator for epoch 135 59.232 ms, 7.54 s total +[ 2023-09-21 10:09:48 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 135 from eval step 0 + +[ 2023-09-21 10:09:49 ] Completed eval batch 0 to device 1,058.407 ms, 1.06 s total +[ 2023-09-21 10:09:50 ] Completed eval batch 0 forward 42.205 ms, 1.10 s total +[ 2023-09-21 10:09:50 ] Completed eval batch 0 recons_loss 0.304 ms, 1.10 s total +[ 2023-09-21 10:09:50 ] Completed eval batch 0 metrics update 0.562 ms, 1.10 s total +Saving checkpoint at epoch 135 val batch 0 +[ 2023-09-21 10:09:50 ] Completed saving temp checkpoint 415.183 ms, 1.52 s total +[ 2023-09-21 10:09:50 ] Completed replacing temp checkpoint with checkpoint 26.111 ms, 1.54 s total +[ 2023-09-21 10:09:50 ] Completed eval batch 1 to device 362.227 ms, 1.91 s total +[ 2023-09-21 10:09:50 ] Completed eval batch 1 forward 40.039 ms, 1.95 s total +[ 2023-09-21 10:09:50 ] Completed eval batch 1 recons_loss 0.570 ms, 1.95 s total +[ 2023-09-21 10:09:51 ] Completed eval batch 1 metrics update 284.821 ms, 2.23 s total +Saving checkpoint at epoch 135 val batch 1 +Epoch 135 val loss: 0.0554 +[ 2023-09-21 10:09:51 ] Completed saving temp checkpoint 439.152 ms, 2.67 s total +[ 2023-09-21 10:09:51 ] Completed replacing temp checkpoint with checkpoint 21.358 ms, 2.69 s total +[ 2023-09-21 10:09:51 ] Completed evaluating generator for epoch 135 51.862 ms, 2.74 s total + + +EPOCH :: 136 + + +[ 2023-09-21 10:09:51 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 136 from training step 0 + +[ 2023-09-21 10:09:54 ] Completed train batch 0 to device 2,975.918 ms, 2.98 s total +[ 2023-09-21 10:09:54 ] Completed train batch 0 generator forward 34.055 ms, 3.01 s total +[ 2023-09-21 10:09:54 ] Completed train batch 0 generator loss: 1.291 66.567 ms, 3.08 s total +[ 2023-09-21 10:09:55 ] Completed train batch 0 generator backward 653.738 ms, 3.73 s total +[ 2023-09-21 10:09:55 ] Completed train batch 0 discriminator loss 0.052 56.996 ms, 3.79 s total +[ 2023-09-21 10:09:55 ] Completed train batch 0 discriminator backward 106.783 ms, 3.89 s total +Epoch [136] Step [0/2], gen_loss: 0.408, disc_loss: 0.017 +[ 2023-09-21 10:09:55 ] Completed train batch 0 metrics update 3.372 ms, 3.90 s total +Saving checkpoint at epoch 136 train batch 0 +[ 2023-09-21 10:09:55 ] Completed saving temp checkpoint 346.642 ms, 4.24 s total +[ 2023-09-21 10:09:55 ] Completed replacing temp checkpoint with checkpoint 25.422 ms, 4.27 s total +[ 2023-09-21 10:09:57 ] Completed train batch 1 to device 1,439.920 ms, 5.71 s total +[ 2023-09-21 10:09:57 ] Completed train batch 1 generator forward 41.072 ms, 5.75 s total +[ 2023-09-21 10:09:57 ] Completed train batch 1 generator loss: 1.191 63.669 ms, 5.81 s total +[ 2023-09-21 10:09:58 ] Completed train batch 1 generator backward 1,082.039 ms, 6.90 s total +[ 2023-09-21 10:09:58 ] Completed train batch 1 discriminator loss 0.049 56.785 ms, 6.95 s total +[ 2023-09-21 10:09:58 ] Completed train batch 1 discriminator backward 68.972 ms, 7.02 s total +Epoch [136] Step [1/2], gen_loss: 0.409, disc_loss: 0.017 +[ 2023-09-21 10:09:58 ] Completed train batch 1 metrics update 5.333 ms, 7.03 s total +Saving checkpoint at epoch 136 train batch 1 +[ 2023-09-21 10:09:59 ] Completed saving temp checkpoint 397.639 ms, 7.42 s total +[ 2023-09-21 10:09:59 ] Completed replacing temp checkpoint with checkpoint 26.047 ms, 7.45 s total +Epoch [136] :: gen_loss: 0.409, disc_loss: 0.017 +[ 2023-09-21 10:09:59 ] Completed training generator for epoch 136 60.052 ms, 7.51 s total +[ 2023-09-21 10:09:59 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 136 from eval step 0 + +[ 2023-09-21 10:10:00 ] Completed eval batch 0 to device 1,158.785 ms, 1.16 s total +[ 2023-09-21 10:10:00 ] Completed eval batch 0 forward 42.824 ms, 1.20 s total +[ 2023-09-21 10:10:00 ] Completed eval batch 0 recons_loss 0.309 ms, 1.20 s total +[ 2023-09-21 10:10:00 ] Completed eval batch 0 metrics update 0.582 ms, 1.20 s total +Saving checkpoint at epoch 136 val batch 0 +[ 2023-09-21 10:10:00 ] Completed saving temp checkpoint 344.932 ms, 1.55 s total +[ 2023-09-21 10:10:00 ] Completed replacing temp checkpoint with checkpoint 35.329 ms, 1.58 s total +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 10:23:29 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 10:23:29 ] Completed importing Timer 0.023 ms, 0.00 s total +[ 2023-09-21 10:23:34 ] Completed importing everything else 4,515.867 ms, 4.52 s total +| distributed init (rank 0): env:// +| distributed init (rank 5): env:// +| distributed init (rank 1): env:// +| distributed init (rank 3): env:// +| distributed init (rank 2): env:// +| distributed init (rank 4): env:// +[ 2023-09-21 10:23:42 ] Completed preliminaries 7,829.495 ms, 12.35 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 10:23:42 ] Completed build datasets 12.511 ms, 12.36 s total +[ 2023-09-21 10:23:42 ] Completed build samplers 0.047 ms, 12.36 s total +[ 2023-09-21 10:23:42 ] Completed build dataloaders 0.146 ms, 12.36 s total +[ 2023-09-21 10:23:42 ] Completed generator to device 78.840 ms, 12.44 s total +[ 2023-09-21 10:23:42 ] Completed discriminator to device 6.999 ms, 12.44 s total +[ 2023-09-21 10:23:42 ] Completed loss functions 403.760 ms, 12.85 s total +[ 2023-09-21 10:23:42 ] Completed models prepped for distribution 91.855 ms, 12.94 s total +[ 2023-09-21 10:23:42 ] Completed optimizers 0.729 ms, 12.94 s total +[ 2023-09-21 10:23:42 ] Completed grad scalers 0.034 ms, 12.94 s total +[ 2023-09-21 10:23:43 ] Completed checkpoint retrieval 524.468 ms, 13.46 s total + + +EPOCH :: 136 + + +[ 2023-09-21 10:23:43 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 136 from training step 2 + +Epoch [136] :: gen_loss: 0.409, disc_loss: 0.017 +[ 2023-09-21 10:23:43 ] Completed training generator for epoch 136 142.907 ms, 0.14 s total +[ 2023-09-21 10:23:43 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 136 from eval step 1 + +[ 2023-09-21 10:23:45 ] Completed eval batch 1 to device 1,479.558 ms, 1.48 s total +[ 2023-09-21 10:23:46 ] Completed eval batch 1 forward 1,158.731 ms, 2.64 s total +[ 2023-09-21 10:23:46 ] Completed eval batch 1 recons_loss 0.311 ms, 2.64 s total +[ 2023-09-21 10:23:46 ] Completed eval batch 1 metrics update 229.434 ms, 2.87 s total +Saving checkpoint at epoch 136 val batch 1 +Epoch 136 val loss: 0.0584 +[ 2023-09-21 10:23:46 ] Completed saving temp checkpoint 461.046 ms, 3.33 s total +[ 2023-09-21 10:23:47 ] Completed replacing temp checkpoint with checkpoint 119.003 ms, 3.45 s total +[ 2023-09-21 10:23:47 ] Completed evaluating generator for epoch 136 48.346 ms, 3.50 s total + + +EPOCH :: 137 + + +[ 2023-09-21 10:23:47 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 137 from training step 0 + +[ 2023-09-21 10:23:50 ] Completed train batch 0 to device 3,163.110 ms, 3.16 s total +[ 2023-09-21 10:23:50 ] Completed train batch 0 generator forward 43.518 ms, 3.21 s total +[ 2023-09-21 10:23:50 ] Completed train batch 0 generator loss: 1.202 79.561 ms, 3.29 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 10:23:50 ] Completed train batch 0 generator backward 590.481 ms, 3.88 s total +[ 2023-09-21 10:23:50 ] Completed train batch 0 discriminator loss 0.058 58.248 ms, 3.93 s total +[ 2023-09-21 10:23:51 ] Completed train batch 0 discriminator backward 102.573 ms, 4.04 s total +Epoch [137] Step [0/2], gen_loss: 0.412, disc_loss: 0.019 +[ 2023-09-21 10:23:51 ] Completed train batch 0 metrics update 4.051 ms, 4.04 s total +Saving checkpoint at epoch 137 train batch 0 +[ 2023-09-21 10:23:51 ] Completed saving temp checkpoint 465.046 ms, 4.51 s total +[ 2023-09-21 10:23:51 ] Completed replacing temp checkpoint with checkpoint 24.310 ms, 4.53 s total +[ 2023-09-21 10:23:53 ] Completed train batch 1 to device 1,641.582 ms, 6.17 s total +[ 2023-09-21 10:23:53 ] Completed train batch 1 generator forward 28.458 ms, 6.20 s total +[ 2023-09-21 10:23:53 ] Completed train batch 1 generator loss: 1.235 63.335 ms, 6.26 s total +[ 2023-09-21 10:23:54 ] Completed train batch 1 generator backward 1,014.821 ms, 7.28 s total +[ 2023-09-21 10:23:54 ] Completed train batch 1 discriminator loss 0.079 56.528 ms, 7.34 s total +[ 2023-09-21 10:23:54 ] Completed train batch 1 discriminator backward 74.696 ms, 7.41 s total +Epoch [137] Step [1/2], gen_loss: 0.408, disc_loss: 0.026 +[ 2023-09-21 10:23:54 ] Completed train batch 1 metrics update 3.564 ms, 7.41 s total +Saving checkpoint at epoch 137 train batch 1 +[ 2023-09-21 10:23:55 ] Completed saving temp checkpoint 553.008 ms, 7.97 s total +[ 2023-09-21 10:23:55 ] Completed replacing temp checkpoint with checkpoint 24.387 ms, 7.99 s total +Epoch [137] :: gen_loss: 0.410, disc_loss: 0.023 +[ 2023-09-21 10:23:55 ] Completed training generator for epoch 137 68.990 ms, 8.06 s total +[ 2023-09-21 10:23:55 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 137 from eval step 0 + +[ 2023-09-21 10:23:56 ] Completed eval batch 0 to device 1,102.988 ms, 1.10 s total +[ 2023-09-21 10:23:56 ] Completed eval batch 0 forward 24.993 ms, 1.13 s total +[ 2023-09-21 10:23:56 ] Completed eval batch 0 recons_loss 0.314 ms, 1.13 s total +[ 2023-09-21 10:23:56 ] Completed eval batch 0 metrics update 0.581 ms, 1.13 s total +Saving checkpoint at epoch 137 val batch 0 +[ 2023-09-21 10:23:56 ] Completed saving temp checkpoint 673.598 ms, 1.80 s total +[ 2023-09-21 10:23:56 ] Completed replacing temp checkpoint with checkpoint 27.446 ms, 1.83 s total +[ 2023-09-21 10:23:57 ] Completed eval batch 1 to device 232.574 ms, 2.06 s total +[ 2023-09-21 10:23:57 ] Completed eval batch 1 forward 21.464 ms, 2.08 s total +[ 2023-09-21 10:23:57 ] Completed eval batch 1 recons_loss 0.358 ms, 2.08 s total +[ 2023-09-21 10:23:57 ] Completed eval batch 1 metrics update 48.838 ms, 2.13 s total +Saving checkpoint at epoch 137 val batch 1 +Epoch 137 val loss: 0.0627 +[ 2023-09-21 10:23:58 ] Completed saving temp checkpoint 762.302 ms, 2.90 s total +[ 2023-09-21 10:23:58 ] Completed replacing temp checkpoint with checkpoint 29.030 ms, 2.92 s total +[ 2023-09-21 10:23:58 ] Completed evaluating generator for epoch 137 48.991 ms, 2.97 s total + + +EPOCH :: 138 + + +[ 2023-09-21 10:23:58 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 138 from training step 0 + +[ 2023-09-21 10:24:01 ] Completed train batch 0 to device 3,036.774 ms, 3.04 s total +[ 2023-09-21 10:24:01 ] Completed train batch 0 generator forward 30.539 ms, 3.07 s total +[ 2023-09-21 10:24:01 ] Completed train batch 0 generator loss: 1.273 66.150 ms, 3.13 s total +[ 2023-09-21 10:24:01 ] Completed train batch 0 generator backward 407.581 ms, 3.54 s total +[ 2023-09-21 10:24:01 ] Completed train batch 0 discriminator loss 0.066 58.353 ms, 3.60 s total +[ 2023-09-21 10:24:01 ] Completed train batch 0 discriminator backward 107.181 ms, 3.71 s total +Epoch [138] Step [0/2], gen_loss: 0.406, disc_loss: 0.022 +[ 2023-09-21 10:24:01 ] Completed train batch 0 metrics update 23.927 ms, 3.73 s total +Saving checkpoint at epoch 138 train batch 0 +[ 2023-09-21 10:24:02 ] Completed saving temp checkpoint 482.197 ms, 4.21 s total +[ 2023-09-21 10:24:02 ] Completed replacing temp checkpoint with checkpoint 35.094 ms, 4.25 s total +[ 2023-09-21 10:24:04 ] Completed train batch 1 to device 1,960.523 ms, 6.21 s total +[ 2023-09-21 10:24:04 ] Completed train batch 1 generator forward 40.631 ms, 6.25 s total +[ 2023-09-21 10:24:04 ] Completed train batch 1 generator loss: 1.270 63.440 ms, 6.31 s total +[ 2023-09-21 10:24:05 ] Completed train batch 1 generator backward 922.469 ms, 7.23 s total +[ 2023-09-21 10:24:05 ] Completed train batch 1 discriminator loss 0.054 56.642 ms, 7.29 s total +[ 2023-09-21 10:24:05 ] Completed train batch 1 discriminator backward 78.110 ms, 7.37 s total +Epoch [138] Step [1/2], gen_loss: 0.414, disc_loss: 0.018 +[ 2023-09-21 10:24:05 ] Completed train batch 1 metrics update 1.203 ms, 7.37 s total +Saving checkpoint at epoch 138 train batch 1 +[ 2023-09-21 10:24:05 ] Completed saving temp checkpoint 498.521 ms, 7.87 s total +[ 2023-09-21 10:24:05 ] Completed replacing temp checkpoint with checkpoint 20.224 ms, 7.89 s total +Epoch [138] :: gen_loss: 0.410, disc_loss: 0.020 +[ 2023-09-21 10:24:06 ] Completed training generator for epoch 138 53.223 ms, 7.94 s total +[ 2023-09-21 10:24:06 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 138 from eval step 0 + +[ 2023-09-21 10:24:07 ] Completed eval batch 0 to device 1,143.013 ms, 1.14 s total +[ 2023-09-21 10:24:07 ] Completed eval batch 0 forward 25.307 ms, 1.17 s total +[ 2023-09-21 10:24:07 ] Completed eval batch 0 recons_loss 0.309 ms, 1.17 s total +[ 2023-09-21 10:24:07 ] Completed eval batch 0 metrics update 0.623 ms, 1.17 s total +Saving checkpoint at epoch 138 val batch 0 +[ 2023-09-21 10:24:07 ] Completed saving temp checkpoint 452.054 ms, 1.62 s total +[ 2023-09-21 10:24:07 ] Completed replacing temp checkpoint with checkpoint 23.590 ms, 1.64 s total +[ 2023-09-21 10:24:08 ] Completed eval batch 1 to device 404.125 ms, 2.05 s total +[ 2023-09-21 10:24:08 ] Completed eval batch 1 forward 40.113 ms, 2.09 s total +[ 2023-09-21 10:24:08 ] Completed eval batch 1 recons_loss 0.452 ms, 2.09 s total +[ 2023-09-21 10:24:08 ] Completed eval batch 1 metrics update 186.879 ms, 2.28 s total +Saving checkpoint at epoch 138 val batch 1 +Epoch 138 val loss: 0.0671 +[ 2023-09-21 10:24:08 ] Completed saving temp checkpoint 498.068 ms, 2.77 s total +[ 2023-09-21 10:24:08 ] Completed replacing temp checkpoint with checkpoint 23.522 ms, 2.80 s total +[ 2023-09-21 10:24:08 ] Completed evaluating generator for epoch 138 51.846 ms, 2.85 s total + + +EPOCH :: 139 + + +[ 2023-09-21 10:24:08 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 139 from training step 0 + +[ 2023-09-21 10:24:12 ] Completed train batch 0 to device 3,223.835 ms, 3.22 s total +[ 2023-09-21 10:24:12 ] Completed train batch 0 generator forward 35.526 ms, 3.26 s total +[ 2023-09-21 10:24:12 ] Completed train batch 0 generator loss: 1.204 65.554 ms, 3.32 s total +[ 2023-09-21 10:24:12 ] Completed train batch 0 generator backward 552.246 ms, 3.88 s total +[ 2023-09-21 10:24:12 ] Completed train batch 0 discriminator loss 0.052 62.744 ms, 3.94 s total +[ 2023-09-21 10:24:12 ] Completed train batch 0 discriminator backward 101.004 ms, 4.04 s total +Epoch [139] Step [0/2], gen_loss: 0.418, disc_loss: 0.018 +[ 2023-09-21 10:24:12 ] Completed train batch 0 metrics update 6.872 ms, 4.05 s total +Saving checkpoint at epoch 139 train batch 0 +[ 2023-09-21 10:24:13 ] Completed saving temp checkpoint 441.238 ms, 4.49 s total +[ 2023-09-21 10:24:13 ] Completed replacing temp checkpoint with checkpoint 23.390 ms, 4.51 s total +[ 2023-09-21 10:24:15 ] Completed train batch 1 to device 1,699.240 ms, 6.21 s total +[ 2023-09-21 10:24:15 ] Completed train batch 1 generator forward 39.389 ms, 6.25 s total +[ 2023-09-21 10:24:15 ] Completed train batch 1 generator loss: 1.225 65.172 ms, 6.32 s total +[ 2023-09-21 10:24:15 ] Completed train batch 1 generator backward 752.446 ms, 7.07 s total +[ 2023-09-21 10:24:16 ] Completed train batch 1 discriminator loss 0.051 57.517 ms, 7.13 s total +[ 2023-09-21 10:24:16 ] Completed train batch 1 discriminator backward 77.423 ms, 7.20 s total +Epoch [139] Step [1/2], gen_loss: 0.422, disc_loss: 0.017 +[ 2023-09-21 10:24:16 ] Completed train batch 1 metrics update 0.603 ms, 7.20 s total +Saving checkpoint at epoch 139 train batch 1 +[ 2023-09-21 10:24:16 ] Completed saving temp checkpoint 473.023 ms, 7.68 s total +[ 2023-09-21 10:24:16 ] Completed replacing temp checkpoint with checkpoint 23.186 ms, 7.70 s total +Epoch [139] :: gen_loss: 0.420, disc_loss: 0.017 +[ 2023-09-21 10:24:16 ] Completed training generator for epoch 139 53.885 ms, 7.75 s total +[ 2023-09-21 10:24:16 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 139 from eval step 0 + +[ 2023-09-21 10:24:17 ] Completed eval batch 0 to device 1,149.328 ms, 1.15 s total +[ 2023-09-21 10:24:17 ] Completed eval batch 0 forward 41.512 ms, 1.19 s total +[ 2023-09-21 10:24:17 ] Completed eval batch 0 recons_loss 0.359 ms, 1.19 s total +[ 2023-09-21 10:24:17 ] Completed eval batch 0 metrics update 0.694 ms, 1.19 s total +Saving checkpoint at epoch 139 val batch 0 +[ 2023-09-21 10:24:18 ] Completed saving temp checkpoint 434.954 ms, 1.63 s total +[ 2023-09-21 10:24:18 ] Completed replacing temp checkpoint with checkpoint 24.408 ms, 1.65 s total +[ 2023-09-21 10:24:18 ] Completed eval batch 1 to device 415.411 ms, 2.07 s total +[ 2023-09-21 10:24:18 ] Completed eval batch 1 forward 21.605 ms, 2.09 s total +[ 2023-09-21 10:24:18 ] Completed eval batch 1 recons_loss 0.367 ms, 2.09 s total +[ 2023-09-21 10:24:18 ] Completed eval batch 1 metrics update 193.887 ms, 2.28 s total +Saving checkpoint at epoch 139 val batch 1 +Epoch 139 val loss: 0.0599 +[ 2023-09-21 10:24:19 ] Completed saving temp checkpoint 476.679 ms, 2.76 s total +[ 2023-09-21 10:24:19 ] Completed replacing temp checkpoint with checkpoint 27.842 ms, 2.79 s total +[ 2023-09-21 10:24:19 ] Completed evaluating generator for epoch 139 49.559 ms, 2.84 s total + + +EPOCH :: 140 + + +[ 2023-09-21 10:24:19 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 140 from training step 0 + +[ 2023-09-21 10:24:22 ] Completed train batch 0 to device 2,988.764 ms, 2.99 s total +[ 2023-09-21 10:24:22 ] Completed train batch 0 generator forward 42.495 ms, 3.03 s total +[ 2023-09-21 10:24:22 ] Completed train batch 0 generator loss: 1.278 66.655 ms, 3.10 s total +[ 2023-09-21 10:24:23 ] Completed train batch 0 generator backward 788.418 ms, 3.89 s total +[ 2023-09-21 10:24:23 ] Completed train batch 0 discriminator loss 0.050 57.698 ms, 3.94 s total +[ 2023-09-21 10:24:23 ] Completed train batch 0 discriminator backward 106.500 ms, 4.05 s total +Epoch [140] Step [0/2], gen_loss: 0.413, disc_loss: 0.017 +[ 2023-09-21 10:24:23 ] Completed train batch 0 metrics update 6.077 ms, 4.06 s total +Saving checkpoint at epoch 140 train batch 0 +[ 2023-09-21 10:24:23 ] Completed saving temp checkpoint 459.635 ms, 4.52 s total +[ 2023-09-21 10:24:24 ] Completed replacing temp checkpoint with checkpoint 30.014 ms, 4.55 s total +[ 2023-09-21 10:24:25 ] Completed train batch 1 to device 1,199.730 ms, 5.75 s total +[ 2023-09-21 10:24:25 ] Completed train batch 1 generator forward 36.197 ms, 5.78 s total +[ 2023-09-21 10:24:25 ] Completed train batch 1 generator loss: 1.193 63.454 ms, 5.85 s total +[ 2023-09-21 10:24:27 ] Completed train batch 1 generator backward 1,697.378 ms, 7.54 s total +[ 2023-09-21 10:24:27 ] Completed train batch 1 discriminator loss 0.051 56.760 ms, 7.60 s total +[ 2023-09-21 10:24:27 ] Completed train batch 1 discriminator backward 79.412 ms, 7.68 s total +Epoch [140] Step [1/2], gen_loss: 0.411, disc_loss: 0.017 +[ 2023-09-21 10:24:27 ] Completed train batch 1 metrics update 5.405 ms, 7.68 s total +Saving checkpoint at epoch 140 train batch 1 +[ 2023-09-21 10:24:27 ] Completed saving temp checkpoint 446.228 ms, 8.13 s total +[ 2023-09-21 10:24:27 ] Completed replacing temp checkpoint with checkpoint 29.909 ms, 8.16 s total +Epoch [140] :: gen_loss: 0.412, disc_loss: 0.017 +[ 2023-09-21 10:24:27 ] Completed training generator for epoch 140 52.212 ms, 8.21 s total +[ 2023-09-21 10:24:27 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 140 from eval step 0 + +[ 2023-09-21 10:24:28 ] Completed eval batch 0 to device 1,170.899 ms, 1.17 s total +[ 2023-09-21 10:24:28 ] Completed eval batch 0 forward 43.275 ms, 1.21 s total +[ 2023-09-21 10:24:28 ] Completed eval batch 0 recons_loss 0.440 ms, 1.21 s total +[ 2023-09-21 10:24:28 ] Completed eval batch 0 metrics update 0.796 ms, 1.22 s total +Saving checkpoint at epoch 140 val batch 0 +[ 2023-09-21 10:24:29 ] Completed saving temp checkpoint 411.828 ms, 1.63 s total +[ 2023-09-21 10:24:29 ] Completed replacing temp checkpoint with checkpoint 29.904 ms, 1.66 s total +[ 2023-09-21 10:24:29 ] Completed eval batch 1 to device 423.048 ms, 2.08 s total +[ 2023-09-21 10:24:29 ] Completed eval batch 1 forward 39.963 ms, 2.12 s total +[ 2023-09-21 10:24:29 ] Completed eval batch 1 recons_loss 0.598 ms, 2.12 s total +[ 2023-09-21 10:24:30 ] Completed eval batch 1 metrics update 214.965 ms, 2.34 s total +Saving checkpoint at epoch 140 val batch 1 +Epoch 140 val loss: 0.0569 +[ 2023-09-21 10:24:30 ] Completed saving temp checkpoint 448.146 ms, 2.78 s total +[ 2023-09-21 10:24:30 ] Completed replacing temp checkpoint with checkpoint 26.566 ms, 2.81 s total +[ 2023-09-21 10:24:30 ] Completed evaluating generator for epoch 140 47.807 ms, 2.86 s total + + +EPOCH :: 141 + + +[ 2023-09-21 10:24:30 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 141 from training step 0 + +[ 2023-09-21 10:24:33 ] Completed train batch 0 to device 3,047.083 ms, 3.05 s total +[ 2023-09-21 10:24:33 ] Completed train batch 0 generator forward 27.354 ms, 3.07 s total +[ 2023-09-21 10:24:33 ] Completed train batch 0 generator loss: 1.209 66.364 ms, 3.14 s total +[ 2023-09-21 10:24:34 ] Completed train batch 0 generator backward 754.758 ms, 3.90 s total +[ 2023-09-21 10:24:34 ] Completed train batch 0 discriminator loss 0.051 59.444 ms, 3.96 s total +[ 2023-09-21 10:24:34 ] Completed train batch 0 discriminator backward 84.337 ms, 4.04 s total +Epoch [141] Step [0/2], gen_loss: 0.407, disc_loss: 0.017 +[ 2023-09-21 10:24:34 ] Completed train batch 0 metrics update 21.526 ms, 4.06 s total +Saving checkpoint at epoch 141 train batch 0 +[ 2023-09-21 10:24:35 ] Completed saving temp checkpoint 425.043 ms, 4.49 s total +[ 2023-09-21 10:24:35 ] Completed replacing temp checkpoint with checkpoint 25.907 ms, 4.51 s total +[ 2023-09-21 10:24:36 ] Completed train batch 1 to device 1,285.156 ms, 5.80 s total +[ 2023-09-21 10:24:36 ] Completed train batch 1 generator forward 41.056 ms, 5.84 s total +[ 2023-09-21 10:24:36 ] Completed train batch 1 generator loss: 1.219 62.446 ms, 5.90 s total +[ 2023-09-21 10:24:38 ] Completed train batch 1 generator backward 1,673.577 ms, 7.57 s total +[ 2023-09-21 10:24:38 ] Completed train batch 1 discriminator loss 0.050 56.573 ms, 7.63 s total +[ 2023-09-21 10:24:38 ] Completed train batch 1 discriminator backward 78.419 ms, 7.71 s total +Epoch [141] Step [1/2], gen_loss: 0.408, disc_loss: 0.017 +[ 2023-09-21 10:24:38 ] Completed train batch 1 metrics update 4.797 ms, 7.71 s total +Saving checkpoint at epoch 141 train batch 1 +[ 2023-09-21 10:24:38 ] Completed saving temp checkpoint 606.448 ms, 8.32 s total +[ 2023-09-21 10:24:38 ] Completed replacing temp checkpoint with checkpoint 27.935 ms, 8.35 s total +Epoch [141] :: gen_loss: 0.408, disc_loss: 0.017 +[ 2023-09-21 10:24:38 ] Completed training generator for epoch 141 49.213 ms, 8.40 s total +[ 2023-09-21 10:24:38 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 141 from eval step 0 + +[ 2023-09-21 10:24:39 ] Completed eval batch 0 to device 1,043.172 ms, 1.04 s total +[ 2023-09-21 10:24:40 ] Completed eval batch 0 forward 41.070 ms, 1.08 s total +[ 2023-09-21 10:24:40 ] Completed eval batch 0 recons_loss 0.338 ms, 1.08 s total +[ 2023-09-21 10:24:40 ] Completed eval batch 0 metrics update 0.599 ms, 1.09 s total +Saving checkpoint at epoch 141 val batch 0 +[ 2023-09-21 10:24:40 ] Completed saving temp checkpoint 507.136 ms, 1.59 s total +[ 2023-09-21 10:24:40 ] Completed replacing temp checkpoint with checkpoint 22.735 ms, 1.62 s total +[ 2023-09-21 10:24:40 ] Completed eval batch 1 to device 294.415 ms, 1.91 s total +[ 2023-09-21 10:24:40 ] Completed eval batch 1 forward 39.570 ms, 1.95 s total +[ 2023-09-21 10:24:40 ] Completed eval batch 1 recons_loss 0.371 ms, 1.95 s total +[ 2023-09-21 10:24:41 ] Completed eval batch 1 metrics update 267.569 ms, 2.22 s total +Saving checkpoint at epoch 141 val batch 1 +Epoch 141 val loss: 0.0621 +[ 2023-09-21 10:24:41 ] Completed saving temp checkpoint 551.478 ms, 2.77 s total +[ 2023-09-21 10:24:41 ] Completed replacing temp checkpoint with checkpoint 30.127 ms, 2.80 s total +[ 2023-09-21 10:24:41 ] Completed evaluating generator for epoch 141 63.638 ms, 2.86 s total + + +EPOCH :: 142 + + +[ 2023-09-21 10:24:41 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 142 from training step 0 + +[ 2023-09-21 10:24:44 ] Completed train batch 0 to device 3,074.391 ms, 3.07 s total +[ 2023-09-21 10:24:44 ] Completed train batch 0 generator forward 34.710 ms, 3.11 s total +[ 2023-09-21 10:24:44 ] Completed train batch 0 generator loss: 1.220 77.270 ms, 3.19 s total +[ 2023-09-21 10:24:45 ] Completed train batch 0 generator backward 553.616 ms, 3.74 s total +[ 2023-09-21 10:24:45 ] Completed train batch 0 discriminator loss 0.049 59.881 ms, 3.80 s total +[ 2023-09-21 10:24:45 ] Completed train batch 0 discriminator backward 116.821 ms, 3.92 s total +Epoch [142] Step [0/2], gen_loss: 0.413, disc_loss: 0.016 +[ 2023-09-21 10:24:45 ] Completed train batch 0 metrics update 5.405 ms, 3.92 s total +Saving checkpoint at epoch 142 train batch 0 +[ 2023-09-21 10:24:46 ] Completed saving temp checkpoint 542.465 ms, 4.46 s total +[ 2023-09-21 10:24:46 ] Completed replacing temp checkpoint with checkpoint 32.838 ms, 4.50 s total +[ 2023-09-21 10:24:48 ] Completed train batch 1 to device 1,873.873 ms, 6.37 s total +[ 2023-09-21 10:24:48 ] Completed train batch 1 generator forward 23.035 ms, 6.39 s total +[ 2023-09-21 10:24:48 ] Completed train batch 1 generator loss: 1.268 64.852 ms, 6.46 s total +[ 2023-09-21 10:24:48 ] Completed train batch 1 generator backward 736.655 ms, 7.20 s total +[ 2023-09-21 10:24:49 ] Completed train batch 1 discriminator loss 0.047 57.429 ms, 7.25 s total +[ 2023-09-21 10:24:49 ] Completed train batch 1 discriminator backward 73.512 ms, 7.33 s total +Epoch [142] Step [1/2], gen_loss: 0.413, disc_loss: 0.016 +[ 2023-09-21 10:24:49 ] Completed train batch 1 metrics update 0.481 ms, 7.33 s total +Saving checkpoint at epoch 142 train batch 1 +[ 2023-09-21 10:24:49 ] Completed saving temp checkpoint 537.766 ms, 7.86 s total +[ 2023-09-21 10:24:49 ] Completed replacing temp checkpoint with checkpoint 24.443 ms, 7.89 s total +Epoch [142] :: gen_loss: 0.413, disc_loss: 0.016 +[ 2023-09-21 10:24:49 ] Completed training generator for epoch 142 78.071 ms, 7.97 s total +[ 2023-09-21 10:24:49 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 142 from eval step 0 + +[ 2023-09-21 10:24:50 ] Completed eval batch 0 to device 1,111.343 ms, 1.11 s total +[ 2023-09-21 10:24:50 ] Completed eval batch 0 forward 25.370 ms, 1.14 s total +[ 2023-09-21 10:24:50 ] Completed eval batch 0 recons_loss 0.314 ms, 1.14 s total +[ 2023-09-21 10:24:50 ] Completed eval batch 0 metrics update 0.620 ms, 1.14 s total +Saving checkpoint at epoch 142 val batch 0 +[ 2023-09-21 10:24:51 ] Completed saving temp checkpoint 493.889 ms, 1.63 s total +[ 2023-09-21 10:24:51 ] Completed replacing temp checkpoint with checkpoint 30.930 ms, 1.66 s total +[ 2023-09-21 10:24:51 ] Completed eval batch 1 to device 315.539 ms, 1.98 s total +[ 2023-09-21 10:24:51 ] Completed eval batch 1 forward 21.540 ms, 2.00 s total +[ 2023-09-21 10:24:51 ] Completed eval batch 1 recons_loss 0.346 ms, 2.00 s total +[ 2023-09-21 10:24:51 ] Completed eval batch 1 metrics update 213.833 ms, 2.21 s total +Saving checkpoint at epoch 142 val batch 1 +Epoch 142 val loss: 0.0629 +[ 2023-09-21 10:24:52 ] Completed saving temp checkpoint 531.291 ms, 2.75 s total +[ 2023-09-21 10:24:52 ] Completed replacing temp checkpoint with checkpoint 23.161 ms, 2.77 s total +[ 2023-09-21 10:24:52 ] Completed evaluating generator for epoch 142 60.419 ms, 2.83 s total + + +EPOCH :: 143 + + +[ 2023-09-21 10:24:52 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 143 from training step 0 + +[ 2023-09-21 10:24:55 ] Completed train batch 0 to device 3,116.593 ms, 3.12 s total +[ 2023-09-21 10:24:55 ] Completed train batch 0 generator forward 33.263 ms, 3.15 s total +[ 2023-09-21 10:24:55 ] Completed train batch 0 generator loss: 1.271 66.432 ms, 3.22 s total +[ 2023-09-21 10:24:56 ] Completed train batch 0 generator backward 515.528 ms, 3.73 s total +[ 2023-09-21 10:24:56 ] Completed train batch 0 discriminator loss 0.047 60.127 ms, 3.79 s total +[ 2023-09-21 10:24:56 ] Completed train batch 0 discriminator backward 84.593 ms, 3.88 s total +Epoch [143] Step [0/2], gen_loss: 0.412, disc_loss: 0.015 +[ 2023-09-21 10:24:56 ] Completed train batch 0 metrics update 6.183 ms, 3.88 s total +Saving checkpoint at epoch 143 train batch 0 +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 10:42:32 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 10:42:32 ] Completed importing Timer 0.021 ms, 0.00 s total +| distributed init (rank 5): env:// +| distributed init (rank 3): env:// +| distributed init (rank 4): env:// +| distributed init (rank 2): env:// +| distributed init (rank 1): env:// +[ 2023-09-21 10:42:40 ] Completed importing everything else 8,772.280 ms, 8.77 s total +| distributed init (rank 0): env:// +[ 2023-09-21 10:42:44 ] Completed preliminaries 4,121.876 ms, 12.89 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 10:42:44 ] Completed build datasets 10.792 ms, 12.90 s total +[ 2023-09-21 10:42:44 ] Completed build samplers 0.066 ms, 12.91 s total +[ 2023-09-21 10:42:44 ] Completed build dataloaders 0.156 ms, 12.91 s total +[ 2023-09-21 10:42:44 ] Completed generator to device 75.073 ms, 12.98 s total +[ 2023-09-21 10:42:44 ] Completed discriminator to device 7.087 ms, 12.99 s total +[ 2023-09-21 10:42:45 ] Completed loss functions 382.795 ms, 13.37 s total +[ 2023-09-21 10:42:45 ] Completed models prepped for distribution 125.037 ms, 13.50 s total +[ 2023-09-21 10:42:45 ] Completed optimizers 0.650 ms, 13.50 s total +[ 2023-09-21 10:42:45 ] Completed grad scalers 0.033 ms, 13.50 s total +[ 2023-09-21 10:42:45 ] Completed checkpoint retrieval 483.028 ms, 13.98 s total + + +EPOCH :: 142 + + +[ 2023-09-21 10:42:45 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 142 from training step 2 + +Epoch [142] :: gen_loss: 0.413, disc_loss: 0.016 +[ 2023-09-21 10:42:46 ] Completed training generator for epoch 142 138.134 ms, 0.14 s total +[ 2023-09-21 10:42:46 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 142 from eval step 2 + +[ 2023-09-21 10:42:46 ] Completed evaluating generator for epoch 142 125.944 ms, 0.13 s total + + +EPOCH :: 143 + + +[ 2023-09-21 10:42:46 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 143 from training step 0 + +[ 2023-09-21 10:42:49 ] Completed train batch 0 to device 3,561.110 ms, 3.56 s total +[ 2023-09-21 10:42:51 ] Completed train batch 0 generator forward 1,416.851 ms, 4.98 s total +[ 2023-09-21 10:42:51 ] Completed train batch 0 generator loss: 1.289 85.950 ms, 5.06 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 10:42:52 ] Completed train batch 0 generator backward 1,309.943 ms, 6.37 s total +[ 2023-09-21 10:42:52 ] Completed train batch 0 discriminator loss 0.046 56.159 ms, 6.43 s total +[ 2023-09-21 10:42:52 ] Completed train batch 0 discriminator backward 109.364 ms, 6.54 s total +Epoch [143] Step [0/2], gen_loss: 0.414, disc_loss: 0.015 +[ 2023-09-21 10:42:52 ] Completed train batch 0 metrics update 6.102 ms, 6.55 s total +Saving checkpoint at epoch 143 train batch 0 +[ 2023-09-21 10:42:53 ] Completed saving temp checkpoint 417.668 ms, 6.96 s total +[ 2023-09-21 10:42:53 ] Completed replacing temp checkpoint with checkpoint 122.121 ms, 7.09 s total +[ 2023-09-21 10:42:53 ] Completed train batch 1 to device 21.261 ms, 7.11 s total +[ 2023-09-21 10:42:53 ] Completed train batch 1 generator forward 36.468 ms, 7.14 s total +[ 2023-09-21 10:42:53 ] Completed train batch 1 generator loss: 1.170 61.987 ms, 7.20 s total +[ 2023-09-21 10:42:55 ] Completed train batch 1 generator backward 1,555.447 ms, 8.76 s total +[ 2023-09-21 10:42:55 ] Completed train batch 1 discriminator loss 0.045 55.824 ms, 8.82 s total +[ 2023-09-21 10:42:55 ] Completed train batch 1 discriminator backward 81.921 ms, 8.90 s total +Epoch [143] Step [1/2], gen_loss: 0.403, disc_loss: 0.015 +[ 2023-09-21 10:42:55 ] Completed train batch 1 metrics update 2.155 ms, 8.90 s total +Saving checkpoint at epoch 143 train batch 1 +[ 2023-09-21 10:42:55 ] Completed saving temp checkpoint 457.607 ms, 9.36 s total +[ 2023-09-21 10:42:55 ] Completed replacing temp checkpoint with checkpoint 31.119 ms, 9.39 s total +Epoch [143] :: gen_loss: 0.409, disc_loss: 0.015 +[ 2023-09-21 10:42:55 ] Completed training generator for epoch 143 54.076 ms, 9.44 s total +[ 2023-09-21 10:42:55 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 143 from eval step 0 + +[ 2023-09-21 10:42:56 ] Completed eval batch 0 to device 1,059.862 ms, 1.06 s total +[ 2023-09-21 10:42:56 ] Completed eval batch 0 forward 29.835 ms, 1.09 s total +[ 2023-09-21 10:42:56 ] Completed eval batch 0 recons_loss 0.266 ms, 1.09 s total +[ 2023-09-21 10:42:56 ] Completed eval batch 0 metrics update 0.539 ms, 1.09 s total +Saving checkpoint at epoch 143 val batch 0 +[ 2023-09-21 10:42:57 ] Completed saving temp checkpoint 432.654 ms, 1.52 s total +[ 2023-09-21 10:42:57 ] Completed replacing temp checkpoint with checkpoint 25.877 ms, 1.55 s total +[ 2023-09-21 10:42:57 ] Completed eval batch 1 to device 401.085 ms, 1.95 s total +[ 2023-09-21 10:42:57 ] Completed eval batch 1 forward 39.608 ms, 1.99 s total +[ 2023-09-21 10:42:57 ] Completed eval batch 1 recons_loss 0.551 ms, 1.99 s total +[ 2023-09-21 10:42:57 ] Completed eval batch 1 metrics update 271.050 ms, 2.26 s total +Saving checkpoint at epoch 143 val batch 1 +Epoch 143 val loss: 0.0686 +[ 2023-09-21 10:42:58 ] Completed saving temp checkpoint 441.700 ms, 2.70 s total +[ 2023-09-21 10:42:58 ] Completed replacing temp checkpoint with checkpoint 33.883 ms, 2.74 s total +[ 2023-09-21 10:42:58 ] Completed evaluating generator for epoch 143 69.174 ms, 2.81 s total + + +EPOCH :: 144 + + +[ 2023-09-21 10:42:58 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 144 from training step 0 + +[ 2023-09-21 10:43:01 ] Completed train batch 0 to device 3,023.098 ms, 3.02 s total +[ 2023-09-21 10:43:01 ] Completed train batch 0 generator forward 24.729 ms, 3.05 s total +[ 2023-09-21 10:43:01 ] Completed train batch 0 generator loss: 1.255 63.634 ms, 3.11 s total +[ 2023-09-21 10:43:02 ] Completed train batch 0 generator backward 691.951 ms, 3.80 s total +[ 2023-09-21 10:43:02 ] Completed train batch 0 discriminator loss 0.047 56.217 ms, 3.86 s total +[ 2023-09-21 10:43:02 ] Completed train batch 0 discriminator backward 94.420 ms, 3.95 s total +Epoch [144] Step [0/2], gen_loss: 0.412, disc_loss: 0.016 +[ 2023-09-21 10:43:02 ] Completed train batch 0 metrics update 6.544 ms, 3.96 s total +Saving checkpoint at epoch 144 train batch 0 +[ 2023-09-21 10:43:02 ] Completed saving temp checkpoint 502.617 ms, 4.46 s total +[ 2023-09-21 10:43:02 ] Completed replacing temp checkpoint with checkpoint 19.565 ms, 4.48 s total +[ 2023-09-21 10:43:04 ] Completed train batch 1 to device 1,288.468 ms, 5.77 s total +[ 2023-09-21 10:43:04 ] Completed train batch 1 generator forward 40.509 ms, 5.81 s total +[ 2023-09-21 10:43:04 ] Completed train batch 1 generator loss: 1.249 62.298 ms, 5.87 s total +[ 2023-09-21 10:43:05 ] Completed train batch 1 generator backward 1,550.041 ms, 7.42 s total +[ 2023-09-21 10:43:05 ] Completed train batch 1 discriminator loss 0.046 55.914 ms, 7.48 s total +[ 2023-09-21 10:43:06 ] Completed train batch 1 discriminator backward 73.898 ms, 7.55 s total +Epoch [144] Step [1/2], gen_loss: 0.415, disc_loss: 0.015 +[ 2023-09-21 10:43:06 ] Completed train batch 1 metrics update 5.344 ms, 7.56 s total +Saving checkpoint at epoch 144 train batch 1 +[ 2023-09-21 10:43:06 ] Completed saving temp checkpoint 700.197 ms, 8.26 s total +[ 2023-09-21 10:43:06 ] Completed replacing temp checkpoint with checkpoint 38.364 ms, 8.30 s total +Epoch [144] :: gen_loss: 0.414, disc_loss: 0.016 +[ 2023-09-21 10:43:06 ] Completed training generator for epoch 144 70.147 ms, 8.37 s total +[ 2023-09-21 10:43:06 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 144 from eval step 0 + +[ 2023-09-21 10:43:08 ] Completed eval batch 0 to device 1,145.354 ms, 1.15 s total +[ 2023-09-21 10:43:08 ] Completed eval batch 0 forward 42.188 ms, 1.19 s total +[ 2023-09-21 10:43:08 ] Completed eval batch 0 recons_loss 0.305 ms, 1.19 s total +[ 2023-09-21 10:43:08 ] Completed eval batch 0 metrics update 0.635 ms, 1.19 s total +Saving checkpoint at epoch 144 val batch 0 +[ 2023-09-21 10:43:08 ] Completed saving temp checkpoint 500.049 ms, 1.69 s total +[ 2023-09-21 10:43:08 ] Completed replacing temp checkpoint with checkpoint 38.193 ms, 1.73 s total +[ 2023-09-21 10:43:08 ] Completed eval batch 1 to device 219.799 ms, 1.95 s total +[ 2023-09-21 10:43:08 ] Completed eval batch 1 forward 38.100 ms, 1.98 s total +[ 2023-09-21 10:43:08 ] Completed eval batch 1 recons_loss 0.333 ms, 1.98 s total +[ 2023-09-21 10:43:08 ] Completed eval batch 1 metrics update 0.307 ms, 1.99 s total +Saving checkpoint at epoch 144 val batch 1 +Epoch 144 val loss: 0.0699 +[ 2023-09-21 10:43:09 ] Completed saving temp checkpoint 525.356 ms, 2.51 s total +[ 2023-09-21 10:43:09 ] Completed replacing temp checkpoint with checkpoint 26.581 ms, 2.54 s total +[ 2023-09-21 10:43:09 ] Completed evaluating generator for epoch 144 71.446 ms, 2.61 s total + + +EPOCH :: 145 + + +[ 2023-09-21 10:43:09 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 145 from training step 0 + +[ 2023-09-21 10:43:12 ] Completed train batch 0 to device 2,902.633 ms, 2.90 s total +[ 2023-09-21 10:43:12 ] Completed train batch 0 generator forward 25.954 ms, 2.93 s total +[ 2023-09-21 10:43:12 ] Completed train batch 0 generator loss: 1.240 65.788 ms, 2.99 s total +[ 2023-09-21 10:43:13 ] Completed train batch 0 generator backward 731.515 ms, 3.73 s total +[ 2023-09-21 10:43:13 ] Completed train batch 0 discriminator loss 0.046 57.425 ms, 3.78 s total +[ 2023-09-21 10:43:13 ] Completed train batch 0 discriminator backward 106.792 ms, 3.89 s total +Epoch [145] Step [0/2], gen_loss: 0.412, disc_loss: 0.015 +[ 2023-09-21 10:43:13 ] Completed train batch 0 metrics update 7.017 ms, 3.90 s total +Saving checkpoint at epoch 145 train batch 0 +[ 2023-09-21 10:43:13 ] Completed saving temp checkpoint 521.419 ms, 4.42 s total +[ 2023-09-21 10:43:13 ] Completed replacing temp checkpoint with checkpoint 30.298 ms, 4.45 s total +[ 2023-09-21 10:43:15 ] Completed train batch 1 to device 1,156.304 ms, 5.61 s total +[ 2023-09-21 10:43:15 ] Completed train batch 1 generator forward 41.118 ms, 5.65 s total +[ 2023-09-21 10:43:15 ] Completed train batch 1 generator loss: 1.202 63.048 ms, 5.71 s total +[ 2023-09-21 10:43:16 ] Completed train batch 1 generator backward 1,419.335 ms, 7.13 s total +[ 2023-09-21 10:43:16 ] Completed train batch 1 discriminator loss 0.048 55.868 ms, 7.18 s total +[ 2023-09-21 10:43:16 ] Completed train batch 1 discriminator backward 75.940 ms, 7.26 s total +Epoch [145] Step [1/2], gen_loss: 0.408, disc_loss: 0.016 +[ 2023-09-21 10:43:16 ] Completed train batch 1 metrics update 1.055 ms, 7.26 s total +Saving checkpoint at epoch 145 train batch 1 +[ 2023-09-21 10:43:17 ] Completed saving temp checkpoint 546.738 ms, 7.81 s total +[ 2023-09-21 10:43:17 ] Completed replacing temp checkpoint with checkpoint 23.977 ms, 7.83 s total +Epoch [145] :: gen_loss: 0.410, disc_loss: 0.016 +[ 2023-09-21 10:43:17 ] Completed training generator for epoch 145 72.989 ms, 7.91 s total +[ 2023-09-21 10:43:17 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 145 from eval step 0 + +[ 2023-09-21 10:43:18 ] Completed eval batch 0 to device 1,074.765 ms, 1.07 s total +[ 2023-09-21 10:43:18 ] Completed eval batch 0 forward 24.918 ms, 1.10 s total +[ 2023-09-21 10:43:18 ] Completed eval batch 0 recons_loss 0.364 ms, 1.10 s total +[ 2023-09-21 10:43:18 ] Completed eval batch 0 metrics update 0.719 ms, 1.10 s total +Saving checkpoint at epoch 145 val batch 0 +[ 2023-09-21 10:43:19 ] Completed saving temp checkpoint 517.401 ms, 1.62 s total +[ 2023-09-21 10:43:19 ] Completed replacing temp checkpoint with checkpoint 38.024 ms, 1.66 s total +[ 2023-09-21 10:43:19 ] Completed eval batch 1 to device 271.178 ms, 1.93 s total +[ 2023-09-21 10:43:19 ] Completed eval batch 1 forward 39.082 ms, 1.97 s total +[ 2023-09-21 10:43:19 ] Completed eval batch 1 recons_loss 0.337 ms, 1.97 s total +[ 2023-09-21 10:43:19 ] Completed eval batch 1 metrics update 115.254 ms, 2.08 s total +Saving checkpoint at epoch 145 val batch 1 +Epoch 145 val loss: 0.0603 +[ 2023-09-21 10:43:20 ] Completed saving temp checkpoint 557.422 ms, 2.64 s total +[ 2023-09-21 10:43:20 ] Completed replacing temp checkpoint with checkpoint 27.651 ms, 2.67 s total +[ 2023-09-21 10:43:20 ] Completed evaluating generator for epoch 145 55.608 ms, 2.72 s total + + +EPOCH :: 146 + + +[ 2023-09-21 10:43:20 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 146 from training step 0 + +[ 2023-09-21 10:43:22 ] Completed train batch 0 to device 2,846.205 ms, 2.85 s total +[ 2023-09-21 10:43:22 ] Completed train batch 0 generator forward 25.253 ms, 2.87 s total +[ 2023-09-21 10:43:23 ] Completed train batch 0 generator loss: 1.243 63.843 ms, 2.94 s total +[ 2023-09-21 10:43:23 ] Completed train batch 0 generator backward 830.507 ms, 3.77 s total +[ 2023-09-21 10:43:23 ] Completed train batch 0 discriminator loss 0.050 55.861 ms, 3.82 s total +[ 2023-09-21 10:43:24 ] Completed train batch 0 discriminator backward 103.338 ms, 3.93 s total +Epoch [146] Step [0/2], gen_loss: 0.410, disc_loss: 0.017 +[ 2023-09-21 10:43:24 ] Completed train batch 0 metrics update 9.692 ms, 3.93 s total +Saving checkpoint at epoch 146 train batch 0 +[ 2023-09-21 10:43:24 ] Completed saving temp checkpoint 521.883 ms, 4.46 s total +[ 2023-09-21 10:43:24 ] Completed replacing temp checkpoint with checkpoint 35.482 ms, 4.49 s total +[ 2023-09-21 10:43:25 ] Completed train batch 1 to device 928.141 ms, 5.42 s total +[ 2023-09-21 10:43:25 ] Completed train batch 1 generator forward 41.860 ms, 5.46 s total +[ 2023-09-21 10:43:25 ] Completed train batch 1 generator loss: 1.250 74.961 ms, 5.54 s total +[ 2023-09-21 10:43:27 ] Completed train batch 1 generator backward 1,750.573 ms, 7.29 s total +[ 2023-09-21 10:43:27 ] Completed train batch 1 discriminator loss 0.048 55.773 ms, 7.34 s total +[ 2023-09-21 10:43:27 ] Completed train batch 1 discriminator backward 76.781 ms, 7.42 s total +Epoch [146] Step [1/2], gen_loss: 0.411, disc_loss: 0.016 +[ 2023-09-21 10:43:27 ] Completed train batch 1 metrics update 5.492 ms, 7.43 s total +Saving checkpoint at epoch 146 train batch 1 +[ 2023-09-21 10:43:28 ] Completed saving temp checkpoint 548.950 ms, 7.97 s total +[ 2023-09-21 10:43:28 ] Completed replacing temp checkpoint with checkpoint 25.355 ms, 8.00 s total +Epoch [146] :: gen_loss: 0.410, disc_loss: 0.016 +[ 2023-09-21 10:43:28 ] Completed training generator for epoch 146 70.477 ms, 8.07 s total +[ 2023-09-21 10:43:28 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 146 from eval step 0 + +[ 2023-09-21 10:43:29 ] Completed eval batch 0 to device 1,032.994 ms, 1.03 s total +[ 2023-09-21 10:43:29 ] Completed eval batch 0 forward 42.508 ms, 1.08 s total +[ 2023-09-21 10:43:29 ] Completed eval batch 0 recons_loss 0.342 ms, 1.08 s total +[ 2023-09-21 10:43:29 ] Completed eval batch 0 metrics update 0.730 ms, 1.08 s total +Saving checkpoint at epoch 146 val batch 0 +[ 2023-09-21 10:43:29 ] Completed saving temp checkpoint 501.648 ms, 1.58 s total +[ 2023-09-21 10:43:29 ] Completed replacing temp checkpoint with checkpoint 40.272 ms, 1.62 s total +[ 2023-09-21 10:43:30 ] Completed eval batch 1 to device 295.863 ms, 1.91 s total +[ 2023-09-21 10:43:30 ] Completed eval batch 1 forward 39.556 ms, 1.95 s total +[ 2023-09-21 10:43:30 ] Completed eval batch 1 recons_loss 0.550 ms, 1.95 s total +[ 2023-09-21 10:43:30 ] Completed eval batch 1 metrics update 159.921 ms, 2.11 s total +Saving checkpoint at epoch 146 val batch 1 +Epoch 146 val loss: 0.0583 +[ 2023-09-21 10:43:30 ] Completed saving temp checkpoint 410.149 ms, 2.52 s total +[ 2023-09-21 10:43:30 ] Completed replacing temp checkpoint with checkpoint 28.198 ms, 2.55 s total +[ 2023-09-21 10:43:30 ] Completed evaluating generator for epoch 146 65.376 ms, 2.62 s total + + +EPOCH :: 147 + + +[ 2023-09-21 10:43:30 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 147 from training step 0 + +[ 2023-09-21 10:43:33 ] Completed train batch 0 to device 2,976.152 ms, 2.98 s total +[ 2023-09-21 10:43:33 ] Completed train batch 0 generator forward 26.329 ms, 3.00 s total +[ 2023-09-21 10:43:33 ] Completed train batch 0 generator loss: 1.264 65.016 ms, 3.07 s total +[ 2023-09-21 10:43:34 ] Completed train batch 0 generator backward 733.242 ms, 3.80 s total +[ 2023-09-21 10:43:34 ] Completed train batch 0 discriminator loss 0.048 56.448 ms, 3.86 s total +[ 2023-09-21 10:43:34 ] Completed train batch 0 discriminator backward 112.700 ms, 3.97 s total +Epoch [147] Step [0/2], gen_loss: 0.410, disc_loss: 0.016 +[ 2023-09-21 10:43:34 ] Completed train batch 0 metrics update 4.522 ms, 3.97 s total +Saving checkpoint at epoch 147 train batch 0 +[ 2023-09-21 10:43:35 ] Completed saving temp checkpoint 432.278 ms, 4.41 s total +[ 2023-09-21 10:43:35 ] Completed replacing temp checkpoint with checkpoint 33.685 ms, 4.44 s total +[ 2023-09-21 10:43:36 ] Completed train batch 1 to device 1,226.178 ms, 5.67 s total +[ 2023-09-21 10:43:36 ] Completed train batch 1 generator forward 22.381 ms, 5.69 s total +[ 2023-09-21 10:43:36 ] Completed train batch 1 generator loss: 1.410 63.359 ms, 5.75 s total +[ 2023-09-21 10:43:37 ] Completed train batch 1 generator backward 1,257.427 ms, 7.01 s total +[ 2023-09-21 10:43:37 ] Completed train batch 1 discriminator loss 0.058 55.626 ms, 7.07 s total +[ 2023-09-21 10:43:37 ] Completed train batch 1 discriminator backward 79.292 ms, 7.14 s total +Epoch [147] Step [1/2], gen_loss: 0.431, disc_loss: 0.020 +[ 2023-09-21 10:43:37 ] Completed train batch 1 metrics update 1.551 ms, 7.15 s total +Saving checkpoint at epoch 147 train batch 1 +[ 2023-09-21 10:43:38 ] Completed saving temp checkpoint 497.113 ms, 7.64 s total +[ 2023-09-21 10:43:38 ] Completed replacing temp checkpoint with checkpoint 30.985 ms, 7.67 s total +Epoch [147] :: gen_loss: 0.421, disc_loss: 0.018 +[ 2023-09-21 10:43:38 ] Completed training generator for epoch 147 82.212 ms, 7.76 s total +[ 2023-09-21 10:43:38 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 147 from eval step 0 + +[ 2023-09-21 10:43:39 ] Completed eval batch 0 to device 1,045.995 ms, 1.05 s total +[ 2023-09-21 10:43:39 ] Completed eval batch 0 forward 42.478 ms, 1.09 s total +[ 2023-09-21 10:43:39 ] Completed eval batch 0 recons_loss 0.314 ms, 1.09 s total +[ 2023-09-21 10:43:39 ] Completed eval batch 0 metrics update 0.593 ms, 1.09 s total +Saving checkpoint at epoch 147 val batch 0 +[ 2023-09-21 10:43:40 ] Completed saving temp checkpoint 409.008 ms, 1.50 s total +[ 2023-09-21 10:43:40 ] Completed replacing temp checkpoint with checkpoint 24.394 ms, 1.52 s total +[ 2023-09-21 10:43:40 ] Completed eval batch 1 to device 392.426 ms, 1.92 s total +[ 2023-09-21 10:43:40 ] Completed eval batch 1 forward 38.755 ms, 1.95 s total +[ 2023-09-21 10:43:40 ] Completed eval batch 1 recons_loss 0.329 ms, 1.95 s total +[ 2023-09-21 10:43:40 ] Completed eval batch 1 metrics update 206.840 ms, 2.16 s total +Saving checkpoint at epoch 147 val batch 1 +Epoch 147 val loss: 0.0547 +[ 2023-09-21 10:43:41 ] Completed saving temp checkpoint 463.453 ms, 2.62 s total +[ 2023-09-21 10:43:41 ] Completed replacing temp checkpoint with checkpoint 31.275 ms, 2.66 s total +[ 2023-09-21 10:43:41 ] Completed evaluating generator for epoch 147 53.828 ms, 2.71 s total + + +EPOCH :: 148 + + +[ 2023-09-21 10:43:41 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 148 from training step 0 + +[ 2023-09-21 10:43:44 ] Completed train batch 0 to device 3,051.711 ms, 3.05 s total +[ 2023-09-21 10:43:44 ] Completed train batch 0 generator forward 30.010 ms, 3.08 s total +[ 2023-09-21 10:43:44 ] Completed train batch 0 generator loss: 1.244 63.977 ms, 3.15 s total +[ 2023-09-21 10:43:45 ] Completed train batch 0 generator backward 678.881 ms, 3.82 s total +[ 2023-09-21 10:43:45 ] Completed train batch 0 discriminator loss 0.065 57.858 ms, 3.88 s total +[ 2023-09-21 10:43:45 ] Completed train batch 0 discriminator backward 104.584 ms, 3.99 s total +Epoch [148] Step [0/2], gen_loss: 0.416, disc_loss: 0.022 +[ 2023-09-21 10:43:45 ] Completed train batch 0 metrics update 4.926 ms, 3.99 s total +Saving checkpoint at epoch 148 train batch 0 +[ 2023-09-21 10:43:45 ] Completed saving temp checkpoint 521.848 ms, 4.51 s total +[ 2023-09-21 10:43:45 ] Completed replacing temp checkpoint with checkpoint 34.304 ms, 4.55 s total +[ 2023-09-21 10:43:47 ] Completed train batch 1 to device 1,304.602 ms, 5.85 s total +[ 2023-09-21 10:43:47 ] Completed train batch 1 generator forward 22.134 ms, 5.87 s total +[ 2023-09-21 10:43:47 ] Completed train batch 1 generator loss: 1.296 63.266 ms, 5.94 s total +[ 2023-09-21 10:43:48 ] Completed train batch 1 generator backward 1,258.418 ms, 7.20 s total +[ 2023-09-21 10:43:48 ] Completed train batch 1 discriminator loss 0.063 56.103 ms, 7.25 s total +[ 2023-09-21 10:43:48 ] Completed train batch 1 discriminator backward 80.426 ms, 7.33 s total +Epoch [148] Step [1/2], gen_loss: 0.428, disc_loss: 0.020 +[ 2023-09-21 10:43:48 ] Completed train batch 1 metrics update 1.536 ms, 7.33 s total +Saving checkpoint at epoch 148 train batch 1 +[ 2023-09-21 10:43:49 ] Completed saving temp checkpoint 631.191 ms, 7.97 s total +[ 2023-09-21 10:43:49 ] Completed replacing temp checkpoint with checkpoint 32.902 ms, 8.00 s total +Epoch [148] :: gen_loss: 0.422, disc_loss: 0.021 +[ 2023-09-21 10:43:49 ] Completed training generator for epoch 148 73.655 ms, 8.07 s total +[ 2023-09-21 10:43:49 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 148 from eval step 0 + +[ 2023-09-21 10:43:50 ] Completed eval batch 0 to device 1,084.074 ms, 1.08 s total +[ 2023-09-21 10:43:50 ] Completed eval batch 0 forward 25.775 ms, 1.11 s total +[ 2023-09-21 10:43:50 ] Completed eval batch 0 recons_loss 0.332 ms, 1.11 s total +[ 2023-09-21 10:43:50 ] Completed eval batch 0 metrics update 0.640 ms, 1.11 s total +Saving checkpoint at epoch 148 val batch 0 +[ 2023-09-21 10:43:51 ] Completed saving temp checkpoint 563.330 ms, 1.67 s total +[ 2023-09-21 10:43:51 ] Completed replacing temp checkpoint with checkpoint 25.021 ms, 1.70 s total +[ 2023-09-21 10:43:51 ] Completed eval batch 1 to device 372.865 ms, 2.07 s total +[ 2023-09-21 10:43:51 ] Completed eval batch 1 forward 21.382 ms, 2.09 s total +[ 2023-09-21 10:43:51 ] Completed eval batch 1 recons_loss 0.400 ms, 2.09 s total +[ 2023-09-21 10:43:51 ] Completed eval batch 1 metrics update 0.328 ms, 2.09 s total +Saving checkpoint at epoch 148 val batch 1 +Epoch 148 val loss: 0.0554 +[ 2023-09-21 10:43:52 ] Completed saving temp checkpoint 612.725 ms, 2.71 s total +[ 2023-09-21 10:43:52 ] Completed replacing temp checkpoint with checkpoint 34.388 ms, 2.74 s total +[ 2023-09-21 10:43:52 ] Completed evaluating generator for epoch 148 57.573 ms, 2.80 s total + + +EPOCH :: 149 + + +[ 2023-09-21 10:43:52 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 149 from training step 0 + +[ 2023-09-21 10:43:55 ] Completed train batch 0 to device 2,906.653 ms, 2.91 s total +[ 2023-09-21 10:43:55 ] Completed train batch 0 generator forward 44.198 ms, 2.95 s total +[ 2023-09-21 10:43:55 ] Completed train batch 0 generator loss: 1.253 64.538 ms, 3.02 s total +[ 2023-09-21 10:43:55 ] Completed train batch 0 generator backward 558.542 ms, 3.57 s total +[ 2023-09-21 10:43:55 ] Completed train batch 0 discriminator loss 0.057 56.515 ms, 3.63 s total +[ 2023-09-21 10:43:55 ] Completed train batch 0 discriminator backward 116.663 ms, 3.75 s total +Epoch [149] Step [0/2], gen_loss: 0.425, disc_loss: 0.019 +[ 2023-09-21 10:43:55 ] Completed train batch 0 metrics update 13.104 ms, 3.76 s total +Saving checkpoint at epoch 149 train batch 0 +[ 2023-09-21 10:43:56 ] Completed saving temp checkpoint 597.449 ms, 4.36 s total +[ 2023-09-21 10:43:56 ] Completed replacing temp checkpoint with checkpoint 34.696 ms, 4.39 s total +[ 2023-09-21 10:43:57 ] Completed train batch 1 to device 1,217.597 ms, 5.61 s total +[ 2023-09-21 10:43:57 ] Completed train batch 1 generator forward 39.660 ms, 5.65 s total +[ 2023-09-21 10:43:57 ] Completed train batch 1 generator loss: 1.286 62.222 ms, 5.71 s total +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 10:57:25 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 10:57:25 ] Completed importing Timer 0.022 ms, 0.00 s total +[ 2023-09-21 10:57:30 ] Completed importing everything else 4,588.223 ms, 4.59 s total +| distributed init (rank 0): env:// +| distributed init (rank 2): env:// +| distributed init (rank 4): env:// +| distributed init (rank 5): env:// +| distributed init (rank 3): env:// +| distributed init (rank 1): env:// +[ 2023-09-21 10:57:37 ] Completed preliminaries 7,529.043 ms, 12.12 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 10:57:37 ] Completed build datasets 11.615 ms, 12.13 s total +[ 2023-09-21 10:57:37 ] Completed build samplers 0.043 ms, 12.13 s total +[ 2023-09-21 10:57:37 ] Completed build dataloaders 0.127 ms, 12.13 s total +[ 2023-09-21 10:57:37 ] Completed generator to device 92.414 ms, 12.22 s total +[ 2023-09-21 10:57:37 ] Completed discriminator to device 6.997 ms, 12.23 s total +[ 2023-09-21 10:57:38 ] Completed loss functions 403.100 ms, 12.63 s total +[ 2023-09-21 10:57:38 ] Completed models prepped for distribution 105.794 ms, 12.74 s total +[ 2023-09-21 10:57:38 ] Completed optimizers 0.694 ms, 12.74 s total +[ 2023-09-21 10:57:38 ] Completed grad scalers 0.030 ms, 12.74 s total +[ 2023-09-21 10:57:38 ] Completed checkpoint retrieval 460.245 ms, 13.20 s total + + +EPOCH :: 149 + + +[ 2023-09-21 10:57:38 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 149 from training step 1 + +[ 2023-09-21 10:57:42 ] Completed train batch 1 to device 3,258.652 ms, 3.26 s total +[ 2023-09-21 10:57:43 ] Completed train batch 1 generator forward 1,106.073 ms, 4.36 s total +[ 2023-09-21 10:57:43 ] Completed train batch 1 generator loss: 1.210 79.032 ms, 4.44 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 10:57:45 ] Completed train batch 1 generator backward 1,941.900 ms, 6.39 s total +[ 2023-09-21 10:57:45 ] Completed train batch 1 discriminator loss 0.053 56.121 ms, 6.44 s total +[ 2023-09-21 10:57:45 ] Completed train batch 1 discriminator backward 85.566 ms, 6.53 s total +Epoch [149] Step [1/2], gen_loss: 0.418, disc_loss: 0.018 +[ 2023-09-21 10:57:45 ] Completed train batch 1 metrics update 5.338 ms, 6.53 s total +Saving checkpoint at epoch 149 train batch 1 +[ 2023-09-21 10:57:45 ] Completed saving temp checkpoint 462.524 ms, 7.00 s total +[ 2023-09-21 10:57:45 ] Completed replacing temp checkpoint with checkpoint 118.984 ms, 7.11 s total +Epoch [149] :: gen_loss: 0.422, disc_loss: 0.019 +[ 2023-09-21 10:57:45 ] Completed training generator for epoch 149 56.589 ms, 7.17 s total +[ 2023-09-21 10:57:45 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 149 from eval step 0 + +[ 2023-09-21 10:57:47 ] Completed eval batch 0 to device 1,103.730 ms, 1.10 s total +[ 2023-09-21 10:57:47 ] Completed eval batch 0 forward 33.267 ms, 1.14 s total +[ 2023-09-21 10:57:47 ] Completed eval batch 0 recons_loss 0.426 ms, 1.14 s total +[ 2023-09-21 10:57:47 ] Completed eval batch 0 metrics update 0.746 ms, 1.14 s total +Saving checkpoint at epoch 149 val batch 0 +[ 2023-09-21 10:57:47 ] Completed saving temp checkpoint 439.329 ms, 1.58 s total +[ 2023-09-21 10:57:47 ] Completed replacing temp checkpoint with checkpoint 31.009 ms, 1.61 s total +[ 2023-09-21 10:57:47 ] Completed eval batch 1 to device 405.169 ms, 2.01 s total +[ 2023-09-21 10:57:47 ] Completed eval batch 1 forward 21.624 ms, 2.04 s total +[ 2023-09-21 10:57:47 ] Completed eval batch 1 recons_loss 0.349 ms, 2.04 s total +[ 2023-09-21 10:57:48 ] Completed eval batch 1 metrics update 171.412 ms, 2.21 s total +Saving checkpoint at epoch 149 val batch 1 +Epoch 149 val loss: 0.0621 +[ 2023-09-21 10:57:48 ] Completed saving temp checkpoint 453.253 ms, 2.66 s total +[ 2023-09-21 10:57:48 ] Completed replacing temp checkpoint with checkpoint 19.542 ms, 2.68 s total +[ 2023-09-21 10:57:48 ] Completed evaluating generator for epoch 149 60.507 ms, 2.74 s total + + +EPOCH :: 150 + + +[ 2023-09-21 10:57:48 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 150 from training step 0 + +[ 2023-09-21 10:57:51 ] Completed train batch 0 to device 2,883.296 ms, 2.88 s total +[ 2023-09-21 10:57:51 ] Completed train batch 0 generator forward 25.535 ms, 2.91 s total +[ 2023-09-21 10:57:51 ] Completed train batch 0 generator loss: 1.257 65.573 ms, 2.97 s total +[ 2023-09-21 10:57:52 ] Completed train batch 0 generator backward 930.514 ms, 3.90 s total +[ 2023-09-21 10:57:52 ] Completed train batch 0 discriminator loss 0.055 59.089 ms, 3.96 s total +[ 2023-09-21 10:57:52 ] Completed train batch 0 discriminator backward 104.074 ms, 4.07 s total +Epoch [150] Step [0/2], gen_loss: 0.418, disc_loss: 0.018 +[ 2023-09-21 10:57:52 ] Completed train batch 0 metrics update 5.988 ms, 4.07 s total +Saving checkpoint at epoch 150 train batch 0 +[ 2023-09-21 10:57:53 ] Completed saving temp checkpoint 458.210 ms, 4.53 s total +[ 2023-09-21 10:57:53 ] Completed replacing temp checkpoint with checkpoint 19.872 ms, 4.55 s total +[ 2023-09-21 10:57:54 ] Completed train batch 1 to device 1,534.312 ms, 6.09 s total +[ 2023-09-21 10:57:54 ] Completed train batch 1 generator forward 26.624 ms, 6.11 s total +[ 2023-09-21 10:57:54 ] Completed train batch 1 generator loss: 1.207 64.620 ms, 6.18 s total +[ 2023-09-21 10:57:56 ] Completed train batch 1 generator backward 1,307.292 ms, 7.49 s total +[ 2023-09-21 10:57:56 ] Completed train batch 1 discriminator loss 0.053 57.185 ms, 7.54 s total +[ 2023-09-21 10:57:56 ] Completed train batch 1 discriminator backward 68.702 ms, 7.61 s total +Epoch [150] Step [1/2], gen_loss: 0.407, disc_loss: 0.018 +[ 2023-09-21 10:57:56 ] Completed train batch 1 metrics update 1.620 ms, 7.61 s total +Saving checkpoint at epoch 150 train batch 1 +[ 2023-09-21 10:57:56 ] Completed saving temp checkpoint 464.388 ms, 8.08 s total +[ 2023-09-21 10:57:56 ] Completed replacing temp checkpoint with checkpoint 24.051 ms, 8.10 s total +Epoch [150] :: gen_loss: 0.413, disc_loss: 0.018 +[ 2023-09-21 10:57:56 ] Completed training generator for epoch 150 50.487 ms, 8.15 s total +[ 2023-09-21 10:57:56 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 150 from eval step 0 + +[ 2023-09-21 10:57:57 ] Completed eval batch 0 to device 1,165.420 ms, 1.17 s total +[ 2023-09-21 10:57:58 ] Completed eval batch 0 forward 28.490 ms, 1.19 s total +[ 2023-09-21 10:57:58 ] Completed eval batch 0 recons_loss 0.318 ms, 1.19 s total +[ 2023-09-21 10:57:58 ] Completed eval batch 0 metrics update 0.579 ms, 1.19 s total +Saving checkpoint at epoch 150 val batch 0 +[ 2023-09-21 10:57:58 ] Completed saving temp checkpoint 440.180 ms, 1.63 s total +[ 2023-09-21 10:57:58 ] Completed replacing temp checkpoint with checkpoint 23.681 ms, 1.66 s total +[ 2023-09-21 10:57:58 ] Completed eval batch 1 to device 471.810 ms, 2.13 s total +[ 2023-09-21 10:57:58 ] Completed eval batch 1 forward 33.146 ms, 2.16 s total +[ 2023-09-21 10:57:58 ] Completed eval batch 1 recons_loss 0.408 ms, 2.16 s total +[ 2023-09-21 10:57:59 ] Completed eval batch 1 metrics update 162.725 ms, 2.33 s total +Saving checkpoint at epoch 150 val batch 1 +Epoch 150 val loss: 0.0683 +[ 2023-09-21 10:57:59 ] Completed saving temp checkpoint 449.929 ms, 2.78 s total +[ 2023-09-21 10:57:59 ] Completed replacing temp checkpoint with checkpoint 26.987 ms, 2.80 s total +[ 2023-09-21 10:57:59 ] Completed evaluating generator for epoch 150 51.532 ms, 2.86 s total + + +EPOCH :: 151 + + +[ 2023-09-21 10:57:59 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 151 from training step 0 + +[ 2023-09-21 10:58:02 ] Completed train batch 0 to device 3,186.571 ms, 3.19 s total +[ 2023-09-21 10:58:02 ] Completed train batch 0 generator forward 25.311 ms, 3.21 s total +[ 2023-09-21 10:58:02 ] Completed train batch 0 generator loss: 1.242 64.836 ms, 3.28 s total +[ 2023-09-21 10:58:03 ] Completed train batch 0 generator backward 591.204 ms, 3.87 s total +[ 2023-09-21 10:58:03 ] Completed train batch 0 discriminator loss 0.053 60.925 ms, 3.93 s total +[ 2023-09-21 10:58:03 ] Completed train batch 0 discriminator backward 75.221 ms, 4.00 s total +Epoch [151] Step [0/2], gen_loss: 0.425, disc_loss: 0.018 +[ 2023-09-21 10:58:03 ] Completed train batch 0 metrics update 5.284 ms, 4.01 s total +Saving checkpoint at epoch 151 train batch 0 +[ 2023-09-21 10:58:04 ] Completed saving temp checkpoint 493.921 ms, 4.50 s total +[ 2023-09-21 10:58:04 ] Completed replacing temp checkpoint with checkpoint 18.737 ms, 4.52 s total +[ 2023-09-21 10:58:05 ] Completed train batch 1 to device 1,675.619 ms, 6.20 s total +[ 2023-09-21 10:58:05 ] Completed train batch 1 generator forward 23.564 ms, 6.22 s total +[ 2023-09-21 10:58:05 ] Completed train batch 1 generator loss: 1.238 64.709 ms, 6.29 s total +[ 2023-09-21 10:58:07 ] Completed train batch 1 generator backward 1,188.705 ms, 7.47 s total +[ 2023-09-21 10:58:07 ] Completed train batch 1 discriminator loss 0.051 56.443 ms, 7.53 s total +[ 2023-09-21 10:58:07 ] Completed train batch 1 discriminator backward 72.627 ms, 7.60 s total +Epoch [151] Step [1/2], gen_loss: 0.415, disc_loss: 0.017 +[ 2023-09-21 10:58:07 ] Completed train batch 1 metrics update 2.375 ms, 7.61 s total +Saving checkpoint at epoch 151 train batch 1 +[ 2023-09-21 10:58:07 ] Completed saving temp checkpoint 487.261 ms, 8.09 s total +[ 2023-09-21 10:58:07 ] Completed replacing temp checkpoint with checkpoint 23.964 ms, 8.12 s total +Epoch [151] :: gen_loss: 0.420, disc_loss: 0.017 +[ 2023-09-21 10:58:07 ] Completed training generator for epoch 151 48.526 ms, 8.17 s total +[ 2023-09-21 10:58:07 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 151 from eval step 0 + +[ 2023-09-21 10:58:08 ] Completed eval batch 0 to device 1,120.697 ms, 1.12 s total +[ 2023-09-21 10:58:08 ] Completed eval batch 0 forward 25.043 ms, 1.15 s total +[ 2023-09-21 10:58:08 ] Completed eval batch 0 recons_loss 0.304 ms, 1.15 s total +[ 2023-09-21 10:58:08 ] Completed eval batch 0 metrics update 0.703 ms, 1.15 s total +Saving checkpoint at epoch 151 val batch 0 +[ 2023-09-21 10:58:09 ] Completed saving temp checkpoint 426.425 ms, 1.57 s total +[ 2023-09-21 10:58:09 ] Completed replacing temp checkpoint with checkpoint 30.617 ms, 1.60 s total +[ 2023-09-21 10:58:09 ] Completed eval batch 1 to device 542.062 ms, 2.15 s total +[ 2023-09-21 10:58:10 ] Completed eval batch 1 forward 39.661 ms, 2.19 s total +[ 2023-09-21 10:58:10 ] Completed eval batch 1 recons_loss 0.340 ms, 2.19 s total +[ 2023-09-21 10:58:10 ] Completed eval batch 1 metrics update 189.996 ms, 2.38 s total +Saving checkpoint at epoch 151 val batch 1 +Epoch 151 val loss: 0.0636 +[ 2023-09-21 10:58:10 ] Completed saving temp checkpoint 430.006 ms, 2.81 s total +[ 2023-09-21 10:58:10 ] Completed replacing temp checkpoint with checkpoint 24.000 ms, 2.83 s total +[ 2023-09-21 10:58:10 ] Completed evaluating generator for epoch 151 47.262 ms, 2.88 s total + + +EPOCH :: 152 + + +[ 2023-09-21 10:58:10 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 152 from training step 0 + +[ 2023-09-21 10:58:13 ] Completed train batch 0 to device 2,811.174 ms, 2.81 s total +[ 2023-09-21 10:58:13 ] Completed train batch 0 generator forward 39.923 ms, 2.85 s total +[ 2023-09-21 10:58:13 ] Completed train batch 0 generator loss: 1.244 64.236 ms, 2.92 s total +[ 2023-09-21 10:58:14 ] Completed train batch 0 generator backward 886.731 ms, 3.80 s total +[ 2023-09-21 10:58:14 ] Completed train batch 0 discriminator loss 0.051 57.405 ms, 3.86 s total +[ 2023-09-21 10:58:14 ] Completed train batch 0 discriminator backward 117.558 ms, 3.98 s total +Epoch [152] Step [0/2], gen_loss: 0.420, disc_loss: 0.017 +[ 2023-09-21 10:58:14 ] Completed train batch 0 metrics update 15.209 ms, 3.99 s total +Saving checkpoint at epoch 152 train batch 0 +[ 2023-09-21 10:58:15 ] Completed saving temp checkpoint 423.455 ms, 4.42 s total +[ 2023-09-21 10:58:15 ] Completed replacing temp checkpoint with checkpoint 16.239 ms, 4.43 s total +[ 2023-09-21 10:58:16 ] Completed train batch 1 to device 1,163.977 ms, 5.60 s total +[ 2023-09-21 10:58:16 ] Completed train batch 1 generator forward 29.892 ms, 5.63 s total +[ 2023-09-21 10:58:16 ] Completed train batch 1 generator loss: 1.347 61.972 ms, 5.69 s total +[ 2023-09-21 10:58:17 ] Completed train batch 1 generator backward 1,444.819 ms, 7.13 s total +[ 2023-09-21 10:58:17 ] Completed train batch 1 discriminator loss 0.050 56.557 ms, 7.19 s total +[ 2023-09-21 10:58:17 ] Completed train batch 1 discriminator backward 65.544 ms, 7.25 s total +Epoch [152] Step [1/2], gen_loss: 0.415, disc_loss: 0.017 +[ 2023-09-21 10:58:17 ] Completed train batch 1 metrics update 5.849 ms, 7.26 s total +Saving checkpoint at epoch 152 train batch 1 +[ 2023-09-21 10:58:18 ] Completed saving temp checkpoint 438.705 ms, 7.70 s total +[ 2023-09-21 10:58:18 ] Completed replacing temp checkpoint with checkpoint 20.777 ms, 7.72 s total +Epoch [152] :: gen_loss: 0.418, disc_loss: 0.017 +[ 2023-09-21 10:58:18 ] Completed training generator for epoch 152 58.139 ms, 7.78 s total +[ 2023-09-21 10:58:18 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 152 from eval step 0 + +[ 2023-09-21 10:58:19 ] Completed eval batch 0 to device 1,130.713 ms, 1.13 s total +[ 2023-09-21 10:58:19 ] Completed eval batch 0 forward 27.057 ms, 1.16 s total +[ 2023-09-21 10:58:19 ] Completed eval batch 0 recons_loss 0.325 ms, 1.16 s total +[ 2023-09-21 10:58:19 ] Completed eval batch 0 metrics update 0.632 ms, 1.16 s total +Saving checkpoint at epoch 152 val batch 0 +[ 2023-09-21 10:58:20 ] Completed saving temp checkpoint 433.626 ms, 1.59 s total +[ 2023-09-21 10:58:20 ] Completed replacing temp checkpoint with checkpoint 24.051 ms, 1.62 s total +[ 2023-09-21 10:58:20 ] Completed eval batch 1 to device 411.947 ms, 2.03 s total +[ 2023-09-21 10:58:20 ] Completed eval batch 1 forward 40.621 ms, 2.07 s total +[ 2023-09-21 10:58:20 ] Completed eval batch 1 recons_loss 0.541 ms, 2.07 s total +[ 2023-09-21 10:58:20 ] Completed eval batch 1 metrics update 197.105 ms, 2.27 s total +Saving checkpoint at epoch 152 val batch 1 +Epoch 152 val loss: 0.0581 +[ 2023-09-21 10:58:21 ] Completed saving temp checkpoint 623.217 ms, 2.89 s total +[ 2023-09-21 10:58:21 ] Completed replacing temp checkpoint with checkpoint 46.093 ms, 2.94 s total +[ 2023-09-21 10:58:21 ] Completed evaluating generator for epoch 152 48.824 ms, 2.98 s total + + +EPOCH :: 153 + + +[ 2023-09-21 10:58:21 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 153 from training step 0 + +[ 2023-09-21 10:58:24 ] Completed train batch 0 to device 3,117.366 ms, 3.12 s total +[ 2023-09-21 10:58:24 ] Completed train batch 0 generator forward 25.600 ms, 3.14 s total +[ 2023-09-21 10:58:24 ] Completed train batch 0 generator loss: 1.202 65.402 ms, 3.21 s total +[ 2023-09-21 10:58:25 ] Completed train batch 0 generator backward 507.061 ms, 3.72 s total +[ 2023-09-21 10:58:25 ] Completed train batch 0 discriminator loss 0.052 64.424 ms, 3.78 s total +[ 2023-09-21 10:58:25 ] Completed train batch 0 discriminator backward 96.414 ms, 3.88 s total +Epoch [153] Step [0/2], gen_loss: 0.411, disc_loss: 0.017 +[ 2023-09-21 10:58:25 ] Completed train batch 0 metrics update 6.155 ms, 3.88 s total +Saving checkpoint at epoch 153 train batch 0 +[ 2023-09-21 10:58:25 ] Completed saving temp checkpoint 452.706 ms, 4.34 s total +[ 2023-09-21 10:58:25 ] Completed replacing temp checkpoint with checkpoint 16.782 ms, 4.35 s total +[ 2023-09-21 10:58:27 ] Completed train batch 1 to device 2,148.747 ms, 6.50 s total +[ 2023-09-21 10:58:28 ] Completed train batch 1 generator forward 41.265 ms, 6.54 s total +[ 2023-09-21 10:58:28 ] Completed train batch 1 generator loss: 1.250 64.441 ms, 6.61 s total +[ 2023-09-21 10:58:28 ] Completed train batch 1 generator backward 716.992 ms, 7.32 s total +[ 2023-09-21 10:58:28 ] Completed train batch 1 discriminator loss 0.052 56.474 ms, 7.38 s total +[ 2023-09-21 10:58:28 ] Completed train batch 1 discriminator backward 74.228 ms, 7.45 s total +Epoch [153] Step [1/2], gen_loss: 0.406, disc_loss: 0.017 +[ 2023-09-21 10:58:28 ] Completed train batch 1 metrics update 5.365 ms, 7.46 s total +Saving checkpoint at epoch 153 train batch 1 +[ 2023-09-21 10:58:29 ] Completed saving temp checkpoint 484.147 ms, 7.94 s total +[ 2023-09-21 10:58:29 ] Completed replacing temp checkpoint with checkpoint 23.783 ms, 7.97 s total +Epoch [153] :: gen_loss: 0.409, disc_loss: 0.017 +[ 2023-09-21 10:58:29 ] Completed training generator for epoch 153 55.202 ms, 8.02 s total +[ 2023-09-21 10:58:29 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 153 from eval step 0 + +[ 2023-09-21 10:58:30 ] Completed eval batch 0 to device 1,164.895 ms, 1.16 s total +[ 2023-09-21 10:58:30 ] Completed eval batch 0 forward 32.862 ms, 1.20 s total +[ 2023-09-21 10:58:30 ] Completed eval batch 0 recons_loss 0.382 ms, 1.20 s total +[ 2023-09-21 10:58:30 ] Completed eval batch 0 metrics update 0.687 ms, 1.20 s total +Saving checkpoint at epoch 153 val batch 0 +[ 2023-09-21 10:58:31 ] Completed saving temp checkpoint 447.940 ms, 1.65 s total +[ 2023-09-21 10:58:31 ] Completed replacing temp checkpoint with checkpoint 21.371 ms, 1.67 s total +[ 2023-09-21 10:58:31 ] Completed eval batch 1 to device 368.625 ms, 2.04 s total +[ 2023-09-21 10:58:31 ] Completed eval batch 1 forward 21.795 ms, 2.06 s total +[ 2023-09-21 10:58:31 ] Completed eval batch 1 recons_loss 0.360 ms, 2.06 s total +[ 2023-09-21 10:58:31 ] Completed eval batch 1 metrics update 224.402 ms, 2.28 s total +Saving checkpoint at epoch 153 val batch 1 +Epoch 153 val loss: 0.0592 +[ 2023-09-21 10:58:32 ] Completed saving temp checkpoint 469.256 ms, 2.75 s total +[ 2023-09-21 10:58:32 ] Completed replacing temp checkpoint with checkpoint 26.219 ms, 2.78 s total +[ 2023-09-21 10:58:32 ] Completed evaluating generator for epoch 153 46.816 ms, 2.83 s total + + +EPOCH :: 154 + + +[ 2023-09-21 10:58:32 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 154 from training step 0 + +[ 2023-09-21 10:58:35 ] Completed train batch 0 to device 3,182.660 ms, 3.18 s total +[ 2023-09-21 10:58:35 ] Completed train batch 0 generator forward 32.745 ms, 3.22 s total +[ 2023-09-21 10:58:35 ] Completed train batch 0 generator loss: 1.237 75.573 ms, 3.29 s total +[ 2023-09-21 10:58:36 ] Completed train batch 0 generator backward 527.516 ms, 3.82 s total +[ 2023-09-21 10:58:36 ] Completed train batch 0 discriminator loss 0.053 57.814 ms, 3.88 s total +[ 2023-09-21 10:58:36 ] Completed train batch 0 discriminator backward 106.554 ms, 3.98 s total +Epoch [154] Step [0/2], gen_loss: 0.411, disc_loss: 0.018 +[ 2023-09-21 10:58:36 ] Completed train batch 0 metrics update 5.708 ms, 3.99 s total +Saving checkpoint at epoch 154 train batch 0 +[ 2023-09-21 10:58:36 ] Completed saving temp checkpoint 546.757 ms, 4.54 s total +[ 2023-09-21 10:58:36 ] Completed replacing temp checkpoint with checkpoint 26.505 ms, 4.56 s total +[ 2023-09-21 10:58:38 ] Completed train batch 1 to device 1,693.154 ms, 6.25 s total +[ 2023-09-21 10:58:38 ] Completed train batch 1 generator forward 22.711 ms, 6.28 s total +[ 2023-09-21 10:58:38 ] Completed train batch 1 generator loss: 1.182 63.680 ms, 6.34 s total +[ 2023-09-21 10:58:39 ] Completed train batch 1 generator backward 1,082.971 ms, 7.42 s total +[ 2023-09-21 10:58:39 ] Completed train batch 1 discriminator loss 0.059 56.555 ms, 7.48 s total +[ 2023-09-21 10:58:39 ] Completed train batch 1 discriminator backward 69.227 ms, 7.55 s total +Epoch [154] Step [1/2], gen_loss: 0.401, disc_loss: 0.020 +[ 2023-09-21 10:58:39 ] Completed train batch 1 metrics update 5.767 ms, 7.56 s total +Saving checkpoint at epoch 154 train batch 1 +[ 2023-09-21 10:58:40 ] Completed saving temp checkpoint 566.742 ms, 8.12 s total +[ 2023-09-21 10:58:40 ] Completed replacing temp checkpoint with checkpoint 35.727 ms, 8.16 s total +Epoch [154] :: gen_loss: 0.406, disc_loss: 0.019 +[ 2023-09-21 10:58:40 ] Completed training generator for epoch 154 74.848 ms, 8.23 s total +[ 2023-09-21 10:58:40 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 154 from eval step 0 + +[ 2023-09-21 10:58:41 ] Completed eval batch 0 to device 1,184.157 ms, 1.18 s total +[ 2023-09-21 10:58:41 ] Completed eval batch 0 forward 32.066 ms, 1.22 s total +[ 2023-09-21 10:58:41 ] Completed eval batch 0 recons_loss 0.353 ms, 1.22 s total +[ 2023-09-21 10:58:41 ] Completed eval batch 0 metrics update 0.671 ms, 1.22 s total +Saving checkpoint at epoch 154 val batch 0 +[ 2023-09-21 10:58:42 ] Completed saving temp checkpoint 441.991 ms, 1.66 s total +[ 2023-09-21 10:58:42 ] Completed replacing temp checkpoint with checkpoint 23.425 ms, 1.68 s total +[ 2023-09-21 10:58:42 ] Completed eval batch 1 to device 427.953 ms, 2.11 s total +[ 2023-09-21 10:58:42 ] Completed eval batch 1 forward 22.208 ms, 2.13 s total +[ 2023-09-21 10:58:42 ] Completed eval batch 1 recons_loss 0.407 ms, 2.13 s total +[ 2023-09-21 10:58:42 ] Completed eval batch 1 metrics update 0.383 ms, 2.13 s total +Saving checkpoint at epoch 154 val batch 1 +Epoch 154 val loss: 0.0558 +[ 2023-09-21 10:58:43 ] Completed saving temp checkpoint 455.896 ms, 2.59 s total +[ 2023-09-21 10:58:43 ] Completed replacing temp checkpoint with checkpoint 20.323 ms, 2.61 s total +[ 2023-09-21 10:58:43 ] Completed evaluating generator for epoch 154 49.392 ms, 2.66 s total + + +EPOCH :: 155 + + +[ 2023-09-21 10:58:43 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 155 from training step 0 + +[ 2023-09-21 10:58:46 ] Completed train batch 0 to device 3,057.070 ms, 3.06 s total +[ 2023-09-21 10:58:46 ] Completed train batch 0 generator forward 26.935 ms, 3.08 s total +[ 2023-09-21 10:58:46 ] Completed train batch 0 generator loss: 1.203 66.341 ms, 3.15 s total +[ 2023-09-21 10:58:46 ] Completed train batch 0 generator backward 620.963 ms, 3.77 s total +[ 2023-09-21 10:58:47 ] Completed train batch 0 discriminator loss 0.071 56.967 ms, 3.83 s total +[ 2023-09-21 10:58:47 ] Completed train batch 0 discriminator backward 101.940 ms, 3.93 s total +Epoch [155] Step [0/2], gen_loss: 0.401, disc_loss: 0.023 +[ 2023-09-21 10:58:47 ] Completed train batch 0 metrics update 12.638 ms, 3.94 s total +Saving checkpoint at epoch 155 train batch 0 +[ 2023-09-21 10:58:47 ] Completed saving temp checkpoint 440.228 ms, 4.38 s total +[ 2023-09-21 10:58:47 ] Completed replacing temp checkpoint with checkpoint 24.667 ms, 4.41 s total +[ 2023-09-21 10:58:49 ] Completed train batch 1 to device 1,461.780 ms, 5.87 s total +[ 2023-09-21 10:58:49 ] Completed train batch 1 generator forward 39.912 ms, 5.91 s total +[ 2023-09-21 10:58:49 ] Completed train batch 1 generator loss: 1.242 63.450 ms, 5.97 s total +[ 2023-09-21 10:58:50 ] Completed train batch 1 generator backward 1,287.871 ms, 7.26 s total +[ 2023-09-21 10:58:50 ] Completed train batch 1 discriminator loss 0.063 56.587 ms, 7.32 s total +[ 2023-09-21 10:58:50 ] Completed train batch 1 discriminator backward 73.250 ms, 7.39 s total +Epoch [155] Step [1/2], gen_loss: 0.415, disc_loss: 0.022 +[ 2023-09-21 10:58:50 ] Completed train batch 1 metrics update 3.680 ms, 7.39 s total +Saving checkpoint at epoch 155 train batch 1 +[ 2023-09-21 10:58:51 ] Completed saving temp checkpoint 491.284 ms, 7.89 s total +[ 2023-09-21 10:58:51 ] Completed replacing temp checkpoint with checkpoint 22.432 ms, 7.91 s total +Epoch [155] :: gen_loss: 0.408, disc_loss: 0.023 +[ 2023-09-21 10:58:51 ] Completed training generator for epoch 155 56.008 ms, 7.96 s total +[ 2023-09-21 10:58:51 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 155 from eval step 0 + +[ 2023-09-21 10:58:52 ] Completed eval batch 0 to device 1,151.396 ms, 1.15 s total +[ 2023-09-21 10:58:52 ] Completed eval batch 0 forward 43.389 ms, 1.19 s total +[ 2023-09-21 10:58:52 ] Completed eval batch 0 recons_loss 0.333 ms, 1.20 s total +[ 2023-09-21 10:58:52 ] Completed eval batch 0 metrics update 0.613 ms, 1.20 s total +Saving checkpoint at epoch 155 val batch 0 +[ 2023-09-21 10:58:52 ] Completed saving temp checkpoint 455.429 ms, 1.65 s total +[ 2023-09-21 10:58:52 ] Completed replacing temp checkpoint with checkpoint 19.801 ms, 1.67 s total +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 11:13:00 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 11:13:00 ] Completed importing Timer 0.030 ms, 0.00 s total +[ 2023-09-21 11:13:05 ] Completed importing everything else 4,714.783 ms, 4.71 s total +| distributed init (rank 1): env:// +| distributed init (rank 3): env:// +| distributed init (rank 2): env:// +| distributed init (rank 4): env:// +| distributed init (rank 0): env:// +| distributed init (rank 5): env:// +[ 2023-09-21 11:13:13 ] Completed preliminaries 7,377.462 ms, 12.09 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 11:13:13 ] Completed build datasets 10.896 ms, 12.10 s total +[ 2023-09-21 11:13:13 ] Completed build samplers 0.049 ms, 12.10 s total +[ 2023-09-21 11:13:13 ] Completed build dataloaders 0.137 ms, 12.10 s total +[ 2023-09-21 11:13:13 ] Completed generator to device 78.193 ms, 12.18 s total +[ 2023-09-21 11:13:13 ] Completed discriminator to device 7.235 ms, 12.19 s total +[ 2023-09-21 11:13:13 ] Completed loss functions 414.811 ms, 12.60 s total +[ 2023-09-21 11:13:13 ] Completed models prepped for distribution 52.126 ms, 12.66 s total +[ 2023-09-21 11:13:13 ] Completed optimizers 0.668 ms, 12.66 s total +[ 2023-09-21 11:13:13 ] Completed grad scalers 0.022 ms, 12.66 s total +[ 2023-09-21 11:13:14 ] Completed checkpoint retrieval 460.843 ms, 13.12 s total + + +EPOCH :: 155 + + +[ 2023-09-21 11:13:14 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 155 from training step 2 + +Epoch [155] :: gen_loss: 0.408, disc_loss: 0.023 +[ 2023-09-21 11:13:14 ] Completed training generator for epoch 155 138.954 ms, 0.14 s total +[ 2023-09-21 11:13:14 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 155 from eval step 1 + +[ 2023-09-21 11:13:15 ] Completed eval batch 1 to device 1,431.437 ms, 1.43 s total +[ 2023-09-21 11:13:16 ] Completed eval batch 1 forward 1,177.239 ms, 2.61 s total +[ 2023-09-21 11:13:16 ] Completed eval batch 1 recons_loss 0.268 ms, 2.61 s total +[ 2023-09-21 11:13:16 ] Completed eval batch 1 metrics update 196.947 ms, 2.81 s total +Saving checkpoint at epoch 155 val batch 1 +Epoch 155 val loss: 0.0614 +[ 2023-09-21 11:13:17 ] Completed saving temp checkpoint 440.641 ms, 3.25 s total +[ 2023-09-21 11:13:17 ] Completed replacing temp checkpoint with checkpoint 141.088 ms, 3.39 s total +[ 2023-09-21 11:13:17 ] Completed evaluating generator for epoch 155 37.005 ms, 3.42 s total + + +EPOCH :: 156 + + +[ 2023-09-21 11:13:17 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 156 from training step 0 + +[ 2023-09-21 11:13:20 ] Completed train batch 0 to device 2,967.470 ms, 2.97 s total +[ 2023-09-21 11:13:20 ] Completed train batch 0 generator forward 39.117 ms, 3.01 s total +[ 2023-09-21 11:13:20 ] Completed train batch 0 generator loss: 1.256 95.929 ms, 3.10 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 11:13:21 ] Completed train batch 0 generator backward 674.508 ms, 3.78 s total +[ 2023-09-21 11:13:21 ] Completed train batch 0 discriminator loss 0.049 57.286 ms, 3.83 s total +[ 2023-09-21 11:13:21 ] Completed train batch 0 discriminator backward 115.254 ms, 3.95 s total +Epoch [156] Step [0/2], gen_loss: 0.414, disc_loss: 0.017 +[ 2023-09-21 11:13:21 ] Completed train batch 0 metrics update 5.312 ms, 3.95 s total +Saving checkpoint at epoch 156 train batch 0 +[ 2023-09-21 11:13:21 ] Completed saving temp checkpoint 417.611 ms, 4.37 s total +[ 2023-09-21 11:13:22 ] Completed replacing temp checkpoint with checkpoint 25.522 ms, 4.40 s total +[ 2023-09-21 11:13:23 ] Completed train batch 1 to device 1,352.741 ms, 5.75 s total +[ 2023-09-21 11:13:23 ] Completed train batch 1 generator forward 22.218 ms, 5.77 s total +[ 2023-09-21 11:13:23 ] Completed train batch 1 generator loss: 1.251 62.264 ms, 5.84 s total +[ 2023-09-21 11:13:24 ] Completed train batch 1 generator backward 1,385.862 ms, 7.22 s total +[ 2023-09-21 11:13:24 ] Completed train batch 1 discriminator loss 0.045 60.205 ms, 7.28 s total +[ 2023-09-21 11:13:24 ] Completed train batch 1 discriminator backward 61.517 ms, 7.34 s total +Epoch [156] Step [1/2], gen_loss: 0.426, disc_loss: 0.015 +[ 2023-09-21 11:13:24 ] Completed train batch 1 metrics update 1.956 ms, 7.34 s total +Saving checkpoint at epoch 156 train batch 1 +[ 2023-09-21 11:13:25 ] Completed saving temp checkpoint 436.508 ms, 7.78 s total +[ 2023-09-21 11:13:25 ] Completed replacing temp checkpoint with checkpoint 20.627 ms, 7.80 s total +Epoch [156] :: gen_loss: 0.420, disc_loss: 0.016 +[ 2023-09-21 11:13:25 ] Completed training generator for epoch 156 51.867 ms, 7.85 s total +[ 2023-09-21 11:13:25 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 156 from eval step 0 + +[ 2023-09-21 11:13:26 ] Completed eval batch 0 to device 1,145.493 ms, 1.15 s total +[ 2023-09-21 11:13:26 ] Completed eval batch 0 forward 31.748 ms, 1.18 s total +[ 2023-09-21 11:13:26 ] Completed eval batch 0 recons_loss 0.402 ms, 1.18 s total +[ 2023-09-21 11:13:26 ] Completed eval batch 0 metrics update 0.738 ms, 1.18 s total +Saving checkpoint at epoch 156 val batch 0 +[ 2023-09-21 11:13:27 ] Completed saving temp checkpoint 367.484 ms, 1.55 s total +[ 2023-09-21 11:13:27 ] Completed replacing temp checkpoint with checkpoint 22.758 ms, 1.57 s total +[ 2023-09-21 11:13:27 ] Completed eval batch 1 to device 464.944 ms, 2.03 s total +[ 2023-09-21 11:13:27 ] Completed eval batch 1 forward 23.428 ms, 2.06 s total +[ 2023-09-21 11:13:27 ] Completed eval batch 1 recons_loss 0.329 ms, 2.06 s total +[ 2023-09-21 11:13:27 ] Completed eval batch 1 metrics update 273.920 ms, 2.33 s total +Saving checkpoint at epoch 156 val batch 1 +Epoch 156 val loss: 0.0636 +[ 2023-09-21 11:13:28 ] Completed saving temp checkpoint 420.426 ms, 2.75 s total +[ 2023-09-21 11:13:28 ] Completed replacing temp checkpoint with checkpoint 16.221 ms, 2.77 s total +[ 2023-09-21 11:13:28 ] Completed evaluating generator for epoch 156 65.548 ms, 2.83 s total + + +EPOCH :: 157 + + +[ 2023-09-21 11:13:28 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 157 from training step 0 + +[ 2023-09-21 11:13:31 ] Completed train batch 0 to device 3,065.257 ms, 3.07 s total +[ 2023-09-21 11:13:31 ] Completed train batch 0 generator forward 43.861 ms, 3.11 s total +[ 2023-09-21 11:13:31 ] Completed train batch 0 generator loss: 1.225 67.641 ms, 3.18 s total +[ 2023-09-21 11:13:32 ] Completed train batch 0 generator backward 749.137 ms, 3.93 s total +[ 2023-09-21 11:13:32 ] Completed train batch 0 discriminator loss 0.046 57.843 ms, 3.98 s total +[ 2023-09-21 11:13:32 ] Completed train batch 0 discriminator backward 97.532 ms, 4.08 s total +Epoch [157] Step [0/2], gen_loss: 0.417, disc_loss: 0.016 +[ 2023-09-21 11:13:32 ] Completed train batch 0 metrics update 6.089 ms, 4.09 s total +Saving checkpoint at epoch 157 train batch 0 +[ 2023-09-21 11:13:32 ] Completed saving temp checkpoint 492.651 ms, 4.58 s total +[ 2023-09-21 11:13:32 ] Completed replacing temp checkpoint with checkpoint 21.601 ms, 4.60 s total +[ 2023-09-21 11:13:34 ] Completed train batch 1 to device 1,245.935 ms, 5.85 s total +[ 2023-09-21 11:13:34 ] Completed train batch 1 generator forward 40.384 ms, 5.89 s total +[ 2023-09-21 11:13:34 ] Completed train batch 1 generator loss: 1.240 68.391 ms, 5.96 s total +[ 2023-09-21 11:13:35 ] Completed train batch 1 generator backward 1,642.776 ms, 7.60 s total +[ 2023-09-21 11:13:35 ] Completed train batch 1 discriminator loss 0.046 56.871 ms, 7.66 s total +[ 2023-09-21 11:13:36 ] Completed train batch 1 discriminator backward 66.908 ms, 7.72 s total +Epoch [157] Step [1/2], gen_loss: 0.418, disc_loss: 0.016 +[ 2023-09-21 11:13:36 ] Completed train batch 1 metrics update 0.775 ms, 7.72 s total +Saving checkpoint at epoch 157 train batch 1 +[ 2023-09-21 11:13:36 ] Completed saving temp checkpoint 416.610 ms, 8.14 s total +[ 2023-09-21 11:13:36 ] Completed replacing temp checkpoint with checkpoint 22.190 ms, 8.16 s total +Epoch [157] :: gen_loss: 0.418, disc_loss: 0.016 +[ 2023-09-21 11:13:36 ] Completed training generator for epoch 157 51.849 ms, 8.21 s total +[ 2023-09-21 11:13:36 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 157 from eval step 0 + +[ 2023-09-21 11:13:37 ] Completed eval batch 0 to device 1,145.786 ms, 1.15 s total +[ 2023-09-21 11:13:37 ] Completed eval batch 0 forward 27.276 ms, 1.17 s total +[ 2023-09-21 11:13:37 ] Completed eval batch 0 recons_loss 0.278 ms, 1.17 s total +[ 2023-09-21 11:13:37 ] Completed eval batch 0 metrics update 0.567 ms, 1.17 s total +Saving checkpoint at epoch 157 val batch 0 +[ 2023-09-21 11:13:38 ] Completed saving temp checkpoint 429.322 ms, 1.60 s total +[ 2023-09-21 11:13:38 ] Completed replacing temp checkpoint with checkpoint 30.365 ms, 1.63 s total +[ 2023-09-21 11:13:38 ] Completed eval batch 1 to device 545.660 ms, 2.18 s total +[ 2023-09-21 11:13:38 ] Completed eval batch 1 forward 22.180 ms, 2.20 s total +[ 2023-09-21 11:13:38 ] Completed eval batch 1 recons_loss 0.414 ms, 2.20 s total +[ 2023-09-21 11:13:38 ] Completed eval batch 1 metrics update 164.753 ms, 2.37 s total +Saving checkpoint at epoch 157 val batch 1 +Epoch 157 val loss: 0.0581 +[ 2023-09-21 11:13:39 ] Completed saving temp checkpoint 534.373 ms, 2.90 s total +[ 2023-09-21 11:13:39 ] Completed replacing temp checkpoint with checkpoint 31.559 ms, 2.93 s total +[ 2023-09-21 11:13:39 ] Completed evaluating generator for epoch 157 74.742 ms, 3.01 s total + + +EPOCH :: 158 + + +[ 2023-09-21 11:13:39 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 158 from training step 0 + +[ 2023-09-21 11:13:42 ] Completed train batch 0 to device 3,127.317 ms, 3.13 s total +[ 2023-09-21 11:13:42 ] Completed train batch 0 generator forward 26.959 ms, 3.15 s total +[ 2023-09-21 11:13:42 ] Completed train batch 0 generator loss: 1.222 66.715 ms, 3.22 s total +[ 2023-09-21 11:13:43 ] Completed train batch 0 generator backward 645.169 ms, 3.87 s total +[ 2023-09-21 11:13:43 ] Completed train batch 0 discriminator loss 0.048 61.805 ms, 3.93 s total +[ 2023-09-21 11:13:43 ] Completed train batch 0 discriminator backward 99.637 ms, 4.03 s total +Epoch [158] Step [0/2], gen_loss: 0.411, disc_loss: 0.016 +[ 2023-09-21 11:13:43 ] Completed train batch 0 metrics update 4.100 ms, 4.03 s total +Saving checkpoint at epoch 158 train batch 0 +[ 2023-09-21 11:13:43 ] Completed saving temp checkpoint 452.702 ms, 4.48 s total +[ 2023-09-21 11:13:44 ] Completed replacing temp checkpoint with checkpoint 23.390 ms, 4.51 s total +[ 2023-09-21 11:13:45 ] Completed train batch 1 to device 1,547.690 ms, 6.06 s total +[ 2023-09-21 11:13:45 ] Completed train batch 1 generator forward 22.277 ms, 6.08 s total +[ 2023-09-21 11:13:45 ] Completed train batch 1 generator loss: 1.224 63.822 ms, 6.14 s total +[ 2023-09-21 11:13:47 ] Completed train batch 1 generator backward 1,405.915 ms, 7.55 s total +[ 2023-09-21 11:13:47 ] Completed train batch 1 discriminator loss 0.048 56.513 ms, 7.60 s total +[ 2023-09-21 11:13:47 ] Completed train batch 1 discriminator backward 73.274 ms, 7.68 s total +Epoch [158] Step [1/2], gen_loss: 0.418, disc_loss: 0.016 +[ 2023-09-21 11:13:47 ] Completed train batch 1 metrics update 5.079 ms, 7.68 s total +Saving checkpoint at epoch 158 train batch 1 +[ 2023-09-21 11:13:47 ] Completed saving temp checkpoint 476.783 ms, 8.16 s total +[ 2023-09-21 11:13:47 ] Completed replacing temp checkpoint with checkpoint 22.559 ms, 8.18 s total +Epoch [158] :: gen_loss: 0.415, disc_loss: 0.016 +[ 2023-09-21 11:13:47 ] Completed training generator for epoch 158 60.523 ms, 8.24 s total +[ 2023-09-21 11:13:47 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 158 from eval step 0 + +[ 2023-09-21 11:13:48 ] Completed eval batch 0 to device 1,150.040 ms, 1.15 s total +[ 2023-09-21 11:13:48 ] Completed eval batch 0 forward 29.131 ms, 1.18 s total +[ 2023-09-21 11:13:48 ] Completed eval batch 0 recons_loss 0.376 ms, 1.18 s total +[ 2023-09-21 11:13:48 ] Completed eval batch 0 metrics update 0.644 ms, 1.18 s total +Saving checkpoint at epoch 158 val batch 0 +[ 2023-09-21 11:13:49 ] Completed saving temp checkpoint 423.280 ms, 1.60 s total +[ 2023-09-21 11:13:49 ] Completed replacing temp checkpoint with checkpoint 31.289 ms, 1.63 s total +[ 2023-09-21 11:13:49 ] Completed eval batch 1 to device 464.634 ms, 2.10 s total +[ 2023-09-21 11:13:49 ] Completed eval batch 1 forward 21.772 ms, 2.12 s total +[ 2023-09-21 11:13:49 ] Completed eval batch 1 recons_loss 0.348 ms, 2.12 s total +[ 2023-09-21 11:13:50 ] Completed eval batch 1 metrics update 147.495 ms, 2.27 s total +Saving checkpoint at epoch 158 val batch 1 +Epoch 158 val loss: 0.0565 +[ 2023-09-21 11:13:50 ] Completed saving temp checkpoint 499.477 ms, 2.77 s total +[ 2023-09-21 11:13:50 ] Completed replacing temp checkpoint with checkpoint 25.554 ms, 2.79 s total +[ 2023-09-21 11:13:50 ] Completed evaluating generator for epoch 158 50.411 ms, 2.84 s total + + +EPOCH :: 159 + + +[ 2023-09-21 11:13:50 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 159 from training step 0 + +[ 2023-09-21 11:13:53 ] Completed train batch 0 to device 3,180.885 ms, 3.18 s total +[ 2023-09-21 11:13:53 ] Completed train batch 0 generator forward 32.338 ms, 3.21 s total +[ 2023-09-21 11:13:53 ] Completed train batch 0 generator loss: 1.182 70.856 ms, 3.28 s total +[ 2023-09-21 11:13:54 ] Completed train batch 0 generator backward 523.307 ms, 3.81 s total +[ 2023-09-21 11:13:54 ] Completed train batch 0 discriminator loss 0.046 56.068 ms, 3.86 s total +[ 2023-09-21 11:13:54 ] Completed train batch 0 discriminator backward 94.783 ms, 3.96 s total +Epoch [159] Step [0/2], gen_loss: 0.416, disc_loss: 0.015 +[ 2023-09-21 11:13:54 ] Completed train batch 0 metrics update 21.699 ms, 3.98 s total +Saving checkpoint at epoch 159 train batch 0 +[ 2023-09-21 11:13:55 ] Completed saving temp checkpoint 460.045 ms, 4.44 s total +[ 2023-09-21 11:13:55 ] Completed replacing temp checkpoint with checkpoint 31.156 ms, 4.47 s total +[ 2023-09-21 11:13:56 ] Completed train batch 1 to device 1,678.937 ms, 6.15 s total +[ 2023-09-21 11:13:56 ] Completed train batch 1 generator forward 40.275 ms, 6.19 s total +[ 2023-09-21 11:13:56 ] Completed train batch 1 generator loss: 1.306 63.576 ms, 6.25 s total +[ 2023-09-21 11:13:57 ] Completed train batch 1 generator backward 1,045.973 ms, 7.30 s total +[ 2023-09-21 11:13:57 ] Completed train batch 1 discriminator loss 0.048 57.006 ms, 7.36 s total +[ 2023-09-21 11:13:58 ] Completed train batch 1 discriminator backward 75.803 ms, 7.43 s total +Epoch [159] Step [1/2], gen_loss: 0.424, disc_loss: 0.016 +[ 2023-09-21 11:13:58 ] Completed train batch 1 metrics update 5.617 ms, 7.44 s total +Saving checkpoint at epoch 159 train batch 1 +[ 2023-09-21 11:13:58 ] Completed saving temp checkpoint 464.381 ms, 7.90 s total +[ 2023-09-21 11:13:58 ] Completed replacing temp checkpoint with checkpoint 21.634 ms, 7.92 s total +Epoch [159] :: gen_loss: 0.420, disc_loss: 0.015 +[ 2023-09-21 11:13:58 ] Completed training generator for epoch 159 52.964 ms, 7.98 s total +[ 2023-09-21 11:13:58 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 159 from eval step 0 + +[ 2023-09-21 11:13:59 ] Completed eval batch 0 to device 1,108.262 ms, 1.11 s total +[ 2023-09-21 11:13:59 ] Completed eval batch 0 forward 32.259 ms, 1.14 s total +[ 2023-09-21 11:13:59 ] Completed eval batch 0 recons_loss 0.368 ms, 1.14 s total +[ 2023-09-21 11:13:59 ] Completed eval batch 0 metrics update 0.692 ms, 1.14 s total +Saving checkpoint at epoch 159 val batch 0 +[ 2023-09-21 11:14:00 ] Completed saving temp checkpoint 423.684 ms, 1.57 s total +[ 2023-09-21 11:14:00 ] Completed replacing temp checkpoint with checkpoint 24.927 ms, 1.59 s total +[ 2023-09-21 11:14:00 ] Completed eval batch 1 to device 382.337 ms, 1.97 s total +[ 2023-09-21 11:14:00 ] Completed eval batch 1 forward 21.565 ms, 1.99 s total +[ 2023-09-21 11:14:00 ] Completed eval batch 1 recons_loss 0.332 ms, 1.99 s total +[ 2023-09-21 11:14:00 ] Completed eval batch 1 metrics update 251.493 ms, 2.25 s total +Saving checkpoint at epoch 159 val batch 1 +Epoch 159 val loss: 0.0622 +[ 2023-09-21 11:14:01 ] Completed saving temp checkpoint 474.632 ms, 2.72 s total +[ 2023-09-21 11:14:01 ] Completed replacing temp checkpoint with checkpoint 23.104 ms, 2.74 s total +[ 2023-09-21 11:14:01 ] Completed evaluating generator for epoch 159 55.994 ms, 2.80 s total + + +EPOCH :: 160 + + +[ 2023-09-21 11:14:01 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 160 from training step 0 + +[ 2023-09-21 11:14:04 ] Completed train batch 0 to device 3,060.703 ms, 3.06 s total +[ 2023-09-21 11:14:04 ] Completed train batch 0 generator forward 37.316 ms, 3.10 s total +[ 2023-09-21 11:14:04 ] Completed train batch 0 generator loss: 1.248 66.920 ms, 3.16 s total +[ 2023-09-21 11:14:05 ] Completed train batch 0 generator backward 632.178 ms, 3.80 s total +[ 2023-09-21 11:14:05 ] Completed train batch 0 discriminator loss 0.048 64.488 ms, 3.86 s total +[ 2023-09-21 11:14:05 ] Completed train batch 0 discriminator backward 78.980 ms, 3.94 s total +Epoch [160] Step [0/2], gen_loss: 0.408, disc_loss: 0.016 +[ 2023-09-21 11:14:05 ] Completed train batch 0 metrics update 17.878 ms, 3.96 s total +Saving checkpoint at epoch 160 train batch 0 +[ 2023-09-21 11:14:05 ] Completed saving temp checkpoint 464.164 ms, 4.42 s total +[ 2023-09-21 11:14:05 ] Completed replacing temp checkpoint with checkpoint 34.739 ms, 4.46 s total +[ 2023-09-21 11:14:07 ] Completed train batch 1 to device 1,437.045 ms, 5.89 s total +[ 2023-09-21 11:14:07 ] Completed train batch 1 generator forward 22.302 ms, 5.92 s total +[ 2023-09-21 11:14:07 ] Completed train batch 1 generator loss: 1.218 63.208 ms, 5.98 s total +[ 2023-09-21 11:14:08 ] Completed train batch 1 generator backward 1,318.264 ms, 7.30 s total +[ 2023-09-21 11:14:08 ] Completed train batch 1 discriminator loss 0.048 56.191 ms, 7.35 s total +[ 2023-09-21 11:14:08 ] Completed train batch 1 discriminator backward 73.177 ms, 7.43 s total +Epoch [160] Step [1/2], gen_loss: 0.419, disc_loss: 0.016 +[ 2023-09-21 11:14:08 ] Completed train batch 1 metrics update 0.561 ms, 7.43 s total +Saving checkpoint at epoch 160 train batch 1 +[ 2023-09-21 11:14:09 ] Completed saving temp checkpoint 477.885 ms, 7.91 s total +[ 2023-09-21 11:14:09 ] Completed replacing temp checkpoint with checkpoint 23.729 ms, 7.93 s total +Epoch [160] :: gen_loss: 0.414, disc_loss: 0.016 +[ 2023-09-21 11:14:09 ] Completed training generator for epoch 160 50.791 ms, 7.98 s total +[ 2023-09-21 11:14:09 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 160 from eval step 0 + +[ 2023-09-21 11:14:10 ] Completed eval batch 0 to device 1,117.640 ms, 1.12 s total +[ 2023-09-21 11:14:10 ] Completed eval batch 0 forward 38.242 ms, 1.16 s total +[ 2023-09-21 11:14:10 ] Completed eval batch 0 recons_loss 0.330 ms, 1.16 s total +[ 2023-09-21 11:14:10 ] Completed eval batch 0 metrics update 0.689 ms, 1.16 s total +Saving checkpoint at epoch 160 val batch 0 +[ 2023-09-21 11:14:10 ] Completed saving temp checkpoint 398.881 ms, 1.56 s total +[ 2023-09-21 11:14:10 ] Completed replacing temp checkpoint with checkpoint 31.499 ms, 1.59 s total +[ 2023-09-21 11:14:11 ] Completed eval batch 1 to device 441.087 ms, 2.03 s total +[ 2023-09-21 11:14:11 ] Completed eval batch 1 forward 29.332 ms, 2.06 s total +[ 2023-09-21 11:14:11 ] Completed eval batch 1 recons_loss 0.328 ms, 2.06 s total +[ 2023-09-21 11:14:11 ] Completed eval batch 1 metrics update 176.758 ms, 2.23 s total +Saving checkpoint at epoch 160 val batch 1 +Epoch 160 val loss: 0.0651 +[ 2023-09-21 11:14:12 ] Completed saving temp checkpoint 447.384 ms, 2.68 s total +[ 2023-09-21 11:14:12 ] Completed replacing temp checkpoint with checkpoint 29.461 ms, 2.71 s total +[ 2023-09-21 11:14:12 ] Completed evaluating generator for epoch 160 64.309 ms, 2.78 s total + + +EPOCH :: 161 + + +[ 2023-09-21 11:14:12 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 161 from training step 0 + +[ 2023-09-21 11:14:15 ] Completed train batch 0 to device 2,987.119 ms, 2.99 s total +[ 2023-09-21 11:14:15 ] Completed train batch 0 generator forward 44.226 ms, 3.03 s total +[ 2023-09-21 11:14:15 ] Completed train batch 0 generator loss: 1.235 66.261 ms, 3.10 s total +[ 2023-09-21 11:14:15 ] Completed train batch 0 generator backward 703.737 ms, 3.80 s total +[ 2023-09-21 11:14:15 ] Completed train batch 0 discriminator loss 0.046 57.034 ms, 3.86 s total +[ 2023-09-21 11:14:16 ] Completed train batch 0 discriminator backward 90.117 ms, 3.95 s total +Epoch [161] Step [0/2], gen_loss: 0.416, disc_loss: 0.015 +[ 2023-09-21 11:14:16 ] Completed train batch 0 metrics update 4.990 ms, 3.95 s total +Saving checkpoint at epoch 161 train batch 0 +[ 2023-09-21 11:14:16 ] Completed saving temp checkpoint 463.270 ms, 4.42 s total +[ 2023-09-21 11:14:16 ] Completed replacing temp checkpoint with checkpoint 32.384 ms, 4.45 s total +[ 2023-09-21 11:14:17 ] Completed train batch 1 to device 1,415.060 ms, 5.86 s total +[ 2023-09-21 11:14:18 ] Completed train batch 1 generator forward 22.308 ms, 5.89 s total +[ 2023-09-21 11:14:18 ] Completed train batch 1 generator loss: 1.281 64.160 ms, 5.95 s total +[ 2023-09-21 11:14:19 ] Completed train batch 1 generator backward 1,317.523 ms, 7.27 s total +[ 2023-09-21 11:14:19 ] Completed train batch 1 discriminator loss 0.043 56.366 ms, 7.32 s total +[ 2023-09-21 11:14:19 ] Completed train batch 1 discriminator backward 71.954 ms, 7.40 s total +Epoch [161] Step [1/2], gen_loss: 0.413, disc_loss: 0.015 +[ 2023-09-21 11:14:19 ] Completed train batch 1 metrics update 2.673 ms, 7.40 s total +Saving checkpoint at epoch 161 train batch 1 +[ 2023-09-21 11:14:19 ] Completed saving temp checkpoint 461.537 ms, 7.86 s total +[ 2023-09-21 11:14:20 ] Completed replacing temp checkpoint with checkpoint 28.838 ms, 7.89 s total +Epoch [161] :: gen_loss: 0.414, disc_loss: 0.015 +[ 2023-09-21 11:14:20 ] Completed training generator for epoch 161 61.098 ms, 7.95 s total +[ 2023-09-21 11:14:20 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 161 from eval step 0 + +[ 2023-09-21 11:14:21 ] Completed eval batch 0 to device 1,060.488 ms, 1.06 s total +[ 2023-09-21 11:14:21 ] Completed eval batch 0 forward 24.724 ms, 1.09 s total +[ 2023-09-21 11:14:21 ] Completed eval batch 0 recons_loss 0.299 ms, 1.09 s total +[ 2023-09-21 11:14:21 ] Completed eval batch 0 metrics update 0.567 ms, 1.09 s total +Saving checkpoint at epoch 161 val batch 0 +[ 2023-09-21 11:14:21 ] Completed saving temp checkpoint 428.982 ms, 1.52 s total +[ 2023-09-21 11:14:21 ] Completed replacing temp checkpoint with checkpoint 23.686 ms, 1.54 s total +[ 2023-09-21 11:14:22 ] Completed eval batch 1 to device 524.977 ms, 2.06 s total +[ 2023-09-21 11:14:22 ] Completed eval batch 1 forward 21.602 ms, 2.09 s total +[ 2023-09-21 11:14:22 ] Completed eval batch 1 recons_loss 0.336 ms, 2.09 s total +[ 2023-09-21 11:14:22 ] Completed eval batch 1 metrics update 136.166 ms, 2.22 s total +Saving checkpoint at epoch 161 val batch 1 +Epoch 161 val loss: 0.0633 +[ 2023-09-21 11:14:22 ] Completed saving temp checkpoint 474.875 ms, 2.70 s total +[ 2023-09-21 11:14:22 ] Completed replacing temp checkpoint with checkpoint 25.641 ms, 2.72 s total +[ 2023-09-21 11:14:22 ] Completed evaluating generator for epoch 161 51.258 ms, 2.77 s total + + +EPOCH :: 162 + + +[ 2023-09-21 11:14:22 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 162 from training step 0 + +[ 2023-09-21 11:14:25 ] Completed train batch 0 to device 3,020.641 ms, 3.02 s total +[ 2023-09-21 11:14:25 ] Completed train batch 0 generator forward 29.925 ms, 3.05 s total +[ 2023-09-21 11:14:25 ] Completed train batch 0 generator loss: 1.230 66.006 ms, 3.12 s total +[ 2023-09-21 11:14:26 ] Completed train batch 0 generator backward 616.488 ms, 3.73 s total +[ 2023-09-21 11:14:26 ] Completed train batch 0 discriminator loss 0.043 57.633 ms, 3.79 s total +[ 2023-09-21 11:14:26 ] Completed train batch 0 discriminator backward 103.355 ms, 3.89 s total +Epoch [162] Step [0/2], gen_loss: 0.423, disc_loss: 0.015 +[ 2023-09-21 11:14:26 ] Completed train batch 0 metrics update 3.208 ms, 3.90 s total +Saving checkpoint at epoch 162 train batch 0 +[ 2023-09-21 11:14:27 ] Completed saving temp checkpoint 423.170 ms, 4.32 s total +[ 2023-09-21 11:14:27 ] Completed replacing temp checkpoint with checkpoint 24.386 ms, 4.34 s total +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 11:25:14 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 11:25:14 ] Completed importing Timer 0.024 ms, 0.00 s total +[ 2023-09-21 11:25:18 ] Completed importing everything else 4,521.837 ms, 4.52 s total +| distributed init (rank 1): env:// +| distributed init (rank 2): env:// +| distributed init (rank 3): env:// +| distributed init (rank 5): env:// +| distributed init (rank 4): env:// +| distributed init (rank 0): env:// +[ 2023-09-21 11:25:27 ] Completed preliminaries 8,082.521 ms, 12.60 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 11:25:27 ] Completed build datasets 10.598 ms, 12.61 s total +[ 2023-09-21 11:25:27 ] Completed build samplers 0.043 ms, 12.62 s total +[ 2023-09-21 11:25:27 ] Completed build dataloaders 0.135 ms, 12.62 s total +[ 2023-09-21 11:25:27 ] Completed generator to device 78.406 ms, 12.69 s total +[ 2023-09-21 11:25:27 ] Completed discriminator to device 7.022 ms, 12.70 s total +[ 2023-09-21 11:25:27 ] Completed loss functions 392.663 ms, 13.09 s total +[ 2023-09-21 11:25:27 ] Completed models prepped for distribution 83.490 ms, 13.18 s total +[ 2023-09-21 11:25:27 ] Completed optimizers 0.673 ms, 13.18 s total +[ 2023-09-21 11:25:27 ] Completed grad scalers 0.027 ms, 13.18 s total +[ 2023-09-21 11:25:28 ] Completed checkpoint retrieval 476.859 ms, 13.65 s total + + +EPOCH :: 162 + + +[ 2023-09-21 11:25:28 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 162 from training step 1 + +[ 2023-09-21 11:25:30 ] Completed train batch 1 to device 2,893.362 ms, 2.89 s total +[ 2023-09-21 11:25:32 ] Completed train batch 1 generator forward 1,178.150 ms, 4.07 s total +[ 2023-09-21 11:25:32 ] Completed train batch 1 generator loss: 1.224 76.550 ms, 4.15 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 11:25:33 ] Completed train batch 1 generator backward 1,781.445 ms, 5.93 s total +[ 2023-09-21 11:25:34 ] Completed train batch 1 discriminator loss 0.045 56.100 ms, 5.99 s total +[ 2023-09-21 11:25:34 ] Completed train batch 1 discriminator backward 79.203 ms, 6.06 s total +Epoch [162] Step [1/2], gen_loss: 0.424, disc_loss: 0.015 +[ 2023-09-21 11:25:34 ] Completed train batch 1 metrics update 1.753 ms, 6.07 s total +Saving checkpoint at epoch 162 train batch 1 +[ 2023-09-21 11:25:34 ] Completed saving temp checkpoint 398.223 ms, 6.46 s total +[ 2023-09-21 11:25:34 ] Completed replacing temp checkpoint with checkpoint 114.188 ms, 6.58 s total +Epoch [162] :: gen_loss: 0.423, disc_loss: 0.015 +[ 2023-09-21 11:25:34 ] Completed training generator for epoch 162 44.197 ms, 6.62 s total +[ 2023-09-21 11:25:34 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 162 from eval step 0 + +[ 2023-09-21 11:25:35 ] Completed eval batch 0 to device 1,129.918 ms, 1.13 s total +[ 2023-09-21 11:25:35 ] Completed eval batch 0 forward 31.245 ms, 1.16 s total +[ 2023-09-21 11:25:35 ] Completed eval batch 0 recons_loss 0.390 ms, 1.16 s total +[ 2023-09-21 11:25:35 ] Completed eval batch 0 metrics update 0.646 ms, 1.16 s total +Saving checkpoint at epoch 162 val batch 0 +[ 2023-09-21 11:25:36 ] Completed saving temp checkpoint 444.320 ms, 1.61 s total +[ 2023-09-21 11:25:36 ] Completed replacing temp checkpoint with checkpoint 31.759 ms, 1.64 s total +[ 2023-09-21 11:25:36 ] Completed eval batch 1 to device 410.344 ms, 2.05 s total +[ 2023-09-21 11:25:36 ] Completed eval batch 1 forward 28.498 ms, 2.08 s total +[ 2023-09-21 11:25:36 ] Completed eval batch 1 recons_loss 0.622 ms, 2.08 s total +[ 2023-09-21 11:25:36 ] Completed eval batch 1 metrics update 150.208 ms, 2.23 s total +Saving checkpoint at epoch 162 val batch 1 +Epoch 162 val loss: 0.0615 +[ 2023-09-21 11:25:37 ] Completed saving temp checkpoint 421.220 ms, 2.65 s total +[ 2023-09-21 11:25:37 ] Completed replacing temp checkpoint with checkpoint 25.333 ms, 2.67 s total +[ 2023-09-21 11:25:37 ] Completed evaluating generator for epoch 162 57.587 ms, 2.73 s total + + +EPOCH :: 163 + + +[ 2023-09-21 11:25:37 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 163 from training step 0 + +[ 2023-09-21 11:25:40 ] Completed train batch 0 to device 3,079.723 ms, 3.08 s total +[ 2023-09-21 11:25:40 ] Completed train batch 0 generator forward 28.932 ms, 3.11 s total +[ 2023-09-21 11:25:40 ] Completed train batch 0 generator loss: 1.358 65.699 ms, 3.17 s total +[ 2023-09-21 11:25:41 ] Completed train batch 0 generator backward 689.187 ms, 3.86 s total +[ 2023-09-21 11:25:41 ] Completed train batch 0 discriminator loss 0.049 57.758 ms, 3.92 s total +[ 2023-09-21 11:25:41 ] Completed train batch 0 discriminator backward 95.933 ms, 4.02 s total +Epoch [163] Step [0/2], gen_loss: 0.422, disc_loss: 0.016 +[ 2023-09-21 11:25:41 ] Completed train batch 0 metrics update 7.342 ms, 4.02 s total +Saving checkpoint at epoch 163 train batch 0 +[ 2023-09-21 11:25:41 ] Completed saving temp checkpoint 420.361 ms, 4.44 s total +[ 2023-09-21 11:25:41 ] Completed replacing temp checkpoint with checkpoint 31.643 ms, 4.48 s total +[ 2023-09-21 11:25:43 ] Completed train batch 1 to device 1,516.230 ms, 5.99 s total +[ 2023-09-21 11:25:43 ] Completed train batch 1 generator forward 41.037 ms, 6.03 s total +[ 2023-09-21 11:25:43 ] Completed train batch 1 generator loss: 1.251 65.003 ms, 6.10 s total +[ 2023-09-21 11:25:44 ] Completed train batch 1 generator backward 925.214 ms, 7.02 s total +[ 2023-09-21 11:25:44 ] Completed train batch 1 discriminator loss 0.045 56.650 ms, 7.08 s total +[ 2023-09-21 11:25:44 ] Completed train batch 1 discriminator backward 73.526 ms, 7.15 s total +Epoch [163] Step [1/2], gen_loss: 0.408, disc_loss: 0.015 +[ 2023-09-21 11:25:44 ] Completed train batch 1 metrics update 1.049 ms, 7.16 s total +Saving checkpoint at epoch 163 train batch 1 +[ 2023-09-21 11:25:45 ] Completed saving temp checkpoint 449.425 ms, 7.60 s total +[ 2023-09-21 11:25:45 ] Completed replacing temp checkpoint with checkpoint 21.801 ms, 7.63 s total +Epoch [163] :: gen_loss: 0.415, disc_loss: 0.016 +[ 2023-09-21 11:25:45 ] Completed training generator for epoch 163 53.046 ms, 7.68 s total +[ 2023-09-21 11:25:45 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 163 from eval step 0 + +[ 2023-09-21 11:25:46 ] Completed eval batch 0 to device 1,152.409 ms, 1.15 s total +[ 2023-09-21 11:25:46 ] Completed eval batch 0 forward 31.038 ms, 1.18 s total +[ 2023-09-21 11:25:46 ] Completed eval batch 0 recons_loss 0.398 ms, 1.18 s total +[ 2023-09-21 11:25:46 ] Completed eval batch 0 metrics update 0.740 ms, 1.18 s total +Saving checkpoint at epoch 163 val batch 0 +[ 2023-09-21 11:25:46 ] Completed saving temp checkpoint 445.603 ms, 1.63 s total +[ 2023-09-21 11:25:46 ] Completed replacing temp checkpoint with checkpoint 31.427 ms, 1.66 s total +[ 2023-09-21 11:25:47 ] Completed eval batch 1 to device 366.416 ms, 2.03 s total +[ 2023-09-21 11:25:47 ] Completed eval batch 1 forward 23.067 ms, 2.05 s total +[ 2023-09-21 11:25:47 ] Completed eval batch 1 recons_loss 0.340 ms, 2.05 s total +[ 2023-09-21 11:25:47 ] Completed eval batch 1 metrics update 223.272 ms, 2.27 s total +Saving checkpoint at epoch 163 val batch 1 +Epoch 163 val loss: 0.0557 +[ 2023-09-21 11:25:47 ] Completed saving temp checkpoint 454.558 ms, 2.73 s total +[ 2023-09-21 11:25:47 ] Completed replacing temp checkpoint with checkpoint 25.107 ms, 2.75 s total +[ 2023-09-21 11:25:47 ] Completed evaluating generator for epoch 163 56.003 ms, 2.81 s total + + +EPOCH :: 164 + + +[ 2023-09-21 11:25:47 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 164 from training step 0 + +[ 2023-09-21 11:25:51 ] Completed train batch 0 to device 3,156.906 ms, 3.16 s total +[ 2023-09-21 11:25:51 ] Completed train batch 0 generator forward 26.668 ms, 3.18 s total +[ 2023-09-21 11:25:51 ] Completed train batch 0 generator loss: 1.236 66.851 ms, 3.25 s total +[ 2023-09-21 11:25:51 ] Completed train batch 0 generator backward 558.601 ms, 3.81 s total +[ 2023-09-21 11:25:51 ] Completed train batch 0 discriminator loss 0.047 61.505 ms, 3.87 s total +[ 2023-09-21 11:25:51 ] Completed train batch 0 discriminator backward 96.898 ms, 3.97 s total +Epoch [164] Step [0/2], gen_loss: 0.418, disc_loss: 0.015 +[ 2023-09-21 11:25:51 ] Completed train batch 0 metrics update 7.771 ms, 3.98 s total +Saving checkpoint at epoch 164 train batch 0 +[ 2023-09-21 11:25:52 ] Completed saving temp checkpoint 442.070 ms, 4.42 s total +[ 2023-09-21 11:25:52 ] Completed replacing temp checkpoint with checkpoint 22.905 ms, 4.44 s total +[ 2023-09-21 11:25:53 ] Completed train batch 1 to device 1,620.735 ms, 6.06 s total +[ 2023-09-21 11:25:53 ] Completed train batch 1 generator forward 22.947 ms, 6.08 s total +[ 2023-09-21 11:25:54 ] Completed train batch 1 generator loss: 1.196 65.168 ms, 6.15 s total +[ 2023-09-21 11:25:55 ] Completed train batch 1 generator backward 1,123.995 ms, 7.27 s total +[ 2023-09-21 11:25:55 ] Completed train batch 1 discriminator loss 0.045 56.500 ms, 7.33 s total +[ 2023-09-21 11:25:55 ] Completed train batch 1 discriminator backward 72.897 ms, 7.40 s total +Epoch [164] Step [1/2], gen_loss: 0.411, disc_loss: 0.015 +[ 2023-09-21 11:25:55 ] Completed train batch 1 metrics update 5.189 ms, 7.41 s total +Saving checkpoint at epoch 164 train batch 1 +[ 2023-09-21 11:25:55 ] Completed saving temp checkpoint 473.408 ms, 7.88 s total +[ 2023-09-21 11:25:55 ] Completed replacing temp checkpoint with checkpoint 28.004 ms, 7.91 s total +Epoch [164] :: gen_loss: 0.415, disc_loss: 0.015 +[ 2023-09-21 11:25:55 ] Completed training generator for epoch 164 60.606 ms, 7.97 s total +[ 2023-09-21 11:25:55 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 164 from eval step 0 + +[ 2023-09-21 11:25:56 ] Completed eval batch 0 to device 1,075.665 ms, 1.08 s total +[ 2023-09-21 11:25:56 ] Completed eval batch 0 forward 25.060 ms, 1.10 s total +[ 2023-09-21 11:25:56 ] Completed eval batch 0 recons_loss 0.346 ms, 1.10 s total +[ 2023-09-21 11:25:56 ] Completed eval batch 0 metrics update 0.628 ms, 1.10 s total +Saving checkpoint at epoch 164 val batch 0 +[ 2023-09-21 11:25:57 ] Completed saving temp checkpoint 413.336 ms, 1.52 s total +[ 2023-09-21 11:25:57 ] Completed replacing temp checkpoint with checkpoint 34.011 ms, 1.55 s total +[ 2023-09-21 11:25:57 ] Completed eval batch 1 to device 343.413 ms, 1.89 s total +[ 2023-09-21 11:25:57 ] Completed eval batch 1 forward 28.974 ms, 1.92 s total +[ 2023-09-21 11:25:57 ] Completed eval batch 1 recons_loss 0.334 ms, 1.92 s total +[ 2023-09-21 11:25:58 ] Completed eval batch 1 metrics update 316.485 ms, 2.24 s total +Saving checkpoint at epoch 164 val batch 1 +Epoch 164 val loss: 0.0610 +[ 2023-09-21 11:25:58 ] Completed saving temp checkpoint 395.672 ms, 2.63 s total +[ 2023-09-21 11:25:58 ] Completed replacing temp checkpoint with checkpoint 21.931 ms, 2.66 s total +[ 2023-09-21 11:25:58 ] Completed evaluating generator for epoch 164 48.680 ms, 2.70 s total + + +EPOCH :: 165 + + +[ 2023-09-21 11:25:58 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 165 from training step 0 + +[ 2023-09-21 11:26:01 ] Completed train batch 0 to device 3,125.896 ms, 3.13 s total +[ 2023-09-21 11:26:01 ] Completed train batch 0 generator forward 31.986 ms, 3.16 s total +[ 2023-09-21 11:26:01 ] Completed train batch 0 generator loss: 1.224 66.639 ms, 3.22 s total +[ 2023-09-21 11:26:02 ] Completed train batch 0 generator backward 682.725 ms, 3.91 s total +[ 2023-09-21 11:26:02 ] Completed train batch 0 discriminator loss 0.044 57.713 ms, 3.96 s total +[ 2023-09-21 11:26:02 ] Completed train batch 0 discriminator backward 106.534 ms, 4.07 s total +Epoch [165] Step [0/2], gen_loss: 0.420, disc_loss: 0.015 +[ 2023-09-21 11:26:02 ] Completed train batch 0 metrics update 4.476 ms, 4.08 s total +Saving checkpoint at epoch 165 train batch 0 +[ 2023-09-21 11:26:03 ] Completed saving temp checkpoint 450.911 ms, 4.53 s total +[ 2023-09-21 11:26:03 ] Completed replacing temp checkpoint with checkpoint 33.096 ms, 4.56 s total +[ 2023-09-21 11:26:04 ] Completed train batch 1 to device 1,376.057 ms, 5.94 s total +[ 2023-09-21 11:26:04 ] Completed train batch 1 generator forward 41.296 ms, 5.98 s total +[ 2023-09-21 11:26:04 ] Completed train batch 1 generator loss: 1.241 64.550 ms, 6.04 s total +[ 2023-09-21 11:26:05 ] Completed train batch 1 generator backward 1,183.704 ms, 7.23 s total +[ 2023-09-21 11:26:05 ] Completed train batch 1 discriminator loss 0.046 56.457 ms, 7.28 s total +[ 2023-09-21 11:26:05 ] Completed train batch 1 discriminator backward 75.709 ms, 7.36 s total +Epoch [165] Step [1/2], gen_loss: 0.420, disc_loss: 0.015 +[ 2023-09-21 11:26:05 ] Completed train batch 1 metrics update 2.259 ms, 7.36 s total +Saving checkpoint at epoch 165 train batch 1 +[ 2023-09-21 11:26:06 ] Completed saving temp checkpoint 680.522 ms, 8.04 s total +[ 2023-09-21 11:26:06 ] Completed replacing temp checkpoint with checkpoint 35.514 ms, 8.08 s total +Epoch [165] :: gen_loss: 0.420, disc_loss: 0.015 +[ 2023-09-21 11:26:06 ] Completed training generator for epoch 165 60.895 ms, 8.14 s total +[ 2023-09-21 11:26:06 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 165 from eval step 0 + +[ 2023-09-21 11:26:07 ] Completed eval batch 0 to device 1,076.073 ms, 1.08 s total +[ 2023-09-21 11:26:07 ] Completed eval batch 0 forward 40.099 ms, 1.12 s total +[ 2023-09-21 11:26:07 ] Completed eval batch 0 recons_loss 0.333 ms, 1.12 s total +[ 2023-09-21 11:26:07 ] Completed eval batch 0 metrics update 0.715 ms, 1.12 s total +Saving checkpoint at epoch 165 val batch 0 +[ 2023-09-21 11:26:08 ] Completed saving temp checkpoint 627.038 ms, 1.74 s total +[ 2023-09-21 11:26:08 ] Completed replacing temp checkpoint with checkpoint 41.649 ms, 1.79 s total +[ 2023-09-21 11:26:08 ] Completed eval batch 1 to device 96.195 ms, 1.88 s total +[ 2023-09-21 11:26:08 ] Completed eval batch 1 forward 38.081 ms, 1.92 s total +[ 2023-09-21 11:26:08 ] Completed eval batch 1 recons_loss 0.340 ms, 1.92 s total +[ 2023-09-21 11:26:08 ] Completed eval batch 1 metrics update 98.824 ms, 2.02 s total +Saving checkpoint at epoch 165 val batch 1 +Epoch 165 val loss: 0.0547 +[ 2023-09-21 11:26:09 ] Completed saving temp checkpoint 665.220 ms, 2.68 s total +[ 2023-09-21 11:26:09 ] Completed replacing temp checkpoint with checkpoint 35.386 ms, 2.72 s total +[ 2023-09-21 11:26:09 ] Completed evaluating generator for epoch 165 55.245 ms, 2.78 s total + + +EPOCH :: 166 + + +[ 2023-09-21 11:26:09 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 166 from training step 0 + +[ 2023-09-21 11:26:12 ] Completed train batch 0 to device 2,840.637 ms, 2.84 s total +[ 2023-09-21 11:26:12 ] Completed train batch 0 generator forward 26.157 ms, 2.87 s total +[ 2023-09-21 11:26:12 ] Completed train batch 0 generator loss: 1.191 66.406 ms, 2.93 s total +[ 2023-09-21 11:26:13 ] Completed train batch 0 generator backward 646.970 ms, 3.58 s total +[ 2023-09-21 11:26:13 ] Completed train batch 0 discriminator loss 0.048 57.232 ms, 3.64 s total +[ 2023-09-21 11:26:13 ] Completed train batch 0 discriminator backward 105.040 ms, 3.74 s total +Epoch [166] Step [0/2], gen_loss: 0.405, disc_loss: 0.016 +[ 2023-09-21 11:26:13 ] Completed train batch 0 metrics update 4.225 ms, 3.75 s total +Saving checkpoint at epoch 166 train batch 0 +[ 2023-09-21 11:26:13 ] Completed saving temp checkpoint 670.999 ms, 4.42 s total +[ 2023-09-21 11:26:13 ] Completed replacing temp checkpoint with checkpoint 36.478 ms, 4.45 s total +[ 2023-09-21 11:26:15 ] Completed train batch 1 to device 1,120.810 ms, 5.57 s total +[ 2023-09-21 11:26:15 ] Completed train batch 1 generator forward 22.445 ms, 5.60 s total +[ 2023-09-21 11:26:15 ] Completed train batch 1 generator loss: 1.260 63.742 ms, 5.66 s total +[ 2023-09-21 11:26:16 ] Completed train batch 1 generator backward 1,525.919 ms, 7.19 s total +[ 2023-09-21 11:26:16 ] Completed train batch 1 discriminator loss 0.055 56.548 ms, 7.24 s total +[ 2023-09-21 11:26:16 ] Completed train batch 1 discriminator backward 74.737 ms, 7.32 s total +Epoch [166] Step [1/2], gen_loss: 0.411, disc_loss: 0.016 +[ 2023-09-21 11:26:16 ] Completed train batch 1 metrics update 2.162 ms, 7.32 s total +Saving checkpoint at epoch 166 train batch 1 +[ 2023-09-21 11:26:17 ] Completed saving temp checkpoint 694.514 ms, 8.02 s total +[ 2023-09-21 11:26:17 ] Completed replacing temp checkpoint with checkpoint 40.289 ms, 8.06 s total +Epoch [166] :: gen_loss: 0.408, disc_loss: 0.016 +[ 2023-09-21 11:26:17 ] Completed training generator for epoch 166 56.289 ms, 8.11 s total +[ 2023-09-21 11:26:17 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 166 from eval step 0 + +[ 2023-09-21 11:26:18 ] Completed eval batch 0 to device 1,100.985 ms, 1.10 s total +[ 2023-09-21 11:26:18 ] Completed eval batch 0 forward 28.571 ms, 1.13 s total +[ 2023-09-21 11:26:18 ] Completed eval batch 0 recons_loss 0.325 ms, 1.13 s total +[ 2023-09-21 11:26:18 ] Completed eval batch 0 metrics update 0.660 ms, 1.13 s total +Saving checkpoint at epoch 166 val batch 0 +[ 2023-09-21 11:26:19 ] Completed saving temp checkpoint 687.935 ms, 1.82 s total +[ 2023-09-21 11:26:19 ] Completed replacing temp checkpoint with checkpoint 43.884 ms, 1.86 s total +[ 2023-09-21 11:26:19 ] Completed eval batch 1 to device 61.283 ms, 1.92 s total +[ 2023-09-21 11:26:19 ] Completed eval batch 1 forward 35.905 ms, 1.96 s total +[ 2023-09-21 11:26:19 ] Completed eval batch 1 recons_loss 0.349 ms, 1.96 s total +[ 2023-09-21 11:26:19 ] Completed eval batch 1 metrics update 68.720 ms, 2.03 s total +Saving checkpoint at epoch 166 val batch 1 +Epoch 166 val loss: 0.0544 +[ 2023-09-21 11:26:20 ] Completed saving temp checkpoint 671.195 ms, 2.70 s total +[ 2023-09-21 11:26:20 ] Completed replacing temp checkpoint with checkpoint 29.609 ms, 2.73 s total +[ 2023-09-21 11:26:20 ] Completed evaluating generator for epoch 166 59.050 ms, 2.79 s total + + +EPOCH :: 167 + + +[ 2023-09-21 11:26:20 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 167 from training step 0 + +[ 2023-09-21 11:26:23 ] Completed train batch 0 to device 2,869.917 ms, 2.87 s total +[ 2023-09-21 11:26:23 ] Completed train batch 0 generator forward 32.227 ms, 2.90 s total +[ 2023-09-21 11:26:23 ] Completed train batch 0 generator loss: 1.225 66.510 ms, 2.97 s total +[ 2023-09-21 11:26:23 ] Completed train batch 0 generator backward 632.250 ms, 3.60 s total +[ 2023-09-21 11:26:24 ] Completed train batch 0 discriminator loss 0.046 57.407 ms, 3.66 s total +[ 2023-09-21 11:26:24 ] Completed train batch 0 discriminator backward 120.961 ms, 3.78 s total +Epoch [167] Step [0/2], gen_loss: 0.412, disc_loss: 0.016 +[ 2023-09-21 11:26:24 ] Completed train batch 0 metrics update 6.131 ms, 3.79 s total +Saving checkpoint at epoch 167 train batch 0 +[ 2023-09-21 11:26:24 ] Completed saving temp checkpoint 633.592 ms, 4.42 s total +[ 2023-09-21 11:26:24 ] Completed replacing temp checkpoint with checkpoint 34.985 ms, 4.45 s total +[ 2023-09-21 11:26:25 ] Completed train batch 1 to device 1,128.124 ms, 5.58 s total +[ 2023-09-21 11:26:26 ] Completed train batch 1 generator forward 32.490 ms, 5.61 s total +[ 2023-09-21 11:26:26 ] Completed train batch 1 generator loss: 1.203 64.510 ms, 5.68 s total +[ 2023-09-21 11:26:27 ] Completed train batch 1 generator backward 1,371.315 ms, 7.05 s total +[ 2023-09-21 11:26:27 ] Completed train batch 1 discriminator loss 0.045 56.699 ms, 7.11 s total +[ 2023-09-21 11:26:27 ] Completed train batch 1 discriminator backward 90.698 ms, 7.20 s total +Epoch [167] Step [1/2], gen_loss: 0.411, disc_loss: 0.015 +[ 2023-09-21 11:26:27 ] Completed train batch 1 metrics update 4.066 ms, 7.20 s total +Saving checkpoint at epoch 167 train batch 1 +[ 2023-09-21 11:26:28 ] Completed saving temp checkpoint 685.223 ms, 7.89 s total +[ 2023-09-21 11:26:28 ] Completed replacing temp checkpoint with checkpoint 28.059 ms, 7.92 s total +Epoch [167] :: gen_loss: 0.411, disc_loss: 0.016 +[ 2023-09-21 11:26:28 ] Completed training generator for epoch 167 54.872 ms, 7.97 s total +[ 2023-09-21 11:26:28 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 167 from eval step 0 + +[ 2023-09-21 11:26:29 ] Completed eval batch 0 to device 1,048.089 ms, 1.05 s total +[ 2023-09-21 11:26:29 ] Completed eval batch 0 forward 24.836 ms, 1.07 s total +[ 2023-09-21 11:26:29 ] Completed eval batch 0 recons_loss 0.320 ms, 1.07 s total +[ 2023-09-21 11:26:29 ] Completed eval batch 0 metrics update 0.565 ms, 1.07 s total +Saving checkpoint at epoch 167 val batch 0 +[ 2023-09-21 11:26:29 ] Completed saving temp checkpoint 478.496 ms, 1.55 s total +[ 2023-09-21 11:26:29 ] Completed replacing temp checkpoint with checkpoint 32.605 ms, 1.58 s total +[ 2023-09-21 11:26:30 ] Completed eval batch 1 to device 267.063 ms, 1.85 s total +[ 2023-09-21 11:26:30 ] Completed eval batch 1 forward 40.766 ms, 1.89 s total +[ 2023-09-21 11:26:30 ] Completed eval batch 1 recons_loss 0.511 ms, 1.89 s total +[ 2023-09-21 11:26:30 ] Completed eval batch 1 metrics update 144.054 ms, 2.04 s total +Saving checkpoint at epoch 167 val batch 1 +Epoch 167 val loss: 0.0567 +[ 2023-09-21 11:26:30 ] Completed saving temp checkpoint 417.111 ms, 2.45 s total +[ 2023-09-21 11:26:30 ] Completed replacing temp checkpoint with checkpoint 21.899 ms, 2.48 s total +[ 2023-09-21 11:26:30 ] Completed evaluating generator for epoch 167 51.508 ms, 2.53 s total + + +EPOCH :: 168 + + +[ 2023-09-21 11:26:30 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 168 from training step 0 + +[ 2023-09-21 11:26:34 ] Completed train batch 0 to device 3,164.026 ms, 3.16 s total +[ 2023-09-21 11:26:34 ] Completed train batch 0 generator forward 37.776 ms, 3.20 s total +[ 2023-09-21 11:26:34 ] Completed train batch 0 generator loss: 1.252 71.726 ms, 3.27 s total +[ 2023-09-21 11:26:34 ] Completed train batch 0 generator backward 570.394 ms, 3.84 s total +[ 2023-09-21 11:26:34 ] Completed train batch 0 discriminator loss 0.046 72.625 ms, 3.92 s total +[ 2023-09-21 11:26:34 ] Completed train batch 0 discriminator backward 81.835 ms, 4.00 s total +Epoch [168] Step [0/2], gen_loss: 0.411, disc_loss: 0.015 +[ 2023-09-21 11:26:34 ] Completed train batch 0 metrics update 4.206 ms, 4.00 s total +Saving checkpoint at epoch 168 train batch 0 +[ 2023-09-21 11:26:35 ] Completed saving temp checkpoint 404.835 ms, 4.41 s total +[ 2023-09-21 11:26:35 ] Completed replacing temp checkpoint with checkpoint 26.809 ms, 4.43 s total +[ 2023-09-21 11:26:37 ] Completed train batch 1 to device 1,734.552 ms, 6.17 s total +[ 2023-09-21 11:26:37 ] Completed train batch 1 generator forward 34.731 ms, 6.20 s total +[ 2023-09-21 11:26:37 ] Completed train batch 1 generator loss: 1.201 72.537 ms, 6.28 s total +[ 2023-09-21 11:26:38 ] Completed train batch 1 generator backward 1,111.451 ms, 7.39 s total +[ 2023-09-21 11:26:38 ] Completed train batch 1 discriminator loss 0.046 56.927 ms, 7.44 s total +[ 2023-09-21 11:26:38 ] Completed train batch 1 discriminator backward 77.429 ms, 7.52 s total +Epoch [168] Step [1/2], gen_loss: 0.402, disc_loss: 0.015 +[ 2023-09-21 11:26:38 ] Completed train batch 1 metrics update 1.550 ms, 7.52 s total +Saving checkpoint at epoch 168 train batch 1 +[ 2023-09-21 11:26:38 ] Completed saving temp checkpoint 440.375 ms, 7.96 s total +[ 2023-09-21 11:26:38 ] Completed replacing temp checkpoint with checkpoint 23.686 ms, 7.99 s total +Epoch [168] :: gen_loss: 0.406, disc_loss: 0.015 +[ 2023-09-21 11:26:38 ] Completed training generator for epoch 168 58.996 ms, 8.05 s total +[ 2023-09-21 11:26:38 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 168 from eval step 0 + +[ 2023-09-21 11:26:40 ] Completed eval batch 0 to device 1,128.753 ms, 1.13 s total +[ 2023-09-21 11:26:40 ] Completed eval batch 0 forward 29.081 ms, 1.16 s total +[ 2023-09-21 11:26:40 ] Completed eval batch 0 recons_loss 0.267 ms, 1.16 s total +[ 2023-09-21 11:26:40 ] Completed eval batch 0 metrics update 0.535 ms, 1.16 s total +Saving checkpoint at epoch 168 val batch 0 +[ 2023-09-21 11:26:40 ] Completed saving temp checkpoint 586.971 ms, 1.75 s total +[ 2023-09-21 11:26:40 ] Completed replacing temp checkpoint with checkpoint 17.180 ms, 1.76 s total +[ 2023-09-21 11:26:40 ] Completed eval batch 1 to device 262.178 ms, 2.02 s total +[ 2023-09-21 11:26:40 ] Completed eval batch 1 forward 22.001 ms, 2.05 s total +[ 2023-09-21 11:26:40 ] Completed eval batch 1 recons_loss 0.345 ms, 2.05 s total +[ 2023-09-21 11:26:41 ] Completed eval batch 1 metrics update 187.725 ms, 2.24 s total +Saving checkpoint at epoch 168 val batch 1 +Epoch 168 val loss: 0.0548 +[ 2023-09-21 11:26:41 ] Completed saving temp checkpoint 410.879 ms, 2.65 s total +[ 2023-09-21 11:26:41 ] Completed replacing temp checkpoint with checkpoint 24.957 ms, 2.67 s total +[ 2023-09-21 11:26:41 ] Completed evaluating generator for epoch 168 62.101 ms, 2.73 s total + + +EPOCH :: 169 + + +[ 2023-09-21 11:26:41 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 169 from training step 0 + +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 11:40:59 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 11:40:59 ] Completed importing Timer 0.024 ms, 0.00 s total +[ 2023-09-21 11:41:04 ] Completed importing everything else 4,648.372 ms, 4.65 s total +| distributed init (rank 2): env:// +| distributed init (rank 5): env:// +| distributed init (rank 1): env:// +| distributed init (rank 3): env:// +| distributed init (rank 4): env:// +| distributed init (rank 0): env:// +[ 2023-09-21 11:41:11 ] Completed preliminaries 7,856.384 ms, 12.50 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 11:41:12 ] Completed build datasets 11.608 ms, 12.52 s total +[ 2023-09-21 11:41:12 ] Completed build samplers 0.044 ms, 12.52 s total +[ 2023-09-21 11:41:12 ] Completed build dataloaders 0.139 ms, 12.52 s total +[ 2023-09-21 11:41:12 ] Completed generator to device 78.638 ms, 12.60 s total +[ 2023-09-21 11:41:12 ] Completed discriminator to device 7.244 ms, 12.60 s total +[ 2023-09-21 11:41:12 ] Completed loss functions 382.366 ms, 12.98 s total +[ 2023-09-21 11:41:12 ] Completed models prepped for distribution 102.065 ms, 13.09 s total +[ 2023-09-21 11:41:12 ] Completed optimizers 0.680 ms, 13.09 s total +[ 2023-09-21 11:41:12 ] Completed grad scalers 0.029 ms, 13.09 s total +[ 2023-09-21 11:41:13 ] Completed checkpoint retrieval 519.796 ms, 13.61 s total + + +EPOCH :: 168 + + +[ 2023-09-21 11:41:13 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 168 from training step 2 + +Epoch [168] :: gen_loss: 0.406, disc_loss: 0.015 +[ 2023-09-21 11:41:13 ] Completed training generator for epoch 168 109.812 ms, 0.11 s total +[ 2023-09-21 11:41:13 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 168 from eval step 2 + +[ 2023-09-21 11:41:13 ] Completed evaluating generator for epoch 168 121.936 ms, 0.12 s total + + +EPOCH :: 169 + + +[ 2023-09-21 11:41:13 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 169 from training step 0 + +[ 2023-09-21 11:41:16 ] Completed train batch 0 to device 3,485.993 ms, 3.49 s total +[ 2023-09-21 11:41:18 ] Completed train batch 0 generator forward 1,536.519 ms, 5.02 s total +[ 2023-09-21 11:41:18 ] Completed train batch 0 generator loss: 1.239 92.599 ms, 5.12 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 11:41:20 ] Completed train batch 0 generator backward 1,555.021 ms, 6.67 s total +[ 2023-09-21 11:41:20 ] Completed train batch 0 discriminator loss 0.046 56.930 ms, 6.73 s total +[ 2023-09-21 11:41:20 ] Completed train batch 0 discriminator backward 84.578 ms, 6.81 s total +Epoch [169] Step [0/2], gen_loss: 0.404, disc_loss: 0.016 +[ 2023-09-21 11:41:20 ] Completed train batch 0 metrics update 19.441 ms, 6.83 s total +Saving checkpoint at epoch 169 train batch 0 +[ 2023-09-21 11:41:20 ] Completed saving temp checkpoint 454.433 ms, 7.29 s total +[ 2023-09-21 11:41:20 ] Completed replacing temp checkpoint with checkpoint 120.329 ms, 7.41 s total +[ 2023-09-21 11:41:20 ] Completed train batch 1 to device 20.052 ms, 7.43 s total +[ 2023-09-21 11:41:20 ] Completed train batch 1 generator forward 25.062 ms, 7.45 s total +[ 2023-09-21 11:41:20 ] Completed train batch 1 generator loss: 1.180 62.879 ms, 7.51 s total +[ 2023-09-21 11:41:22 ] Completed train batch 1 generator backward 1,325.656 ms, 8.84 s total +[ 2023-09-21 11:41:22 ] Completed train batch 1 discriminator loss 0.047 56.547 ms, 8.90 s total +[ 2023-09-21 11:41:22 ] Completed train batch 1 discriminator backward 80.041 ms, 8.98 s total +Epoch [169] Step [1/2], gen_loss: 0.404, disc_loss: 0.016 +[ 2023-09-21 11:41:22 ] Completed train batch 1 metrics update 1.553 ms, 8.98 s total +Saving checkpoint at epoch 169 train batch 1 +[ 2023-09-21 11:41:22 ] Completed saving temp checkpoint 447.335 ms, 9.42 s total +[ 2023-09-21 11:41:22 ] Completed replacing temp checkpoint with checkpoint 22.635 ms, 9.45 s total +Epoch [169] :: gen_loss: 0.404, disc_loss: 0.016 +[ 2023-09-21 11:41:22 ] Completed training generator for epoch 169 36.249 ms, 9.48 s total +[ 2023-09-21 11:41:22 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 169 from eval step 0 + +[ 2023-09-21 11:41:23 ] Completed eval batch 0 to device 1,160.101 ms, 1.16 s total +[ 2023-09-21 11:41:24 ] Completed eval batch 0 forward 25.846 ms, 1.19 s total +[ 2023-09-21 11:41:24 ] Completed eval batch 0 recons_loss 0.368 ms, 1.19 s total +[ 2023-09-21 11:41:24 ] Completed eval batch 0 metrics update 0.653 ms, 1.19 s total +Saving checkpoint at epoch 169 val batch 0 +[ 2023-09-21 11:41:24 ] Completed saving temp checkpoint 420.058 ms, 1.61 s total +[ 2023-09-21 11:41:24 ] Completed replacing temp checkpoint with checkpoint 30.611 ms, 1.64 s total +[ 2023-09-21 11:41:24 ] Completed eval batch 1 to device 414.237 ms, 2.05 s total +[ 2023-09-21 11:41:24 ] Completed eval batch 1 forward 22.021 ms, 2.07 s total +[ 2023-09-21 11:41:24 ] Completed eval batch 1 recons_loss 0.360 ms, 2.07 s total +[ 2023-09-21 11:41:25 ] Completed eval batch 1 metrics update 253.209 ms, 2.33 s total +Saving checkpoint at epoch 169 val batch 1 +Epoch 169 val loss: 0.0562 +[ 2023-09-21 11:41:25 ] Completed saving temp checkpoint 465.446 ms, 2.79 s total +[ 2023-09-21 11:41:25 ] Completed replacing temp checkpoint with checkpoint 31.004 ms, 2.82 s total +[ 2023-09-21 11:41:25 ] Completed evaluating generator for epoch 169 50.821 ms, 2.87 s total + + +EPOCH :: 170 + + +[ 2023-09-21 11:41:25 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 170 from training step 0 + +[ 2023-09-21 11:41:28 ] Completed train batch 0 to device 2,872.545 ms, 2.87 s total +[ 2023-09-21 11:41:28 ] Completed train batch 0 generator forward 41.665 ms, 2.91 s total +[ 2023-09-21 11:41:28 ] Completed train batch 0 generator loss: 1.181 65.086 ms, 2.98 s total +[ 2023-09-21 11:41:29 ] Completed train batch 0 generator backward 924.455 ms, 3.90 s total +[ 2023-09-21 11:41:29 ] Completed train batch 0 discriminator loss 0.048 78.923 ms, 3.98 s total +[ 2023-09-21 11:41:29 ] Completed train batch 0 discriminator backward 68.686 ms, 4.05 s total +Epoch [170] Step [0/2], gen_loss: 0.397, disc_loss: 0.016 +[ 2023-09-21 11:41:29 ] Completed train batch 0 metrics update 9.333 ms, 4.06 s total +Saving checkpoint at epoch 170 train batch 0 +[ 2023-09-21 11:41:30 ] Completed saving temp checkpoint 433.949 ms, 4.49 s total +[ 2023-09-21 11:41:30 ] Completed replacing temp checkpoint with checkpoint 32.119 ms, 4.53 s total +[ 2023-09-21 11:41:31 ] Completed train batch 1 to device 1,121.554 ms, 5.65 s total +[ 2023-09-21 11:41:31 ] Completed train batch 1 generator forward 41.710 ms, 5.69 s total +[ 2023-09-21 11:41:31 ] Completed train batch 1 generator loss: 1.243 62.334 ms, 5.75 s total +[ 2023-09-21 11:41:33 ] Completed train batch 1 generator backward 1,849.906 ms, 7.60 s total +[ 2023-09-21 11:41:33 ] Completed train batch 1 discriminator loss 0.049 59.353 ms, 7.66 s total +[ 2023-09-21 11:41:33 ] Completed train batch 1 discriminator backward 59.574 ms, 7.72 s total +Epoch [170] Step [1/2], gen_loss: 0.401, disc_loss: 0.016 +[ 2023-09-21 11:41:33 ] Completed train batch 1 metrics update 5.229 ms, 7.73 s total +Saving checkpoint at epoch 170 train batch 1 +[ 2023-09-21 11:41:33 ] Completed saving temp checkpoint 430.513 ms, 8.16 s total +[ 2023-09-21 11:41:33 ] Completed replacing temp checkpoint with checkpoint 27.721 ms, 8.18 s total +Epoch [170] :: gen_loss: 0.399, disc_loss: 0.016 +[ 2023-09-21 11:41:33 ] Completed training generator for epoch 170 60.305 ms, 8.24 s total +[ 2023-09-21 11:41:33 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 170 from eval step 0 + +[ 2023-09-21 11:41:35 ] Completed eval batch 0 to device 1,168.538 ms, 1.17 s total +[ 2023-09-21 11:41:35 ] Completed eval batch 0 forward 43.054 ms, 1.21 s total +[ 2023-09-21 11:41:35 ] Completed eval batch 0 recons_loss 0.326 ms, 1.21 s total +[ 2023-09-21 11:41:35 ] Completed eval batch 0 metrics update 0.626 ms, 1.21 s total +Saving checkpoint at epoch 170 val batch 0 +[ 2023-09-21 11:41:35 ] Completed saving temp checkpoint 406.886 ms, 1.62 s total +[ 2023-09-21 11:41:35 ] Completed replacing temp checkpoint with checkpoint 23.977 ms, 1.64 s total +[ 2023-09-21 11:41:35 ] Completed eval batch 1 to device 417.337 ms, 2.06 s total +[ 2023-09-21 11:41:36 ] Completed eval batch 1 forward 39.745 ms, 2.10 s total +[ 2023-09-21 11:41:36 ] Completed eval batch 1 recons_loss 0.328 ms, 2.10 s total +[ 2023-09-21 11:41:36 ] Completed eval batch 1 metrics update 202.466 ms, 2.30 s total +Saving checkpoint at epoch 170 val batch 1 +Epoch 170 val loss: 0.0601 +[ 2023-09-21 11:41:36 ] Completed saving temp checkpoint 449.920 ms, 2.75 s total +[ 2023-09-21 11:41:36 ] Completed replacing temp checkpoint with checkpoint 17.849 ms, 2.77 s total +[ 2023-09-21 11:41:36 ] Completed evaluating generator for epoch 170 47.031 ms, 2.82 s total + + +EPOCH :: 171 + + +[ 2023-09-21 11:41:36 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 171 from training step 0 + +[ 2023-09-21 11:41:39 ] Completed train batch 0 to device 3,135.607 ms, 3.14 s total +[ 2023-09-21 11:41:39 ] Completed train batch 0 generator forward 34.738 ms, 3.17 s total +[ 2023-09-21 11:41:39 ] Completed train batch 0 generator loss: 1.227 70.455 ms, 3.24 s total +[ 2023-09-21 11:41:40 ] Completed train batch 0 generator backward 750.686 ms, 3.99 s total +[ 2023-09-21 11:41:40 ] Completed train batch 0 discriminator loss 0.050 57.345 ms, 4.05 s total +[ 2023-09-21 11:41:40 ] Completed train batch 0 discriminator backward 83.589 ms, 4.13 s total +Epoch [171] Step [0/2], gen_loss: 0.404, disc_loss: 0.017 +[ 2023-09-21 11:41:40 ] Completed train batch 0 metrics update 12.744 ms, 4.15 s total +Saving checkpoint at epoch 171 train batch 0 +[ 2023-09-21 11:41:41 ] Completed saving temp checkpoint 426.930 ms, 4.57 s total +[ 2023-09-21 11:41:41 ] Completed replacing temp checkpoint with checkpoint 22.216 ms, 4.59 s total +[ 2023-09-21 11:41:42 ] Completed train batch 1 to device 1,440.698 ms, 6.04 s total +[ 2023-09-21 11:41:42 ] Completed train batch 1 generator forward 25.730 ms, 6.06 s total +[ 2023-09-21 11:41:42 ] Completed train batch 1 generator loss: 1.190 65.041 ms, 6.13 s total +[ 2023-09-21 11:41:44 ] Completed train batch 1 generator backward 1,509.776 ms, 7.64 s total +[ 2023-09-21 11:41:44 ] Completed train batch 1 discriminator loss 0.050 57.125 ms, 7.69 s total +[ 2023-09-21 11:41:44 ] Completed train batch 1 discriminator backward 74.227 ms, 7.77 s total +Epoch [171] Step [1/2], gen_loss: 0.400, disc_loss: 0.017 +[ 2023-09-21 11:41:44 ] Completed train batch 1 metrics update 5.456 ms, 7.77 s total +Saving checkpoint at epoch 171 train batch 1 +[ 2023-09-21 11:41:44 ] Completed saving temp checkpoint 447.719 ms, 8.22 s total +[ 2023-09-21 11:41:44 ] Completed replacing temp checkpoint with checkpoint 20.580 ms, 8.24 s total +Epoch [171] :: gen_loss: 0.402, disc_loss: 0.017 +[ 2023-09-21 11:41:45 ] Completed training generator for epoch 171 52.815 ms, 8.29 s total +[ 2023-09-21 11:41:45 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 171 from eval step 0 + +[ 2023-09-21 11:41:46 ] Completed eval batch 0 to device 1,118.334 ms, 1.12 s total +[ 2023-09-21 11:41:46 ] Completed eval batch 0 forward 30.652 ms, 1.15 s total +[ 2023-09-21 11:41:46 ] Completed eval batch 0 recons_loss 0.232 ms, 1.15 s total +[ 2023-09-21 11:41:46 ] Completed eval batch 0 metrics update 0.502 ms, 1.15 s total +Saving checkpoint at epoch 171 val batch 0 +[ 2023-09-21 11:41:46 ] Completed saving temp checkpoint 430.765 ms, 1.58 s total +[ 2023-09-21 11:41:46 ] Completed replacing temp checkpoint with checkpoint 21.486 ms, 1.60 s total +[ 2023-09-21 11:41:47 ] Completed eval batch 1 to device 427.644 ms, 2.03 s total +[ 2023-09-21 11:41:47 ] Completed eval batch 1 forward 40.235 ms, 2.07 s total +[ 2023-09-21 11:41:47 ] Completed eval batch 1 recons_loss 0.379 ms, 2.07 s total +[ 2023-09-21 11:41:47 ] Completed eval batch 1 metrics update 183.428 ms, 2.25 s total +Saving checkpoint at epoch 171 val batch 1 +Epoch 171 val loss: 0.0552 +[ 2023-09-21 11:41:47 ] Completed saving temp checkpoint 458.372 ms, 2.71 s total +[ 2023-09-21 11:41:47 ] Completed replacing temp checkpoint with checkpoint 19.880 ms, 2.73 s total +[ 2023-09-21 11:41:47 ] Completed evaluating generator for epoch 171 47.259 ms, 2.78 s total + + +EPOCH :: 172 + + +[ 2023-09-21 11:41:47 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 172 from training step 0 + +[ 2023-09-21 11:41:51 ] Completed train batch 0 to device 3,208.948 ms, 3.21 s total +[ 2023-09-21 11:41:51 ] Completed train batch 0 generator forward 27.690 ms, 3.24 s total +[ 2023-09-21 11:41:51 ] Completed train batch 0 generator loss: 1.238 66.448 ms, 3.30 s total +[ 2023-09-21 11:41:51 ] Completed train batch 0 generator backward 569.341 ms, 3.87 s total +[ 2023-09-21 11:41:51 ] Completed train batch 0 discriminator loss 0.051 58.379 ms, 3.93 s total +[ 2023-09-21 11:41:51 ] Completed train batch 0 discriminator backward 96.484 ms, 4.03 s total +Epoch [172] Step [0/2], gen_loss: 0.403, disc_loss: 0.017 +[ 2023-09-21 11:41:51 ] Completed train batch 0 metrics update 1.866 ms, 4.03 s total +Saving checkpoint at epoch 172 train batch 0 +[ 2023-09-21 11:41:52 ] Completed saving temp checkpoint 686.612 ms, 4.72 s total +[ 2023-09-21 11:41:52 ] Completed replacing temp checkpoint with checkpoint 37.654 ms, 4.75 s total +[ 2023-09-21 11:41:54 ] Completed train batch 1 to device 1,433.906 ms, 6.19 s total +[ 2023-09-21 11:41:54 ] Completed train batch 1 generator forward 22.728 ms, 6.21 s total +[ 2023-09-21 11:41:54 ] Completed train batch 1 generator loss: 1.236 63.619 ms, 6.27 s total +[ 2023-09-21 11:41:55 ] Completed train batch 1 generator backward 1,048.845 ms, 7.32 s total +[ 2023-09-21 11:41:55 ] Completed train batch 1 discriminator loss 0.051 57.711 ms, 7.38 s total +[ 2023-09-21 11:41:55 ] Completed train batch 1 discriminator backward 59.919 ms, 7.44 s total +Epoch [172] Step [1/2], gen_loss: 0.411, disc_loss: 0.017 +[ 2023-09-21 11:41:55 ] Completed train batch 1 metrics update 2.161 ms, 7.44 s total +Saving checkpoint at epoch 172 train batch 1 +[ 2023-09-21 11:41:55 ] Completed saving temp checkpoint 526.052 ms, 7.97 s total +[ 2023-09-21 11:41:55 ] Completed replacing temp checkpoint with checkpoint 26.661 ms, 8.00 s total +Epoch [172] :: gen_loss: 0.407, disc_loss: 0.017 +[ 2023-09-21 11:41:55 ] Completed training generator for epoch 172 59.329 ms, 8.05 s total +[ 2023-09-21 11:41:55 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 172 from eval step 0 + +[ 2023-09-21 11:41:57 ] Completed eval batch 0 to device 1,175.525 ms, 1.18 s total +[ 2023-09-21 11:41:57 ] Completed eval batch 0 forward 32.246 ms, 1.21 s total +[ 2023-09-21 11:41:57 ] Completed eval batch 0 recons_loss 0.340 ms, 1.21 s total +[ 2023-09-21 11:41:57 ] Completed eval batch 0 metrics update 0.660 ms, 1.21 s total +Saving checkpoint at epoch 172 val batch 0 +[ 2023-09-21 11:41:57 ] Completed saving temp checkpoint 489.607 ms, 1.70 s total +[ 2023-09-21 11:41:57 ] Completed replacing temp checkpoint with checkpoint 29.349 ms, 1.73 s total +[ 2023-09-21 11:41:57 ] Completed eval batch 1 to device 386.135 ms, 2.11 s total +[ 2023-09-21 11:41:58 ] Completed eval batch 1 forward 21.802 ms, 2.14 s total +[ 2023-09-21 11:41:58 ] Completed eval batch 1 recons_loss 0.368 ms, 2.14 s total +[ 2023-09-21 11:41:58 ] Completed eval batch 1 metrics update 55.901 ms, 2.19 s total +Saving checkpoint at epoch 172 val batch 1 +Epoch 172 val loss: 0.0533 +[ 2023-09-21 11:41:58 ] Completed saving temp checkpoint 730.972 ms, 2.92 s total +[ 2023-09-21 11:41:58 ] Completed replacing temp checkpoint with checkpoint 33.438 ms, 2.96 s total +[ 2023-09-21 11:41:58 ] Completed evaluating generator for epoch 172 59.247 ms, 3.02 s total + + +EPOCH :: 173 + + +[ 2023-09-21 11:41:58 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 173 from training step 0 + +[ 2023-09-21 11:42:01 ] Completed train batch 0 to device 3,016.777 ms, 3.02 s total +[ 2023-09-21 11:42:01 ] Completed train batch 0 generator forward 26.796 ms, 3.04 s total +[ 2023-09-21 11:42:02 ] Completed train batch 0 generator loss: 1.180 66.310 ms, 3.11 s total +[ 2023-09-21 11:42:02 ] Completed train batch 0 generator backward 342.952 ms, 3.45 s total +[ 2023-09-21 11:42:02 ] Completed train batch 0 discriminator loss 0.048 58.064 ms, 3.51 s total +[ 2023-09-21 11:42:02 ] Completed train batch 0 discriminator backward 91.419 ms, 3.60 s total +Epoch [173] Step [0/2], gen_loss: 0.407, disc_loss: 0.016 +[ 2023-09-21 11:42:02 ] Completed train batch 0 metrics update 20.550 ms, 3.62 s total +Saving checkpoint at epoch 173 train batch 0 +[ 2023-09-21 11:42:02 ] Completed saving temp checkpoint 470.360 ms, 4.09 s total +[ 2023-09-21 11:42:03 ] Completed replacing temp checkpoint with checkpoint 33.998 ms, 4.13 s total +[ 2023-09-21 11:42:05 ] Completed train batch 1 to device 2,049.233 ms, 6.18 s total +[ 2023-09-21 11:42:05 ] Completed train batch 1 generator forward 24.536 ms, 6.20 s total +[ 2023-09-21 11:42:05 ] Completed train batch 1 generator loss: 1.249 63.467 ms, 6.26 s total +[ 2023-09-21 11:42:05 ] Completed train batch 1 generator backward 409.985 ms, 6.67 s total +[ 2023-09-21 11:42:05 ] Completed train batch 1 discriminator loss 0.045 56.335 ms, 6.73 s total +[ 2023-09-21 11:42:05 ] Completed train batch 1 discriminator backward 70.685 ms, 6.80 s total +Epoch [173] Step [1/2], gen_loss: 0.406, disc_loss: 0.015 +[ 2023-09-21 11:42:05 ] Completed train batch 1 metrics update 2.723 ms, 6.80 s total +Saving checkpoint at epoch 173 train batch 1 +[ 2023-09-21 11:42:06 ] Completed saving temp checkpoint 489.845 ms, 7.29 s total +[ 2023-09-21 11:42:06 ] Completed replacing temp checkpoint with checkpoint 30.426 ms, 7.32 s total +Epoch [173] :: gen_loss: 0.407, disc_loss: 0.016 +[ 2023-09-21 11:42:06 ] Completed training generator for epoch 173 54.302 ms, 7.38 s total +[ 2023-09-21 11:42:06 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 173 from eval step 0 + +[ 2023-09-21 11:42:07 ] Completed eval batch 0 to device 1,183.647 ms, 1.18 s total +[ 2023-09-21 11:42:07 ] Completed eval batch 0 forward 41.398 ms, 1.23 s total +[ 2023-09-21 11:42:07 ] Completed eval batch 0 recons_loss 0.324 ms, 1.23 s total +[ 2023-09-21 11:42:07 ] Completed eval batch 0 metrics update 0.676 ms, 1.23 s total +Saving checkpoint at epoch 173 val batch 0 +[ 2023-09-21 11:42:07 ] Completed saving temp checkpoint 469.214 ms, 1.70 s total +[ 2023-09-21 11:42:08 ] Completed replacing temp checkpoint with checkpoint 33.954 ms, 1.73 s total +[ 2023-09-21 11:42:08 ] Completed eval batch 1 to device 371.205 ms, 2.10 s total +[ 2023-09-21 11:42:08 ] Completed eval batch 1 forward 40.917 ms, 2.14 s total +[ 2023-09-21 11:42:08 ] Completed eval batch 1 recons_loss 0.531 ms, 2.14 s total +[ 2023-09-21 11:42:08 ] Completed eval batch 1 metrics update 176.558 ms, 2.32 s total +Saving checkpoint at epoch 173 val batch 1 +Epoch 173 val loss: 0.0566 +[ 2023-09-21 11:42:09 ] Completed saving temp checkpoint 511.542 ms, 2.83 s total +[ 2023-09-21 11:42:09 ] Completed replacing temp checkpoint with checkpoint 24.737 ms, 2.85 s total +[ 2023-09-21 11:42:09 ] Completed evaluating generator for epoch 173 63.037 ms, 2.92 s total + + +EPOCH :: 174 + + +[ 2023-09-21 11:42:09 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 174 from training step 0 + +[ 2023-09-21 11:42:12 ] Completed train batch 0 to device 3,146.555 ms, 3.15 s total +[ 2023-09-21 11:42:12 ] Completed train batch 0 generator forward 33.893 ms, 3.18 s total +[ 2023-09-21 11:42:12 ] Completed train batch 0 generator loss: 1.211 68.530 ms, 3.25 s total +[ 2023-09-21 11:42:13 ] Completed train batch 0 generator backward 589.703 ms, 3.84 s total +[ 2023-09-21 11:42:13 ] Completed train batch 0 discriminator loss 0.044 57.433 ms, 3.90 s total +[ 2023-09-21 11:42:13 ] Completed train batch 0 discriminator backward 87.933 ms, 3.98 s total +Epoch [174] Step [0/2], gen_loss: 0.408, disc_loss: 0.015 +[ 2023-09-21 11:42:13 ] Completed train batch 0 metrics update 4.095 ms, 3.99 s total +Saving checkpoint at epoch 174 train batch 0 +[ 2023-09-21 11:42:13 ] Completed saving temp checkpoint 513.667 ms, 4.50 s total +[ 2023-09-21 11:42:13 ] Completed replacing temp checkpoint with checkpoint 25.424 ms, 4.53 s total +[ 2023-09-21 11:42:15 ] Completed train batch 1 to device 1,604.250 ms, 6.13 s total +[ 2023-09-21 11:42:15 ] Completed train batch 1 generator forward 22.454 ms, 6.15 s total +[ 2023-09-21 11:42:15 ] Completed train batch 1 generator loss: 1.172 64.437 ms, 6.22 s total +[ 2023-09-21 11:42:16 ] Completed train batch 1 generator backward 1,237.122 ms, 7.46 s total +[ 2023-09-21 11:42:16 ] Completed train batch 1 discriminator loss 0.042 56.834 ms, 7.51 s total +[ 2023-09-21 11:42:16 ] Completed train batch 1 discriminator backward 72.926 ms, 7.59 s total +Epoch [174] Step [1/2], gen_loss: 0.411, disc_loss: 0.014 +[ 2023-09-21 11:42:16 ] Completed train batch 1 metrics update 1.797 ms, 7.59 s total +Saving checkpoint at epoch 174 train batch 1 +[ 2023-09-21 11:42:17 ] Completed saving temp checkpoint 504.615 ms, 8.09 s total +[ 2023-09-21 11:42:17 ] Completed replacing temp checkpoint with checkpoint 32.890 ms, 8.12 s total +Epoch [174] :: gen_loss: 0.410, disc_loss: 0.014 +[ 2023-09-21 11:42:17 ] Completed training generator for epoch 174 72.298 ms, 8.20 s total +[ 2023-09-21 11:42:17 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 174 from eval step 0 + +[ 2023-09-21 11:42:18 ] Completed eval batch 0 to device 1,114.382 ms, 1.11 s total +[ 2023-09-21 11:42:18 ] Completed eval batch 0 forward 26.863 ms, 1.14 s total +[ 2023-09-21 11:42:18 ] Completed eval batch 0 recons_loss 0.299 ms, 1.14 s total +[ 2023-09-21 11:42:18 ] Completed eval batch 0 metrics update 0.604 ms, 1.14 s total +Saving checkpoint at epoch 174 val batch 0 +[ 2023-09-21 11:42:19 ] Completed saving temp checkpoint 475.944 ms, 1.62 s total +[ 2023-09-21 11:42:19 ] Completed replacing temp checkpoint with checkpoint 29.186 ms, 1.65 s total +[ 2023-09-21 11:42:19 ] Completed eval batch 1 to device 481.662 ms, 2.13 s total +[ 2023-09-21 11:42:19 ] Completed eval batch 1 forward 21.773 ms, 2.15 s total +[ 2023-09-21 11:42:19 ] Completed eval batch 1 recons_loss 0.345 ms, 2.15 s total +[ 2023-09-21 11:42:19 ] Completed eval batch 1 metrics update 10.519 ms, 2.16 s total +Saving checkpoint at epoch 174 val batch 1 +Epoch 174 val loss: 0.0612 +[ 2023-09-21 11:42:20 ] Completed saving temp checkpoint 505.052 ms, 2.67 s total +[ 2023-09-21 11:42:20 ] Completed replacing temp checkpoint with checkpoint 26.120 ms, 2.69 s total +[ 2023-09-21 11:42:20 ] Completed evaluating generator for epoch 174 57.481 ms, 2.75 s total + + +EPOCH :: 175 + + +[ 2023-09-21 11:42:20 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 175 from training step 0 + +[ 2023-09-21 11:42:23 ] Completed train batch 0 to device 3,042.224 ms, 3.04 s total +[ 2023-09-21 11:42:23 ] Completed train batch 0 generator forward 43.902 ms, 3.09 s total +[ 2023-09-21 11:42:23 ] Completed train batch 0 generator loss: 1.180 65.134 ms, 3.15 s total +[ 2023-09-21 11:42:23 ] Completed train batch 0 generator backward 593.289 ms, 3.74 s total +[ 2023-09-21 11:42:23 ] Completed train batch 0 discriminator loss 0.042 58.392 ms, 3.80 s total +[ 2023-09-21 11:42:24 ] Completed train batch 0 discriminator backward 106.200 ms, 3.91 s total +Epoch [175] Step [0/2], gen_loss: 0.410, disc_loss: 0.014 +[ 2023-09-21 11:42:24 ] Completed train batch 0 metrics update 5.814 ms, 3.91 s total +Saving checkpoint at epoch 175 train batch 0 +[ 2023-09-21 11:42:24 ] Completed saving temp checkpoint 466.276 ms, 4.38 s total +[ 2023-09-21 11:42:24 ] Completed replacing temp checkpoint with checkpoint 31.236 ms, 4.41 s total +[ 2023-09-21 11:42:25 ] Completed train batch 1 to device 1,338.562 ms, 5.75 s total +[ 2023-09-21 11:42:25 ] Completed train batch 1 generator forward 40.453 ms, 5.79 s total +[ 2023-09-21 11:42:25 ] Completed train batch 1 generator loss: 1.206 64.340 ms, 5.86 s total +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 11:54:56 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 11:54:56 ] Completed importing Timer 0.022 ms, 0.00 s total +[ 2023-09-21 11:55:00 ] Completed importing everything else 4,608.471 ms, 4.61 s total +| distributed init (rank 2): env:// +| distributed init (rank 4): env:// +| distributed init (rank 3): env:// +| distributed init (rank 0): env:// +| distributed init (rank 5): env:// +| distributed init (rank 1): env:// +[ 2023-09-21 11:55:08 ] Completed preliminaries 7,586.636 ms, 12.20 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 11:55:08 ] Completed build datasets 10.651 ms, 12.21 s total +[ 2023-09-21 11:55:08 ] Completed build samplers 0.042 ms, 12.21 s total +[ 2023-09-21 11:55:08 ] Completed build dataloaders 0.130 ms, 12.21 s total +[ 2023-09-21 11:55:08 ] Completed generator to device 77.599 ms, 12.28 s total +[ 2023-09-21 11:55:08 ] Completed discriminator to device 7.065 ms, 12.29 s total +[ 2023-09-21 11:55:08 ] Completed loss functions 388.715 ms, 12.68 s total +[ 2023-09-21 11:55:08 ] Completed models prepped for distribution 84.495 ms, 12.76 s total +[ 2023-09-21 11:55:08 ] Completed optimizers 0.692 ms, 12.76 s total +[ 2023-09-21 11:55:08 ] Completed grad scalers 0.029 ms, 12.76 s total +[ 2023-09-21 11:55:09 ] Completed checkpoint retrieval 460.794 ms, 13.23 s total + + +EPOCH :: 175 + + +[ 2023-09-21 11:55:09 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 175 from training step 1 + +[ 2023-09-21 11:55:12 ] Completed train batch 1 to device 3,578.585 ms, 3.58 s total +[ 2023-09-21 11:55:13 ] Completed train batch 1 generator forward 1,133.782 ms, 4.71 s total +[ 2023-09-21 11:55:14 ] Completed train batch 1 generator loss: 1.264 78.271 ms, 4.79 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 11:55:15 ] Completed train batch 1 generator backward 1,466.445 ms, 6.26 s total +[ 2023-09-21 11:55:15 ] Completed train batch 1 discriminator loss 0.041 65.422 ms, 6.32 s total +[ 2023-09-21 11:55:15 ] Completed train batch 1 discriminator backward 63.294 ms, 6.39 s total +Epoch [175] Step [1/2], gen_loss: 0.414, disc_loss: 0.014 +[ 2023-09-21 11:55:15 ] Completed train batch 1 metrics update 5.272 ms, 6.39 s total +Saving checkpoint at epoch 175 train batch 1 +[ 2023-09-21 11:55:16 ] Completed saving temp checkpoint 483.116 ms, 6.87 s total +[ 2023-09-21 11:55:16 ] Completed replacing temp checkpoint with checkpoint 134.847 ms, 7.01 s total +Epoch [175] :: gen_loss: 0.412, disc_loss: 0.014 +[ 2023-09-21 11:55:16 ] Completed training generator for epoch 175 55.545 ms, 7.06 s total +[ 2023-09-21 11:55:16 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 175 from eval step 0 + +[ 2023-09-21 11:55:17 ] Completed eval batch 0 to device 1,080.645 ms, 1.08 s total +[ 2023-09-21 11:55:17 ] Completed eval batch 0 forward 25.617 ms, 1.11 s total +[ 2023-09-21 11:55:17 ] Completed eval batch 0 recons_loss 0.396 ms, 1.11 s total +[ 2023-09-21 11:55:17 ] Completed eval batch 0 metrics update 0.757 ms, 1.11 s total +Saving checkpoint at epoch 175 val batch 0 +[ 2023-09-21 11:55:17 ] Completed saving temp checkpoint 396.540 ms, 1.50 s total +[ 2023-09-21 11:55:17 ] Completed replacing temp checkpoint with checkpoint 22.988 ms, 1.53 s total +[ 2023-09-21 11:55:18 ] Completed eval batch 1 to device 454.453 ms, 1.98 s total +[ 2023-09-21 11:55:18 ] Completed eval batch 1 forward 40.149 ms, 2.02 s total +[ 2023-09-21 11:55:18 ] Completed eval batch 1 recons_loss 0.619 ms, 2.02 s total +[ 2023-09-21 11:55:18 ] Completed eval batch 1 metrics update 182.432 ms, 2.20 s total +Saving checkpoint at epoch 175 val batch 1 +Epoch 175 val loss: 0.0616 +[ 2023-09-21 11:55:18 ] Completed saving temp checkpoint 453.196 ms, 2.66 s total +[ 2023-09-21 11:55:18 ] Completed replacing temp checkpoint with checkpoint 18.401 ms, 2.68 s total +[ 2023-09-21 11:55:19 ] Completed evaluating generator for epoch 175 73.499 ms, 2.75 s total + + +EPOCH :: 176 + + +[ 2023-09-21 11:55:19 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 176 from training step 0 + +[ 2023-09-21 11:55:22 ] Completed train batch 0 to device 3,184.023 ms, 3.18 s total +[ 2023-09-21 11:55:22 ] Completed train batch 0 generator forward 29.091 ms, 3.21 s total +[ 2023-09-21 11:55:22 ] Completed train batch 0 generator loss: 1.190 66.950 ms, 3.28 s total +[ 2023-09-21 11:55:22 ] Completed train batch 0 generator backward 551.713 ms, 3.83 s total +[ 2023-09-21 11:55:22 ] Completed train batch 0 discriminator loss 0.041 57.718 ms, 3.89 s total +[ 2023-09-21 11:55:23 ] Completed train batch 0 discriminator backward 104.326 ms, 3.99 s total +Epoch [176] Step [0/2], gen_loss: 0.414, disc_loss: 0.014 +[ 2023-09-21 11:55:23 ] Completed train batch 0 metrics update 0.768 ms, 3.99 s total +Saving checkpoint at epoch 176 train batch 0 +[ 2023-09-21 11:55:23 ] Completed saving temp checkpoint 530.215 ms, 4.52 s total +[ 2023-09-21 11:55:23 ] Completed replacing temp checkpoint with checkpoint 51.631 ms, 4.58 s total +[ 2023-09-21 11:55:25 ] Completed train batch 1 to device 1,584.070 ms, 6.16 s total +[ 2023-09-21 11:55:25 ] Completed train batch 1 generator forward 22.577 ms, 6.18 s total +[ 2023-09-21 11:55:25 ] Completed train batch 1 generator loss: 1.222 63.747 ms, 6.25 s total +[ 2023-09-21 11:55:26 ] Completed train batch 1 generator backward 1,224.750 ms, 7.47 s total +[ 2023-09-21 11:55:26 ] Completed train batch 1 discriminator loss 0.044 56.647 ms, 7.53 s total +[ 2023-09-21 11:55:26 ] Completed train batch 1 discriminator backward 81.077 ms, 7.61 s total +Epoch [176] Step [1/2], gen_loss: 0.411, disc_loss: 0.015 +[ 2023-09-21 11:55:26 ] Completed train batch 1 metrics update 3.098 ms, 7.61 s total +Saving checkpoint at epoch 176 train batch 1 +[ 2023-09-21 11:55:27 ] Completed saving temp checkpoint 434.500 ms, 8.05 s total +[ 2023-09-21 11:55:27 ] Completed replacing temp checkpoint with checkpoint 21.815 ms, 8.07 s total +Epoch [176] :: gen_loss: 0.412, disc_loss: 0.014 +[ 2023-09-21 11:55:27 ] Completed training generator for epoch 176 51.041 ms, 8.12 s total +[ 2023-09-21 11:55:27 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 176 from eval step 0 + +[ 2023-09-21 11:55:28 ] Completed eval batch 0 to device 1,155.664 ms, 1.16 s total +[ 2023-09-21 11:55:28 ] Completed eval batch 0 forward 32.331 ms, 1.19 s total +[ 2023-09-21 11:55:28 ] Completed eval batch 0 recons_loss 0.342 ms, 1.19 s total +[ 2023-09-21 11:55:28 ] Completed eval batch 0 metrics update 0.585 ms, 1.19 s total +Saving checkpoint at epoch 176 val batch 0 +[ 2023-09-21 11:55:28 ] Completed saving temp checkpoint 450.520 ms, 1.64 s total +[ 2023-09-21 11:55:28 ] Completed replacing temp checkpoint with checkpoint 32.414 ms, 1.67 s total +[ 2023-09-21 11:55:29 ] Completed eval batch 1 to device 514.295 ms, 2.19 s total +[ 2023-09-21 11:55:29 ] Completed eval batch 1 forward 40.463 ms, 2.23 s total +[ 2023-09-21 11:55:29 ] Completed eval batch 1 recons_loss 0.629 ms, 2.23 s total +[ 2023-09-21 11:55:29 ] Completed eval batch 1 metrics update 67.697 ms, 2.29 s total +Saving checkpoint at epoch 176 val batch 1 +Epoch 176 val loss: 0.0547 +[ 2023-09-21 11:55:29 ] Completed saving temp checkpoint 434.090 ms, 2.73 s total +[ 2023-09-21 11:55:29 ] Completed replacing temp checkpoint with checkpoint 23.422 ms, 2.75 s total +[ 2023-09-21 11:55:29 ] Completed evaluating generator for epoch 176 46.398 ms, 2.80 s total + + +EPOCH :: 177 + + +[ 2023-09-21 11:55:29 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 177 from training step 0 + +[ 2023-09-21 11:55:33 ] Completed train batch 0 to device 3,153.221 ms, 3.15 s total +[ 2023-09-21 11:55:33 ] Completed train batch 0 generator forward 28.456 ms, 3.18 s total +[ 2023-09-21 11:55:33 ] Completed train batch 0 generator loss: 1.243 70.794 ms, 3.25 s total +[ 2023-09-21 11:55:33 ] Completed train batch 0 generator backward 653.857 ms, 3.91 s total +[ 2023-09-21 11:55:33 ] Completed train batch 0 discriminator loss 0.047 57.252 ms, 3.96 s total +[ 2023-09-21 11:55:34 ] Completed train batch 0 discriminator backward 102.380 ms, 4.07 s total +Epoch [177] Step [0/2], gen_loss: 0.412, disc_loss: 0.015 +[ 2023-09-21 11:55:34 ] Completed train batch 0 metrics update 7.791 ms, 4.07 s total +Saving checkpoint at epoch 177 train batch 0 +[ 2023-09-21 11:55:34 ] Completed saving temp checkpoint 445.893 ms, 4.52 s total +[ 2023-09-21 11:55:34 ] Completed replacing temp checkpoint with checkpoint 21.854 ms, 4.54 s total +[ 2023-09-21 11:55:36 ] Completed train batch 1 to device 1,543.488 ms, 6.08 s total +[ 2023-09-21 11:55:36 ] Completed train batch 1 generator forward 41.575 ms, 6.13 s total +[ 2023-09-21 11:55:36 ] Completed train batch 1 generator loss: 1.186 65.806 ms, 6.19 s total +[ 2023-09-21 11:55:37 ] Completed train batch 1 generator backward 1,481.731 ms, 7.67 s total +[ 2023-09-21 11:55:37 ] Completed train batch 1 discriminator loss 0.046 56.613 ms, 7.73 s total +[ 2023-09-21 11:55:37 ] Completed train batch 1 discriminator backward 74.444 ms, 7.81 s total +Epoch [177] Step [1/2], gen_loss: 0.410, disc_loss: 0.015 +[ 2023-09-21 11:55:37 ] Completed train batch 1 metrics update 0.645 ms, 7.81 s total +Saving checkpoint at epoch 177 train batch 1 +[ 2023-09-21 11:55:38 ] Completed saving temp checkpoint 500.971 ms, 8.31 s total +[ 2023-09-21 11:55:38 ] Completed replacing temp checkpoint with checkpoint 22.579 ms, 8.33 s total +Epoch [177] :: gen_loss: 0.411, disc_loss: 0.015 +[ 2023-09-21 11:55:38 ] Completed training generator for epoch 177 50.330 ms, 8.38 s total +[ 2023-09-21 11:55:38 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 177 from eval step 0 + +[ 2023-09-21 11:55:39 ] Completed eval batch 0 to device 1,141.644 ms, 1.14 s total +[ 2023-09-21 11:55:39 ] Completed eval batch 0 forward 26.630 ms, 1.17 s total +[ 2023-09-21 11:55:39 ] Completed eval batch 0 recons_loss 0.285 ms, 1.17 s total +[ 2023-09-21 11:55:39 ] Completed eval batch 0 metrics update 0.537 ms, 1.17 s total +Saving checkpoint at epoch 177 val batch 0 +[ 2023-09-21 11:55:39 ] Completed saving temp checkpoint 427.430 ms, 1.60 s total +[ 2023-09-21 11:55:39 ] Completed replacing temp checkpoint with checkpoint 16.816 ms, 1.61 s total +[ 2023-09-21 11:55:40 ] Completed eval batch 1 to device 493.746 ms, 2.11 s total +[ 2023-09-21 11:55:40 ] Completed eval batch 1 forward 22.222 ms, 2.13 s total +[ 2023-09-21 11:55:40 ] Completed eval batch 1 recons_loss 0.388 ms, 2.13 s total +[ 2023-09-21 11:55:40 ] Completed eval batch 1 metrics update 98.831 ms, 2.23 s total +Saving checkpoint at epoch 177 val batch 1 +Epoch 177 val loss: 0.0603 +[ 2023-09-21 11:55:40 ] Completed saving temp checkpoint 405.121 ms, 2.63 s total +[ 2023-09-21 11:55:41 ] Completed replacing temp checkpoint with checkpoint 23.565 ms, 2.66 s total +[ 2023-09-21 11:55:41 ] Completed evaluating generator for epoch 177 45.899 ms, 2.70 s total + + +EPOCH :: 178 + + +[ 2023-09-21 11:55:41 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 178 from training step 0 + +[ 2023-09-21 11:55:43 ] Completed train batch 0 to device 2,822.681 ms, 2.82 s total +[ 2023-09-21 11:55:43 ] Completed train batch 0 generator forward 25.290 ms, 2.85 s total +[ 2023-09-21 11:55:43 ] Completed train batch 0 generator loss: 1.255 64.436 ms, 2.91 s total +[ 2023-09-21 11:55:44 ] Completed train batch 0 generator backward 951.519 ms, 3.86 s total +[ 2023-09-21 11:55:44 ] Completed train batch 0 discriminator loss 0.043 56.639 ms, 3.92 s total +[ 2023-09-21 11:55:45 ] Completed train batch 0 discriminator backward 100.051 ms, 4.02 s total +Epoch [178] Step [0/2], gen_loss: 0.414, disc_loss: 0.015 +[ 2023-09-21 11:55:45 ] Completed train batch 0 metrics update 5.685 ms, 4.03 s total +Saving checkpoint at epoch 178 train batch 0 +[ 2023-09-21 11:55:45 ] Completed saving temp checkpoint 422.428 ms, 4.45 s total +[ 2023-09-21 11:55:45 ] Completed replacing temp checkpoint with checkpoint 22.470 ms, 4.47 s total +[ 2023-09-21 11:55:46 ] Completed train batch 1 to device 1,124.516 ms, 5.60 s total +[ 2023-09-21 11:55:46 ] Completed train batch 1 generator forward 40.621 ms, 5.64 s total +[ 2023-09-21 11:55:46 ] Completed train batch 1 generator loss: 1.147 62.268 ms, 5.70 s total +[ 2023-09-21 11:55:48 ] Completed train batch 1 generator backward 1,337.882 ms, 7.04 s total +[ 2023-09-21 11:55:48 ] Completed train batch 1 discriminator loss 0.040 56.477 ms, 7.09 s total +[ 2023-09-21 11:55:48 ] Completed train batch 1 discriminator backward 72.523 ms, 7.17 s total +Epoch [178] Step [1/2], gen_loss: 0.406, disc_loss: 0.014 +[ 2023-09-21 11:55:48 ] Completed train batch 1 metrics update 1.318 ms, 7.17 s total +Saving checkpoint at epoch 178 train batch 1 +[ 2023-09-21 11:55:48 ] Completed saving temp checkpoint 451.931 ms, 7.62 s total +[ 2023-09-21 11:55:48 ] Completed replacing temp checkpoint with checkpoint 23.515 ms, 7.64 s total +Epoch [178] :: gen_loss: 0.410, disc_loss: 0.014 +[ 2023-09-21 11:55:48 ] Completed training generator for epoch 178 49.204 ms, 7.69 s total +[ 2023-09-21 11:55:48 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 178 from eval step 0 + +[ 2023-09-21 11:55:49 ] Completed eval batch 0 to device 1,048.537 ms, 1.05 s total +[ 2023-09-21 11:55:49 ] Completed eval batch 0 forward 29.968 ms, 1.08 s total +[ 2023-09-21 11:55:49 ] Completed eval batch 0 recons_loss 0.231 ms, 1.08 s total +[ 2023-09-21 11:55:49 ] Completed eval batch 0 metrics update 0.508 ms, 1.08 s total +Saving checkpoint at epoch 178 val batch 0 +[ 2023-09-21 11:55:50 ] Completed saving temp checkpoint 400.022 ms, 1.48 s total +[ 2023-09-21 11:55:50 ] Completed replacing temp checkpoint with checkpoint 30.433 ms, 1.51 s total +[ 2023-09-21 11:55:50 ] Completed eval batch 1 to device 545.215 ms, 2.05 s total +[ 2023-09-21 11:55:50 ] Completed eval batch 1 forward 40.308 ms, 2.10 s total +[ 2023-09-21 11:55:50 ] Completed eval batch 1 recons_loss 0.579 ms, 2.10 s total +[ 2023-09-21 11:55:50 ] Completed eval batch 1 metrics update 130.806 ms, 2.23 s total +Saving checkpoint at epoch 178 val batch 1 +Epoch 178 val loss: 0.0627 +[ 2023-09-21 11:55:51 ] Completed saving temp checkpoint 441.129 ms, 2.67 s total +[ 2023-09-21 11:55:51 ] Completed replacing temp checkpoint with checkpoint 22.438 ms, 2.69 s total +[ 2023-09-21 11:55:51 ] Completed evaluating generator for epoch 178 44.152 ms, 2.73 s total + + +EPOCH :: 179 + + +[ 2023-09-21 11:55:51 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 179 from training step 0 + +[ 2023-09-21 11:55:54 ] Completed train batch 0 to device 3,120.576 ms, 3.12 s total +[ 2023-09-21 11:55:54 ] Completed train batch 0 generator forward 27.411 ms, 3.15 s total +[ 2023-09-21 11:55:54 ] Completed train batch 0 generator loss: 1.361 67.896 ms, 3.22 s total +[ 2023-09-21 11:55:55 ] Completed train batch 0 generator backward 602.768 ms, 3.82 s total +[ 2023-09-21 11:55:55 ] Completed train batch 0 discriminator loss 0.044 76.709 ms, 3.90 s total +[ 2023-09-21 11:55:55 ] Completed train batch 0 discriminator backward 68.586 ms, 3.96 s total +Epoch [179] Step [0/2], gen_loss: 0.420, disc_loss: 0.016 +[ 2023-09-21 11:55:55 ] Completed train batch 0 metrics update 3.663 ms, 3.97 s total +Saving checkpoint at epoch 179 train batch 0 +[ 2023-09-21 11:55:55 ] Completed saving temp checkpoint 468.887 ms, 4.44 s total +[ 2023-09-21 11:55:55 ] Completed replacing temp checkpoint with checkpoint 23.875 ms, 4.46 s total +[ 2023-09-21 11:55:57 ] Completed train batch 1 to device 1,534.235 ms, 5.99 s total +[ 2023-09-21 11:55:57 ] Completed train batch 1 generator forward 24.919 ms, 6.02 s total +[ 2023-09-21 11:55:57 ] Completed train batch 1 generator loss: 1.214 64.782 ms, 6.08 s total +[ 2023-09-21 11:55:58 ] Completed train batch 1 generator backward 1,100.762 ms, 7.19 s total +[ 2023-09-21 11:55:58 ] Completed train batch 1 discriminator loss 0.047 57.867 ms, 7.24 s total +[ 2023-09-21 11:55:58 ] Completed train batch 1 discriminator backward 61.491 ms, 7.30 s total +Epoch [179] Step [1/2], gen_loss: 0.409, disc_loss: 0.016 +[ 2023-09-21 11:55:58 ] Completed train batch 1 metrics update 1.742 ms, 7.31 s total +Saving checkpoint at epoch 179 train batch 1 +[ 2023-09-21 11:55:59 ] Completed saving temp checkpoint 438.934 ms, 7.75 s total +[ 2023-09-21 11:55:59 ] Completed replacing temp checkpoint with checkpoint 20.262 ms, 7.77 s total +Epoch [179] :: gen_loss: 0.415, disc_loss: 0.016 +[ 2023-09-21 11:55:59 ] Completed training generator for epoch 179 46.613 ms, 7.81 s total +[ 2023-09-21 11:55:59 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 179 from eval step 0 + +[ 2023-09-21 11:56:00 ] Completed eval batch 0 to device 1,105.376 ms, 1.11 s total +[ 2023-09-21 11:56:00 ] Completed eval batch 0 forward 40.177 ms, 1.15 s total +[ 2023-09-21 11:56:00 ] Completed eval batch 0 recons_loss 0.290 ms, 1.15 s total +[ 2023-09-21 11:56:00 ] Completed eval batch 0 metrics update 0.574 ms, 1.15 s total +Saving checkpoint at epoch 179 val batch 0 +[ 2023-09-21 11:56:00 ] Completed saving temp checkpoint 413.616 ms, 1.56 s total +[ 2023-09-21 11:56:00 ] Completed replacing temp checkpoint with checkpoint 28.615 ms, 1.59 s total +[ 2023-09-21 11:56:01 ] Completed eval batch 1 to device 410.275 ms, 2.00 s total +[ 2023-09-21 11:56:01 ] Completed eval batch 1 forward 38.299 ms, 2.04 s total +[ 2023-09-21 11:56:01 ] Completed eval batch 1 recons_loss 0.336 ms, 2.04 s total +[ 2023-09-21 11:56:01 ] Completed eval batch 1 metrics update 214.229 ms, 2.25 s total +Saving checkpoint at epoch 179 val batch 1 +Epoch 179 val loss: 0.0540 +[ 2023-09-21 11:56:02 ] Completed saving temp checkpoint 491.611 ms, 2.74 s total +[ 2023-09-21 11:56:02 ] Completed replacing temp checkpoint with checkpoint 28.067 ms, 2.77 s total +[ 2023-09-21 11:56:02 ] Completed evaluating generator for epoch 179 46.420 ms, 2.82 s total + + +EPOCH :: 180 + + +[ 2023-09-21 11:56:02 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 180 from training step 0 + +[ 2023-09-21 11:56:05 ] Completed train batch 0 to device 3,051.947 ms, 3.05 s total +[ 2023-09-21 11:56:05 ] Completed train batch 0 generator forward 42.083 ms, 3.09 s total +[ 2023-09-21 11:56:05 ] Completed train batch 0 generator loss: 1.235 64.929 ms, 3.16 s total +[ 2023-09-21 11:56:05 ] Completed train batch 0 generator backward 602.480 ms, 3.76 s total +[ 2023-09-21 11:56:05 ] Completed train batch 0 discriminator loss 0.052 56.642 ms, 3.82 s total +[ 2023-09-21 11:56:06 ] Completed train batch 0 discriminator backward 96.670 ms, 3.91 s total +Epoch [180] Step [0/2], gen_loss: 0.416, disc_loss: 0.016 +[ 2023-09-21 11:56:06 ] Completed train batch 0 metrics update 5.926 ms, 3.92 s total +Saving checkpoint at epoch 180 train batch 0 +[ 2023-09-21 11:56:06 ] Completed saving temp checkpoint 385.767 ms, 4.31 s total +[ 2023-09-21 11:56:06 ] Completed replacing temp checkpoint with checkpoint 27.090 ms, 4.33 s total +[ 2023-09-21 11:56:07 ] Completed train batch 1 to device 1,502.097 ms, 5.84 s total +[ 2023-09-21 11:56:07 ] Completed train batch 1 generator forward 41.273 ms, 5.88 s total +[ 2023-09-21 11:56:08 ] Completed train batch 1 generator loss: 1.205 63.900 ms, 5.94 s total +[ 2023-09-21 11:56:09 ] Completed train batch 1 generator backward 1,008.468 ms, 6.95 s total +[ 2023-09-21 11:56:09 ] Completed train batch 1 discriminator loss 0.043 56.410 ms, 7.01 s total +[ 2023-09-21 11:56:09 ] Completed train batch 1 discriminator backward 75.282 ms, 7.08 s total +Epoch [180] Step [1/2], gen_loss: 0.408, disc_loss: 0.015 +[ 2023-09-21 11:56:09 ] Completed train batch 1 metrics update 1.516 ms, 7.08 s total +Saving checkpoint at epoch 180 train batch 1 +[ 2023-09-21 11:56:09 ] Completed saving temp checkpoint 384.637 ms, 7.47 s total +[ 2023-09-21 11:56:09 ] Completed replacing temp checkpoint with checkpoint 24.087 ms, 7.49 s total +Epoch [180] :: gen_loss: 0.412, disc_loss: 0.016 +[ 2023-09-21 11:56:09 ] Completed training generator for epoch 180 51.234 ms, 7.54 s total +[ 2023-09-21 11:56:09 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 180 from eval step 0 + +[ 2023-09-21 11:56:10 ] Completed eval batch 0 to device 1,031.088 ms, 1.03 s total +[ 2023-09-21 11:56:10 ] Completed eval batch 0 forward 40.342 ms, 1.07 s total +[ 2023-09-21 11:56:10 ] Completed eval batch 0 recons_loss 0.261 ms, 1.07 s total +[ 2023-09-21 11:56:10 ] Completed eval batch 0 metrics update 100.407 ms, 1.17 s total +Saving checkpoint at epoch 180 val batch 0 +[ 2023-09-21 11:56:11 ] Completed saving temp checkpoint 409.334 ms, 1.58 s total +[ 2023-09-21 11:56:11 ] Completed replacing temp checkpoint with checkpoint 25.429 ms, 1.61 s total +[ 2023-09-21 11:56:11 ] Completed eval batch 1 to device 346.283 ms, 1.95 s total +[ 2023-09-21 11:56:11 ] Completed eval batch 1 forward 41.030 ms, 1.99 s total +[ 2023-09-21 11:56:11 ] Completed eval batch 1 recons_loss 0.527 ms, 1.99 s total +[ 2023-09-21 11:56:12 ] Completed eval batch 1 metrics update 386.130 ms, 2.38 s total +Saving checkpoint at epoch 180 val batch 1 +Epoch 180 val loss: 0.0587 +[ 2023-09-21 11:56:12 ] Completed saving temp checkpoint 426.155 ms, 2.81 s total +[ 2023-09-21 11:56:12 ] Completed replacing temp checkpoint with checkpoint 32.344 ms, 2.84 s total +[ 2023-09-21 11:56:12 ] Completed evaluating generator for epoch 180 48.435 ms, 2.89 s total + + +EPOCH :: 181 + + +[ 2023-09-21 11:56:12 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 181 from training step 0 + +[ 2023-09-21 11:56:15 ] Completed train batch 0 to device 3,264.514 ms, 3.26 s total +[ 2023-09-21 11:56:15 ] Completed train batch 0 generator forward 43.745 ms, 3.31 s total +[ 2023-09-21 11:56:15 ] Completed train batch 0 generator loss: 1.195 66.612 ms, 3.37 s total +[ 2023-09-21 11:56:16 ] Completed train batch 0 generator backward 523.194 ms, 3.90 s total +[ 2023-09-21 11:56:16 ] Completed train batch 0 discriminator loss 0.044 59.108 ms, 3.96 s total +[ 2023-09-21 11:56:16 ] Completed train batch 0 discriminator backward 90.093 ms, 4.05 s total +Epoch [181] Step [0/2], gen_loss: 0.407, disc_loss: 0.015 +[ 2023-09-21 11:56:16 ] Completed train batch 0 metrics update 17.860 ms, 4.07 s total +Saving checkpoint at epoch 181 train batch 0 +[ 2023-09-21 11:56:17 ] Completed saving temp checkpoint 460.309 ms, 4.53 s total +[ 2023-09-21 11:56:17 ] Completed replacing temp checkpoint with checkpoint 28.818 ms, 4.55 s total +[ 2023-09-21 11:56:18 ] Completed train batch 1 to device 1,752.306 ms, 6.31 s total +[ 2023-09-21 11:56:18 ] Completed train batch 1 generator forward 22.534 ms, 6.33 s total +[ 2023-09-21 11:56:18 ] Completed train batch 1 generator loss: 1.209 63.508 ms, 6.39 s total +[ 2023-09-21 11:56:20 ] Completed train batch 1 generator backward 1,226.040 ms, 7.62 s total +[ 2023-09-21 11:56:20 ] Completed train batch 1 discriminator loss 0.044 56.978 ms, 7.68 s total +[ 2023-09-21 11:56:20 ] Completed train batch 1 discriminator backward 58.997 ms, 7.73 s total +Epoch [181] Step [1/2], gen_loss: 0.412, disc_loss: 0.015 +[ 2023-09-21 11:56:20 ] Completed train batch 1 metrics update 5.495 ms, 7.74 s total +Saving checkpoint at epoch 181 train batch 1 +[ 2023-09-21 11:56:20 ] Completed saving temp checkpoint 399.986 ms, 8.14 s total +[ 2023-09-21 11:56:20 ] Completed replacing temp checkpoint with checkpoint 15.908 ms, 8.16 s total +Epoch [181] :: gen_loss: 0.410, disc_loss: 0.015 +[ 2023-09-21 11:56:20 ] Completed training generator for epoch 181 47.869 ms, 8.20 s total +[ 2023-09-21 11:56:20 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 181 from eval step 0 + +[ 2023-09-21 11:56:21 ] Completed eval batch 0 to device 1,191.484 ms, 1.19 s total +[ 2023-09-21 11:56:21 ] Completed eval batch 0 forward 23.989 ms, 1.22 s total +[ 2023-09-21 11:56:21 ] Completed eval batch 0 recons_loss 0.233 ms, 1.22 s total +[ 2023-09-21 11:56:21 ] Completed eval batch 0 metrics update 0.535 ms, 1.22 s total +Saving checkpoint at epoch 181 val batch 0 +[ 2023-09-21 11:56:22 ] Completed saving temp checkpoint 414.483 ms, 1.63 s total +[ 2023-09-21 11:56:22 ] Completed replacing temp checkpoint with checkpoint 35.669 ms, 1.67 s total +[ 2023-09-21 11:56:22 ] Completed eval batch 1 to device 467.453 ms, 2.13 s total +[ 2023-09-21 11:56:22 ] Completed eval batch 1 forward 40.327 ms, 2.17 s total +[ 2023-09-21 11:56:22 ] Completed eval batch 1 recons_loss 0.565 ms, 2.17 s total +[ 2023-09-21 11:56:23 ] Completed eval batch 1 metrics update 219.018 ms, 2.39 s total +Saving checkpoint at epoch 181 val batch 1 +Epoch 181 val loss: 0.0640 +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 12:08:56 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 12:08:56 ] Completed importing Timer 0.026 ms, 0.00 s total +| distributed init (rank 1): env:// +| distributed init (rank 2): env:// +| distributed init (rank 5): env:// +| distributed init (rank 4): env:// +| distributed init (rank 3): env:// +[ 2023-09-21 12:09:05 ] Completed importing everything else 9,238.949 ms, 9.24 s total +| distributed init (rank 0): env:// +[ 2023-09-21 12:09:08 ] Completed preliminaries 3,502.036 ms, 12.74 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 12:09:08 ] Completed build datasets 10.847 ms, 12.75 s total +[ 2023-09-21 12:09:08 ] Completed build samplers 0.047 ms, 12.75 s total +[ 2023-09-21 12:09:08 ] Completed build dataloaders 0.133 ms, 12.75 s total +[ 2023-09-21 12:09:09 ] Completed generator to device 77.193 ms, 12.83 s total +[ 2023-09-21 12:09:09 ] Completed discriminator to device 7.026 ms, 12.84 s total +[ 2023-09-21 12:09:09 ] Completed loss functions 408.844 ms, 13.25 s total +[ 2023-09-21 12:09:09 ] Completed models prepped for distribution 85.618 ms, 13.33 s total +[ 2023-09-21 12:09:09 ] Completed optimizers 0.674 ms, 13.33 s total +[ 2023-09-21 12:09:09 ] Completed grad scalers 0.029 ms, 13.33 s total +[ 2023-09-21 12:09:10 ] Completed checkpoint retrieval 484.520 ms, 13.82 s total + + +EPOCH :: 181 + + +[ 2023-09-21 12:09:10 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 181 from training step 2 + +Epoch [181] :: gen_loss: 0.410, disc_loss: 0.015 +[ 2023-09-21 12:09:10 ] Completed training generator for epoch 181 134.924 ms, 0.13 s total +[ 2023-09-21 12:09:10 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 181 from eval step 1 + +[ 2023-09-21 12:09:11 ] Completed eval batch 1 to device 1,253.566 ms, 1.25 s total +[ 2023-09-21 12:09:12 ] Completed eval batch 1 forward 1,040.842 ms, 2.29 s total +[ 2023-09-21 12:09:12 ] Completed eval batch 1 recons_loss 0.272 ms, 2.29 s total +[ 2023-09-21 12:09:13 ] Completed eval batch 1 metrics update 785.372 ms, 3.08 s total +Saving checkpoint at epoch 181 val batch 1 +Epoch 181 val loss: 0.0630 +[ 2023-09-21 12:09:13 ] Completed saving temp checkpoint 416.044 ms, 3.50 s total +[ 2023-09-21 12:09:13 ] Completed replacing temp checkpoint with checkpoint 143.875 ms, 3.64 s total +[ 2023-09-21 12:09:13 ] Completed evaluating generator for epoch 181 38.465 ms, 3.68 s total + + +EPOCH :: 182 + + +[ 2023-09-21 12:09:13 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 182 from training step 0 + +[ 2023-09-21 12:09:16 ] Completed train batch 0 to device 2,893.146 ms, 2.89 s total +[ 2023-09-21 12:09:16 ] Completed train batch 0 generator forward 46.166 ms, 2.94 s total +[ 2023-09-21 12:09:16 ] Completed train batch 0 generator loss: 1.259 80.743 ms, 3.02 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 12:09:17 ] Completed train batch 0 generator backward 932.663 ms, 3.95 s total +[ 2023-09-21 12:09:17 ] Completed train batch 0 discriminator loss 0.042 85.500 ms, 4.04 s total +[ 2023-09-21 12:09:17 ] Completed train batch 0 discriminator backward 89.178 ms, 4.13 s total +Epoch [182] Step [0/2], gen_loss: 0.420, disc_loss: 0.015 +[ 2023-09-21 12:09:17 ] Completed train batch 0 metrics update 8.357 ms, 4.14 s total +Saving checkpoint at epoch 182 train batch 0 +[ 2023-09-21 12:09:18 ] Completed saving temp checkpoint 366.287 ms, 4.50 s total +[ 2023-09-21 12:09:18 ] Completed replacing temp checkpoint with checkpoint 19.085 ms, 4.52 s total +[ 2023-09-21 12:09:19 ] Completed train batch 1 to device 1,059.093 ms, 5.58 s total +[ 2023-09-21 12:09:19 ] Completed train batch 1 generator forward 26.755 ms, 5.61 s total +[ 2023-09-21 12:09:19 ] Completed train batch 1 generator loss: 1.223 63.740 ms, 5.67 s total +[ 2023-09-21 12:09:21 ] Completed train batch 1 generator backward 1,560.909 ms, 7.23 s total +[ 2023-09-21 12:09:21 ] Completed train batch 1 discriminator loss 0.045 56.066 ms, 7.29 s total +[ 2023-09-21 12:09:21 ] Completed train batch 1 discriminator backward 74.776 ms, 7.36 s total +Epoch [182] Step [1/2], gen_loss: 0.415, disc_loss: 0.014 +[ 2023-09-21 12:09:21 ] Completed train batch 1 metrics update 5.025 ms, 7.37 s total +Saving checkpoint at epoch 182 train batch 1 +[ 2023-09-21 12:09:21 ] Completed saving temp checkpoint 454.935 ms, 7.82 s total +[ 2023-09-21 12:09:21 ] Completed replacing temp checkpoint with checkpoint 22.081 ms, 7.84 s total +Epoch [182] :: gen_loss: 0.417, disc_loss: 0.015 +[ 2023-09-21 12:09:21 ] Completed training generator for epoch 182 55.383 ms, 7.90 s total +[ 2023-09-21 12:09:21 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 182 from eval step 0 + +[ 2023-09-21 12:09:22 ] Completed eval batch 0 to device 1,103.286 ms, 1.10 s total +[ 2023-09-21 12:09:22 ] Completed eval batch 0 forward 41.576 ms, 1.14 s total +[ 2023-09-21 12:09:22 ] Completed eval batch 0 recons_loss 0.269 ms, 1.15 s total +[ 2023-09-21 12:09:22 ] Completed eval batch 0 metrics update 0.541 ms, 1.15 s total +Saving checkpoint at epoch 182 val batch 0 +[ 2023-09-21 12:09:23 ] Completed saving temp checkpoint 377.670 ms, 1.52 s total +[ 2023-09-21 12:09:23 ] Completed replacing temp checkpoint with checkpoint 21.492 ms, 1.54 s total +[ 2023-09-21 12:09:23 ] Completed eval batch 1 to device 427.531 ms, 1.97 s total +[ 2023-09-21 12:09:23 ] Completed eval batch 1 forward 21.251 ms, 1.99 s total +[ 2023-09-21 12:09:23 ] Completed eval batch 1 recons_loss 0.332 ms, 1.99 s total +[ 2023-09-21 12:09:24 ] Completed eval batch 1 metrics update 250.768 ms, 2.24 s total +Saving checkpoint at epoch 182 val batch 1 +Epoch 182 val loss: 0.0538 +[ 2023-09-21 12:09:24 ] Completed saving temp checkpoint 449.917 ms, 2.69 s total +[ 2023-09-21 12:09:24 ] Completed replacing temp checkpoint with checkpoint 29.528 ms, 2.72 s total +[ 2023-09-21 12:09:24 ] Completed evaluating generator for epoch 182 50.438 ms, 2.77 s total + + +EPOCH :: 183 + + +[ 2023-09-21 12:09:24 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 183 from training step 0 + +[ 2023-09-21 12:09:27 ] Completed train batch 0 to device 3,153.371 ms, 3.15 s total +[ 2023-09-21 12:09:27 ] Completed train batch 0 generator forward 25.154 ms, 3.18 s total +[ 2023-09-21 12:09:27 ] Completed train batch 0 generator loss: 1.243 64.513 ms, 3.24 s total +[ 2023-09-21 12:09:28 ] Completed train batch 0 generator backward 628.141 ms, 3.87 s total +[ 2023-09-21 12:09:28 ] Completed train batch 0 discriminator loss 0.046 59.483 ms, 3.93 s total +[ 2023-09-21 12:09:28 ] Completed train batch 0 discriminator backward 101.156 ms, 4.03 s total +Epoch [183] Step [0/2], gen_loss: 0.406, disc_loss: 0.014 +[ 2023-09-21 12:09:28 ] Completed train batch 0 metrics update 4.840 ms, 4.04 s total +Saving checkpoint at epoch 183 train batch 0 +[ 2023-09-21 12:09:29 ] Completed saving temp checkpoint 452.494 ms, 4.49 s total +[ 2023-09-21 12:09:29 ] Completed replacing temp checkpoint with checkpoint 24.637 ms, 4.51 s total +[ 2023-09-21 12:09:30 ] Completed train batch 1 to device 1,665.051 ms, 6.18 s total +[ 2023-09-21 12:09:30 ] Completed train batch 1 generator forward 22.292 ms, 6.20 s total +[ 2023-09-21 12:09:30 ] Completed train batch 1 generator loss: 1.231 62.524 ms, 6.26 s total +[ 2023-09-21 12:09:31 ] Completed train batch 1 generator backward 1,162.524 ms, 7.43 s total +[ 2023-09-21 12:09:32 ] Completed train batch 1 discriminator loss 0.042 55.731 ms, 7.48 s total +[ 2023-09-21 12:09:32 ] Completed train batch 1 discriminator backward 81.581 ms, 7.56 s total +Epoch [183] Step [1/2], gen_loss: 0.410, disc_loss: 0.014 +[ 2023-09-21 12:09:32 ] Completed train batch 1 metrics update 5.386 ms, 7.57 s total +Saving checkpoint at epoch 183 train batch 1 +[ 2023-09-21 12:09:32 ] Completed saving temp checkpoint 605.962 ms, 8.17 s total +[ 2023-09-21 12:09:32 ] Completed replacing temp checkpoint with checkpoint 36.372 ms, 8.21 s total +Epoch [183] :: gen_loss: 0.408, disc_loss: 0.014 +[ 2023-09-21 12:09:32 ] Completed training generator for epoch 183 69.601 ms, 8.28 s total +[ 2023-09-21 12:09:32 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 183 from eval step 0 + +[ 2023-09-21 12:09:33 ] Completed eval batch 0 to device 1,015.590 ms, 1.02 s total +[ 2023-09-21 12:09:33 ] Completed eval batch 0 forward 32.649 ms, 1.05 s total +[ 2023-09-21 12:09:33 ] Completed eval batch 0 recons_loss 0.515 ms, 1.05 s total +[ 2023-09-21 12:09:33 ] Completed eval batch 0 metrics update 0.899 ms, 1.05 s total +Saving checkpoint at epoch 183 val batch 0 +[ 2023-09-21 12:09:34 ] Completed saving temp checkpoint 705.208 ms, 1.75 s total +[ 2023-09-21 12:09:34 ] Completed replacing temp checkpoint with checkpoint 39.912 ms, 1.79 s total +[ 2023-09-21 12:09:34 ] Completed eval batch 1 to device 62.496 ms, 1.86 s total +[ 2023-09-21 12:09:34 ] Completed eval batch 1 forward 21.572 ms, 1.88 s total +[ 2023-09-21 12:09:34 ] Completed eval batch 1 recons_loss 0.328 ms, 1.88 s total +[ 2023-09-21 12:09:34 ] Completed eval batch 1 metrics update 222.138 ms, 2.10 s total +Saving checkpoint at epoch 183 val batch 1 +Epoch 183 val loss: 0.0536 +[ 2023-09-21 12:09:35 ] Completed saving temp checkpoint 718.259 ms, 2.82 s total +[ 2023-09-21 12:09:35 ] Completed replacing temp checkpoint with checkpoint 32.616 ms, 2.85 s total +[ 2023-09-21 12:09:35 ] Completed evaluating generator for epoch 183 53.109 ms, 2.91 s total + + +EPOCH :: 184 + + +[ 2023-09-21 12:09:35 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 184 from training step 0 + +[ 2023-09-21 12:09:38 ] Completed train batch 0 to device 3,056.016 ms, 3.06 s total +[ 2023-09-21 12:09:38 ] Completed train batch 0 generator forward 27.357 ms, 3.08 s total +[ 2023-09-21 12:09:38 ] Completed train batch 0 generator loss: 1.255 65.893 ms, 3.15 s total +[ 2023-09-21 12:09:39 ] Completed train batch 0 generator backward 434.521 ms, 3.58 s total +[ 2023-09-21 12:09:39 ] Completed train batch 0 discriminator loss 0.042 56.820 ms, 3.64 s total +[ 2023-09-21 12:09:39 ] Completed train batch 0 discriminator backward 109.093 ms, 3.75 s total +Epoch [184] Step [0/2], gen_loss: 0.419, disc_loss: 0.014 +[ 2023-09-21 12:09:39 ] Completed train batch 0 metrics update 26.213 ms, 3.78 s total +Saving checkpoint at epoch 184 train batch 0 +[ 2023-09-21 12:09:40 ] Completed saving temp checkpoint 660.412 ms, 4.44 s total +[ 2023-09-21 12:09:40 ] Completed replacing temp checkpoint with checkpoint 35.361 ms, 4.47 s total +[ 2023-09-21 12:09:41 ] Completed train batch 1 to device 1,447.497 ms, 5.92 s total +[ 2023-09-21 12:09:41 ] Completed train batch 1 generator forward 40.921 ms, 5.96 s total +[ 2023-09-21 12:09:41 ] Completed train batch 1 generator loss: 1.223 62.770 ms, 6.02 s total +[ 2023-09-21 12:09:42 ] Completed train batch 1 generator backward 1,141.387 ms, 7.16 s total +[ 2023-09-21 12:09:42 ] Completed train batch 1 discriminator loss 0.040 56.528 ms, 7.22 s total +[ 2023-09-21 12:09:43 ] Completed train batch 1 discriminator backward 78.499 ms, 7.30 s total +Epoch [184] Step [1/2], gen_loss: 0.413, disc_loss: 0.014 +[ 2023-09-21 12:09:43 ] Completed train batch 1 metrics update 5.011 ms, 7.30 s total +Saving checkpoint at epoch 184 train batch 1 +[ 2023-09-21 12:09:43 ] Completed saving temp checkpoint 642.069 ms, 7.95 s total +[ 2023-09-21 12:09:43 ] Completed replacing temp checkpoint with checkpoint 23.633 ms, 7.97 s total +Epoch [184] :: gen_loss: 0.416, disc_loss: 0.014 +[ 2023-09-21 12:09:43 ] Completed training generator for epoch 184 67.647 ms, 8.04 s total +[ 2023-09-21 12:09:43 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 184 from eval step 0 + +[ 2023-09-21 12:09:44 ] Completed eval batch 0 to device 1,047.085 ms, 1.05 s total +[ 2023-09-21 12:09:44 ] Completed eval batch 0 forward 28.804 ms, 1.08 s total +[ 2023-09-21 12:09:44 ] Completed eval batch 0 recons_loss 0.304 ms, 1.08 s total +[ 2023-09-21 12:09:44 ] Completed eval batch 0 metrics update 0.618 ms, 1.08 s total +Saving checkpoint at epoch 184 val batch 0 +[ 2023-09-21 12:09:45 ] Completed saving temp checkpoint 604.690 ms, 1.68 s total +[ 2023-09-21 12:09:45 ] Completed replacing temp checkpoint with checkpoint 40.234 ms, 1.72 s total +[ 2023-09-21 12:09:45 ] Completed eval batch 1 to device 194.263 ms, 1.92 s total +[ 2023-09-21 12:09:45 ] Completed eval batch 1 forward 39.394 ms, 1.96 s total +[ 2023-09-21 12:09:45 ] Completed eval batch 1 recons_loss 0.561 ms, 1.96 s total +[ 2023-09-21 12:09:45 ] Completed eval batch 1 metrics update 104.185 ms, 2.06 s total +Saving checkpoint at epoch 184 val batch 1 +Epoch 184 val loss: 0.0520 +[ 2023-09-21 12:09:46 ] Completed saving temp checkpoint 640.058 ms, 2.70 s total +[ 2023-09-21 12:09:46 ] Completed replacing temp checkpoint with checkpoint 42.253 ms, 2.74 s total +[ 2023-09-21 12:09:46 ] Completed evaluating generator for epoch 184 63.184 ms, 2.81 s total + + +EPOCH :: 185 + + +[ 2023-09-21 12:09:46 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 185 from training step 0 + +[ 2023-09-21 12:09:49 ] Completed train batch 0 to device 2,967.089 ms, 2.97 s total +[ 2023-09-21 12:09:49 ] Completed train batch 0 generator forward 30.898 ms, 3.00 s total +[ 2023-09-21 12:09:49 ] Completed train batch 0 generator loss: 1.270 65.516 ms, 3.06 s total +[ 2023-09-21 12:09:50 ] Completed train batch 0 generator backward 635.398 ms, 3.70 s total +[ 2023-09-21 12:09:50 ] Completed train batch 0 discriminator loss 0.042 61.089 ms, 3.76 s total +[ 2023-09-21 12:09:50 ] Completed train batch 0 discriminator backward 99.166 ms, 3.86 s total +Epoch [185] Step [0/2], gen_loss: 0.408, disc_loss: 0.014 +[ 2023-09-21 12:09:50 ] Completed train batch 0 metrics update 24.431 ms, 3.88 s total +Saving checkpoint at epoch 185 train batch 0 +[ 2023-09-21 12:09:51 ] Completed saving temp checkpoint 652.367 ms, 4.54 s total +[ 2023-09-21 12:09:51 ] Completed replacing temp checkpoint with checkpoint 33.555 ms, 4.57 s total +[ 2023-09-21 12:09:52 ] Completed train batch 1 to device 1,129.459 ms, 5.70 s total +[ 2023-09-21 12:09:52 ] Completed train batch 1 generator forward 40.725 ms, 5.74 s total +[ 2023-09-21 12:09:52 ] Completed train batch 1 generator loss: 1.278 62.868 ms, 5.80 s total +[ 2023-09-21 12:09:53 ] Completed train batch 1 generator backward 1,344.056 ms, 7.15 s total +[ 2023-09-21 12:09:53 ] Completed train batch 1 discriminator loss 0.045 55.664 ms, 7.20 s total +[ 2023-09-21 12:09:53 ] Completed train batch 1 discriminator backward 95.327 ms, 7.30 s total +Epoch [185] Step [1/2], gen_loss: 0.414, disc_loss: 0.015 +[ 2023-09-21 12:09:53 ] Completed train batch 1 metrics update 4.228 ms, 7.30 s total +Saving checkpoint at epoch 185 train batch 1 +[ 2023-09-21 12:09:54 ] Completed saving temp checkpoint 646.293 ms, 7.95 s total +[ 2023-09-21 12:09:54 ] Completed replacing temp checkpoint with checkpoint 38.583 ms, 7.99 s total +Epoch [185] :: gen_loss: 0.411, disc_loss: 0.015 +[ 2023-09-21 12:09:54 ] Completed training generator for epoch 185 67.415 ms, 8.05 s total +[ 2023-09-21 12:09:54 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 185 from eval step 0 + +[ 2023-09-21 12:09:55 ] Completed eval batch 0 to device 1,064.657 ms, 1.06 s total +[ 2023-09-21 12:09:55 ] Completed eval batch 0 forward 24.706 ms, 1.09 s total +[ 2023-09-21 12:09:55 ] Completed eval batch 0 recons_loss 0.372 ms, 1.09 s total +[ 2023-09-21 12:09:55 ] Completed eval batch 0 metrics update 0.653 ms, 1.09 s total +Saving checkpoint at epoch 185 val batch 0 +[ 2023-09-21 12:09:56 ] Completed saving temp checkpoint 574.752 ms, 1.67 s total +[ 2023-09-21 12:09:56 ] Completed replacing temp checkpoint with checkpoint 35.158 ms, 1.70 s total +[ 2023-09-21 12:09:56 ] Completed eval batch 1 to device 141.773 ms, 1.84 s total +[ 2023-09-21 12:09:56 ] Completed eval batch 1 forward 21.060 ms, 1.86 s total +[ 2023-09-21 12:09:56 ] Completed eval batch 1 recons_loss 0.340 ms, 1.86 s total +[ 2023-09-21 12:09:56 ] Completed eval batch 1 metrics update 182.295 ms, 2.05 s total +Saving checkpoint at epoch 185 val batch 1 +Epoch 185 val loss: 0.0576 +[ 2023-09-21 12:09:57 ] Completed saving temp checkpoint 614.337 ms, 2.66 s total +[ 2023-09-21 12:09:57 ] Completed replacing temp checkpoint with checkpoint 34.842 ms, 2.69 s total +[ 2023-09-21 12:09:57 ] Completed evaluating generator for epoch 185 53.874 ms, 2.75 s total + + +EPOCH :: 186 + + +[ 2023-09-21 12:09:57 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 186 from training step 0 + +[ 2023-09-21 12:10:00 ] Completed train batch 0 to device 3,007.182 ms, 3.01 s total +[ 2023-09-21 12:10:00 ] Completed train batch 0 generator forward 26.387 ms, 3.03 s total +[ 2023-09-21 12:10:00 ] Completed train batch 0 generator loss: 1.191 67.014 ms, 3.10 s total +[ 2023-09-21 12:10:01 ] Completed train batch 0 generator backward 733.632 ms, 3.83 s total +[ 2023-09-21 12:10:01 ] Completed train batch 0 discriminator loss 0.051 56.597 ms, 3.89 s total +[ 2023-09-21 12:10:01 ] Completed train batch 0 discriminator backward 106.574 ms, 4.00 s total +Epoch [186] Step [0/2], gen_loss: 0.406, disc_loss: 0.017 +[ 2023-09-21 12:10:01 ] Completed train batch 0 metrics update 9.932 ms, 4.01 s total +Saving checkpoint at epoch 186 train batch 0 +[ 2023-09-21 12:10:01 ] Completed saving temp checkpoint 523.603 ms, 4.53 s total +[ 2023-09-21 12:10:01 ] Completed replacing temp checkpoint with checkpoint 34.547 ms, 4.57 s total +[ 2023-09-21 12:10:03 ] Completed train batch 1 to device 1,429.899 ms, 6.00 s total +[ 2023-09-21 12:10:03 ] Completed train batch 1 generator forward 40.693 ms, 6.04 s total +[ 2023-09-21 12:10:03 ] Completed train batch 1 generator loss: 1.168 63.650 ms, 6.10 s total +[ 2023-09-21 12:10:04 ] Completed train batch 1 generator backward 1,459.625 ms, 7.56 s total +[ 2023-09-21 12:10:04 ] Completed train batch 1 discriminator loss 0.049 55.599 ms, 7.61 s total +[ 2023-09-21 12:10:05 ] Completed train batch 1 discriminator backward 74.180 ms, 7.69 s total +Epoch [186] Step [1/2], gen_loss: 0.411, disc_loss: 0.016 +[ 2023-09-21 12:10:05 ] Completed train batch 1 metrics update 1.378 ms, 7.69 s total +Saving checkpoint at epoch 186 train batch 1 +[ 2023-09-21 12:10:05 ] Completed saving temp checkpoint 415.808 ms, 8.11 s total +[ 2023-09-21 12:10:05 ] Completed replacing temp checkpoint with checkpoint 25.966 ms, 8.13 s total +Epoch [186] :: gen_loss: 0.408, disc_loss: 0.017 +[ 2023-09-21 12:10:05 ] Completed training generator for epoch 186 56.432 ms, 8.19 s total +[ 2023-09-21 12:10:05 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 186 from eval step 0 + +[ 2023-09-21 12:10:06 ] Completed eval batch 0 to device 1,056.172 ms, 1.06 s total +[ 2023-09-21 12:10:06 ] Completed eval batch 0 forward 28.235 ms, 1.08 s total +[ 2023-09-21 12:10:06 ] Completed eval batch 0 recons_loss 0.318 ms, 1.08 s total +[ 2023-09-21 12:10:06 ] Completed eval batch 0 metrics update 64.212 ms, 1.15 s total +Saving checkpoint at epoch 186 val batch 0 +[ 2023-09-21 12:10:07 ] Completed saving temp checkpoint 320.335 ms, 1.47 s total +[ 2023-09-21 12:10:07 ] Completed replacing temp checkpoint with checkpoint 27.649 ms, 1.50 s total +[ 2023-09-21 12:10:07 ] Completed eval batch 1 to device 388.145 ms, 1.89 s total +[ 2023-09-21 12:10:07 ] Completed eval batch 1 forward 40.636 ms, 1.93 s total +[ 2023-09-21 12:10:07 ] Completed eval batch 1 recons_loss 0.553 ms, 1.93 s total +[ 2023-09-21 12:10:07 ] Completed eval batch 1 metrics update 470.458 ms, 2.40 s total +Saving checkpoint at epoch 186 val batch 1 +Epoch 186 val loss: 0.0653 +[ 2023-09-21 12:10:08 ] Completed saving temp checkpoint 368.315 ms, 2.77 s total +[ 2023-09-21 12:10:08 ] Completed replacing temp checkpoint with checkpoint 26.730 ms, 2.79 s total +[ 2023-09-21 12:10:08 ] Completed evaluating generator for epoch 186 52.461 ms, 2.84 s total + + +EPOCH :: 187 + + +[ 2023-09-21 12:10:08 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 187 from training step 0 + +[ 2023-09-21 12:10:11 ] Completed train batch 0 to device 2,925.142 ms, 2.93 s total +[ 2023-09-21 12:10:11 ] Completed train batch 0 generator forward 33.871 ms, 2.96 s total +[ 2023-09-21 12:10:11 ] Completed train batch 0 generator loss: 1.301 65.356 ms, 3.02 s total +[ 2023-09-21 12:10:12 ] Completed train batch 0 generator backward 861.857 ms, 3.89 s total +[ 2023-09-21 12:10:12 ] Completed train batch 0 discriminator loss 0.045 56.487 ms, 3.94 s total +[ 2023-09-21 12:10:12 ] Completed train batch 0 discriminator backward 107.322 ms, 4.05 s total +Epoch [187] Step [0/2], gen_loss: 0.420, disc_loss: 0.015 +[ 2023-09-21 12:10:12 ] Completed train batch 0 metrics update 4.270 ms, 4.05 s total +Saving checkpoint at epoch 187 train batch 0 +[ 2023-09-21 12:10:12 ] Completed saving temp checkpoint 368.999 ms, 4.42 s total +[ 2023-09-21 12:10:12 ] Completed replacing temp checkpoint with checkpoint 18.588 ms, 4.44 s total +[ 2023-09-21 12:10:14 ] Completed train batch 1 to device 1,160.039 ms, 5.60 s total +[ 2023-09-21 12:10:14 ] Completed train batch 1 generator forward 23.815 ms, 5.63 s total +[ 2023-09-21 12:10:14 ] Completed train batch 1 generator loss: 1.154 62.893 ms, 5.69 s total +[ 2023-09-21 12:10:15 ] Completed train batch 1 generator backward 1,803.792 ms, 7.49 s total +[ 2023-09-21 12:10:15 ] Completed train batch 1 discriminator loss 0.043 55.849 ms, 7.55 s total +[ 2023-09-21 12:10:16 ] Completed train batch 1 discriminator backward 72.336 ms, 7.62 s total +Epoch [187] Step [1/2], gen_loss: 0.407, disc_loss: 0.015 +[ 2023-09-21 12:10:16 ] Completed train batch 1 metrics update 1.431 ms, 7.62 s total +Saving checkpoint at epoch 187 train batch 1 +[ 2023-09-21 12:10:16 ] Completed saving temp checkpoint 532.572 ms, 8.15 s total +[ 2023-09-21 12:10:16 ] Completed replacing temp checkpoint with checkpoint 17.955 ms, 8.17 s total +Epoch [187] :: gen_loss: 0.414, disc_loss: 0.015 +[ 2023-09-21 12:10:16 ] Completed training generator for epoch 187 70.142 ms, 8.24 s total +[ 2023-09-21 12:10:16 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 187 from eval step 0 + +[ 2023-09-21 12:10:17 ] Completed eval batch 0 to device 1,050.685 ms, 1.05 s total +[ 2023-09-21 12:10:17 ] Completed eval batch 0 forward 29.533 ms, 1.08 s total +[ 2023-09-21 12:10:17 ] Completed eval batch 0 recons_loss 0.360 ms, 1.08 s total +[ 2023-09-21 12:10:17 ] Completed eval batch 0 metrics update 0.580 ms, 1.08 s total +Saving checkpoint at epoch 187 val batch 0 +[ 2023-09-21 12:10:18 ] Completed saving temp checkpoint 383.832 ms, 1.46 s total +[ 2023-09-21 12:10:18 ] Completed replacing temp checkpoint with checkpoint 21.252 ms, 1.49 s total +[ 2023-09-21 12:10:18 ] Completed eval batch 1 to device 457.564 ms, 1.94 s total +[ 2023-09-21 12:10:18 ] Completed eval batch 1 forward 40.154 ms, 1.98 s total +[ 2023-09-21 12:10:18 ] Completed eval batch 1 recons_loss 0.540 ms, 1.98 s total +[ 2023-09-21 12:10:18 ] Completed eval batch 1 metrics update 236.482 ms, 2.22 s total +Saving checkpoint at epoch 187 val batch 1 +Epoch 187 val loss: 0.0614 +[ 2023-09-21 12:10:19 ] Completed saving temp checkpoint 447.948 ms, 2.67 s total +[ 2023-09-21 12:10:19 ] Completed replacing temp checkpoint with checkpoint 23.904 ms, 2.69 s total +[ 2023-09-21 12:10:19 ] Completed evaluating generator for epoch 187 52.807 ms, 2.75 s total + + +EPOCH :: 188 + + +[ 2023-09-21 12:10:19 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 188 from training step 0 + +[ 2023-09-21 12:10:22 ] Completed train batch 0 to device 3,012.585 ms, 3.01 s total +[ 2023-09-21 12:10:22 ] Completed train batch 0 generator forward 42.225 ms, 3.05 s total +[ 2023-09-21 12:10:22 ] Completed train batch 0 generator loss: 1.187 65.096 ms, 3.12 s total +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 12:22:54 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 12:22:54 ] Completed importing Timer 0.024 ms, 0.00 s total +[ 2023-09-21 12:22:58 ] Completed importing everything else 4,479.507 ms, 4.48 s total +| distributed init (rank 0): env:// +| distributed init (rank 3): env:// +| distributed init (rank 2): env:// +| distributed init (rank 5): env:// +| distributed init (rank 4): env:// +| distributed init (rank 1): env:// +[ 2023-09-21 12:23:06 ] Completed preliminaries 7,543.978 ms, 12.02 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 12:23:06 ] Completed build datasets 10.945 ms, 12.03 s total +[ 2023-09-21 12:23:06 ] Completed build samplers 0.045 ms, 12.03 s total +[ 2023-09-21 12:23:06 ] Completed build dataloaders 0.132 ms, 12.03 s total +[ 2023-09-21 12:23:06 ] Completed generator to device 95.311 ms, 12.13 s total +[ 2023-09-21 12:23:06 ] Completed discriminator to device 7.136 ms, 12.14 s total +[ 2023-09-21 12:23:06 ] Completed loss functions 440.268 ms, 12.58 s total +[ 2023-09-21 12:23:06 ] Completed models prepped for distribution 33.044 ms, 12.61 s total +[ 2023-09-21 12:23:06 ] Completed optimizers 0.683 ms, 12.61 s total +[ 2023-09-21 12:23:06 ] Completed grad scalers 0.028 ms, 12.61 s total +[ 2023-09-21 12:23:07 ] Completed checkpoint retrieval 465.850 ms, 13.08 s total + + +EPOCH :: 187 + + +[ 2023-09-21 12:23:07 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 187 from training step 2 + +Epoch [187] :: gen_loss: 0.414, disc_loss: 0.015 +[ 2023-09-21 12:23:07 ] Completed training generator for epoch 187 128.920 ms, 0.13 s total +[ 2023-09-21 12:23:07 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 187 from eval step 2 + +[ 2023-09-21 12:23:07 ] Completed evaluating generator for epoch 187 132.591 ms, 0.13 s total + + +EPOCH :: 188 + + +[ 2023-09-21 12:23:07 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 188 from training step 0 + +[ 2023-09-21 12:23:11 ] Completed train batch 0 to device 3,763.023 ms, 3.76 s total +[ 2023-09-21 12:23:12 ] Completed train batch 0 generator forward 1,493.576 ms, 5.26 s total +[ 2023-09-21 12:23:12 ] Completed train batch 0 generator loss: 1.240 76.171 ms, 5.33 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 12:23:14 ] Completed train batch 0 generator backward 1,430.317 ms, 6.76 s total +[ 2023-09-21 12:23:14 ] Completed train batch 0 discriminator loss 0.044 61.575 ms, 6.82 s total +[ 2023-09-21 12:23:14 ] Completed train batch 0 discriminator backward 97.384 ms, 6.92 s total +Epoch [188] Step [0/2], gen_loss: 0.413, disc_loss: 0.014 +[ 2023-09-21 12:23:14 ] Completed train batch 0 metrics update 6.181 ms, 6.93 s total +Saving checkpoint at epoch 188 train batch 0 +[ 2023-09-21 12:23:14 ] Completed saving temp checkpoint 463.299 ms, 7.39 s total +[ 2023-09-21 12:23:14 ] Completed replacing temp checkpoint with checkpoint 125.565 ms, 7.52 s total +[ 2023-09-21 12:23:15 ] Completed train batch 1 to device 21.164 ms, 7.54 s total +[ 2023-09-21 12:23:15 ] Completed train batch 1 generator forward 36.600 ms, 7.57 s total +[ 2023-09-21 12:23:15 ] Completed train batch 1 generator loss: 1.227 61.871 ms, 7.64 s total +[ 2023-09-21 12:23:16 ] Completed train batch 1 generator backward 1,283.026 ms, 8.92 s total +[ 2023-09-21 12:23:16 ] Completed train batch 1 discriminator loss 0.045 56.626 ms, 8.98 s total +[ 2023-09-21 12:23:16 ] Completed train batch 1 discriminator backward 78.775 ms, 9.06 s total +Epoch [188] Step [1/2], gen_loss: 0.408, disc_loss: 0.014 +[ 2023-09-21 12:23:16 ] Completed train batch 1 metrics update 2.633 ms, 9.06 s total +Saving checkpoint at epoch 188 train batch 1 +[ 2023-09-21 12:23:17 ] Completed saving temp checkpoint 492.256 ms, 9.55 s total +[ 2023-09-21 12:23:17 ] Completed replacing temp checkpoint with checkpoint 30.528 ms, 9.58 s total +Epoch [188] :: gen_loss: 0.411, disc_loss: 0.014 +[ 2023-09-21 12:23:17 ] Completed training generator for epoch 188 44.266 ms, 9.62 s total +[ 2023-09-21 12:23:17 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 188 from eval step 0 + +[ 2023-09-21 12:23:18 ] Completed eval batch 0 to device 1,137.692 ms, 1.14 s total +[ 2023-09-21 12:23:18 ] Completed eval batch 0 forward 27.644 ms, 1.17 s total +[ 2023-09-21 12:23:18 ] Completed eval batch 0 recons_loss 0.375 ms, 1.17 s total +[ 2023-09-21 12:23:18 ] Completed eval batch 0 metrics update 0.741 ms, 1.17 s total +Saving checkpoint at epoch 188 val batch 0 +[ 2023-09-21 12:23:18 ] Completed saving temp checkpoint 423.826 ms, 1.59 s total +[ 2023-09-21 12:23:18 ] Completed replacing temp checkpoint with checkpoint 20.268 ms, 1.61 s total +[ 2023-09-21 12:23:19 ] Completed eval batch 1 to device 420.724 ms, 2.03 s total +[ 2023-09-21 12:23:19 ] Completed eval batch 1 forward 39.800 ms, 2.07 s total +[ 2023-09-21 12:23:19 ] Completed eval batch 1 recons_loss 0.624 ms, 2.07 s total +[ 2023-09-21 12:23:19 ] Completed eval batch 1 metrics update 165.474 ms, 2.24 s total +Saving checkpoint at epoch 188 val batch 1 +Epoch 188 val loss: 0.0583 +[ 2023-09-21 12:23:19 ] Completed saving temp checkpoint 449.881 ms, 2.69 s total +[ 2023-09-21 12:23:19 ] Completed replacing temp checkpoint with checkpoint 30.888 ms, 2.72 s total +[ 2023-09-21 12:23:19 ] Completed evaluating generator for epoch 188 52.674 ms, 2.77 s total + + +EPOCH :: 189 + + +[ 2023-09-21 12:23:19 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 189 from training step 0 + +[ 2023-09-21 12:23:22 ] Completed train batch 0 to device 3,083.714 ms, 3.08 s total +[ 2023-09-21 12:23:22 ] Completed train batch 0 generator forward 33.438 ms, 3.12 s total +[ 2023-09-21 12:23:23 ] Completed train batch 0 generator loss: 1.199 67.598 ms, 3.18 s total +[ 2023-09-21 12:23:23 ] Completed train batch 0 generator backward 713.460 ms, 3.90 s total +[ 2023-09-21 12:23:23 ] Completed train batch 0 discriminator loss 0.043 58.003 ms, 3.96 s total +[ 2023-09-21 12:23:23 ] Completed train batch 0 discriminator backward 117.793 ms, 4.07 s total +Epoch [189] Step [0/2], gen_loss: 0.410, disc_loss: 0.014 +[ 2023-09-21 12:23:23 ] Completed train batch 0 metrics update 15.029 ms, 4.09 s total +Saving checkpoint at epoch 189 train batch 0 +[ 2023-09-21 12:23:24 ] Completed saving temp checkpoint 459.088 ms, 4.55 s total +[ 2023-09-21 12:23:24 ] Completed replacing temp checkpoint with checkpoint 33.252 ms, 4.58 s total +[ 2023-09-21 12:23:25 ] Completed train batch 1 to device 1,278.973 ms, 5.86 s total +[ 2023-09-21 12:23:25 ] Completed train batch 1 generator forward 22.148 ms, 5.88 s total +[ 2023-09-21 12:23:25 ] Completed train batch 1 generator loss: 1.149 63.396 ms, 5.95 s total +[ 2023-09-21 12:23:27 ] Completed train batch 1 generator backward 1,362.920 ms, 7.31 s total +[ 2023-09-21 12:23:27 ] Completed train batch 1 discriminator loss 0.041 56.368 ms, 7.37 s total +[ 2023-09-21 12:23:27 ] Completed train batch 1 discriminator backward 90.904 ms, 7.46 s total +Epoch [189] Step [1/2], gen_loss: 0.413, disc_loss: 0.014 +[ 2023-09-21 12:23:27 ] Completed train batch 1 metrics update 5.716 ms, 7.46 s total +Saving checkpoint at epoch 189 train batch 1 +[ 2023-09-21 12:23:27 ] Completed saving temp checkpoint 489.600 ms, 7.95 s total +[ 2023-09-21 12:23:27 ] Completed replacing temp checkpoint with checkpoint 23.439 ms, 7.97 s total +Epoch [189] :: gen_loss: 0.412, disc_loss: 0.014 +[ 2023-09-21 12:23:27 ] Completed training generator for epoch 189 69.021 ms, 8.04 s total +[ 2023-09-21 12:23:27 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 189 from eval step 0 + +[ 2023-09-21 12:23:29 ] Completed eval batch 0 to device 1,097.442 ms, 1.10 s total +[ 2023-09-21 12:23:29 ] Completed eval batch 0 forward 24.745 ms, 1.12 s total +[ 2023-09-21 12:23:29 ] Completed eval batch 0 recons_loss 0.304 ms, 1.12 s total +[ 2023-09-21 12:23:29 ] Completed eval batch 0 metrics update 0.587 ms, 1.12 s total +Saving checkpoint at epoch 189 val batch 0 +[ 2023-09-21 12:23:29 ] Completed saving temp checkpoint 415.682 ms, 1.54 s total +[ 2023-09-21 12:23:29 ] Completed replacing temp checkpoint with checkpoint 30.897 ms, 1.57 s total +[ 2023-09-21 12:23:29 ] Completed eval batch 1 to device 405.033 ms, 1.97 s total +[ 2023-09-21 12:23:29 ] Completed eval batch 1 forward 39.887 ms, 2.01 s total +[ 2023-09-21 12:23:29 ] Completed eval batch 1 recons_loss 0.332 ms, 2.01 s total +[ 2023-09-21 12:23:30 ] Completed eval batch 1 metrics update 261.545 ms, 2.28 s total +Saving checkpoint at epoch 189 val batch 1 +Epoch 189 val loss: 0.0676 +[ 2023-09-21 12:23:30 ] Completed saving temp checkpoint 456.603 ms, 2.73 s total +[ 2023-09-21 12:23:30 ] Completed replacing temp checkpoint with checkpoint 29.096 ms, 2.76 s total +[ 2023-09-21 12:23:30 ] Completed evaluating generator for epoch 189 48.669 ms, 2.81 s total + + +EPOCH :: 190 + + +[ 2023-09-21 12:23:30 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 190 from training step 0 + +[ 2023-09-21 12:23:33 ] Completed train batch 0 to device 3,187.687 ms, 3.19 s total +[ 2023-09-21 12:23:33 ] Completed train batch 0 generator forward 33.726 ms, 3.22 s total +[ 2023-09-21 12:23:34 ] Completed train batch 0 generator loss: 1.236 73.204 ms, 3.29 s total +[ 2023-09-21 12:23:34 ] Completed train batch 0 generator backward 545.387 ms, 3.84 s total +[ 2023-09-21 12:23:34 ] Completed train batch 0 discriminator loss 0.041 58.661 ms, 3.90 s total +[ 2023-09-21 12:23:34 ] Completed train batch 0 discriminator backward 100.072 ms, 4.00 s total +Epoch [190] Step [0/2], gen_loss: 0.417, disc_loss: 0.014 +[ 2023-09-21 12:23:34 ] Completed train batch 0 metrics update 5.955 ms, 4.00 s total +Saving checkpoint at epoch 190 train batch 0 +[ 2023-09-21 12:23:35 ] Completed saving temp checkpoint 432.377 ms, 4.44 s total +[ 2023-09-21 12:23:35 ] Completed replacing temp checkpoint with checkpoint 21.446 ms, 4.46 s total +[ 2023-09-21 12:23:36 ] Completed train batch 1 to device 1,600.673 ms, 6.06 s total +[ 2023-09-21 12:23:36 ] Completed train batch 1 generator forward 22.378 ms, 6.08 s total +[ 2023-09-21 12:23:36 ] Completed train batch 1 generator loss: 1.306 66.371 ms, 6.15 s total +[ 2023-09-21 12:23:37 ] Completed train batch 1 generator backward 1,100.626 ms, 7.25 s total +[ 2023-09-21 12:23:38 ] Completed train batch 1 discriminator loss 0.039 56.464 ms, 7.31 s total +[ 2023-09-21 12:23:38 ] Completed train batch 1 discriminator backward 93.762 ms, 7.40 s total +Epoch [190] Step [1/2], gen_loss: 0.413, disc_loss: 0.013 +[ 2023-09-21 12:23:38 ] Completed train batch 1 metrics update 6.119 ms, 7.40 s total +Saving checkpoint at epoch 190 train batch 1 +[ 2023-09-21 12:23:38 ] Completed saving temp checkpoint 447.979 ms, 7.85 s total +[ 2023-09-21 12:23:38 ] Completed replacing temp checkpoint with checkpoint 16.709 ms, 7.87 s total +Epoch [190] :: gen_loss: 0.415, disc_loss: 0.013 +[ 2023-09-21 12:23:38 ] Completed training generator for epoch 190 54.859 ms, 7.92 s total +[ 2023-09-21 12:23:38 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 190 from eval step 0 + +[ 2023-09-21 12:23:39 ] Completed eval batch 0 to device 1,214.441 ms, 1.21 s total +[ 2023-09-21 12:23:39 ] Completed eval batch 0 forward 28.336 ms, 1.24 s total +[ 2023-09-21 12:23:39 ] Completed eval batch 0 recons_loss 0.385 ms, 1.24 s total +[ 2023-09-21 12:23:39 ] Completed eval batch 0 metrics update 0.796 ms, 1.24 s total +Saving checkpoint at epoch 190 val batch 0 +[ 2023-09-21 12:23:40 ] Completed saving temp checkpoint 401.555 ms, 1.65 s total +[ 2023-09-21 12:23:40 ] Completed replacing temp checkpoint with checkpoint 26.941 ms, 1.67 s total +[ 2023-09-21 12:23:40 ] Completed eval batch 1 to device 572.058 ms, 2.24 s total +[ 2023-09-21 12:23:40 ] Completed eval batch 1 forward 36.860 ms, 2.28 s total +[ 2023-09-21 12:23:40 ] Completed eval batch 1 recons_loss 0.357 ms, 2.28 s total +[ 2023-09-21 12:23:40 ] Completed eval batch 1 metrics update 31.269 ms, 2.31 s total +Saving checkpoint at epoch 190 val batch 1 +Epoch 190 val loss: 0.0593 +[ 2023-09-21 12:23:41 ] Completed saving temp checkpoint 427.613 ms, 2.74 s total +[ 2023-09-21 12:23:41 ] Completed replacing temp checkpoint with checkpoint 22.912 ms, 2.76 s total +[ 2023-09-21 12:23:41 ] Completed evaluating generator for epoch 190 52.895 ms, 2.82 s total + + +EPOCH :: 191 + + +[ 2023-09-21 12:23:41 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 191 from training step 0 + +[ 2023-09-21 12:23:44 ] Completed train batch 0 to device 3,117.582 ms, 3.12 s total +[ 2023-09-21 12:23:44 ] Completed train batch 0 generator forward 25.376 ms, 3.14 s total +[ 2023-09-21 12:23:44 ] Completed train batch 0 generator loss: 1.159 65.185 ms, 3.21 s total +[ 2023-09-21 12:23:45 ] Completed train batch 0 generator backward 633.837 ms, 3.84 s total +[ 2023-09-21 12:23:45 ] Completed train batch 0 discriminator loss 0.039 57.189 ms, 3.90 s total +[ 2023-09-21 12:23:45 ] Completed train batch 0 discriminator backward 112.665 ms, 4.01 s total +Epoch [191] Step [0/2], gen_loss: 0.411, disc_loss: 0.013 +[ 2023-09-21 12:23:45 ] Completed train batch 0 metrics update 5.266 ms, 4.02 s total +Saving checkpoint at epoch 191 train batch 0 +[ 2023-09-21 12:23:45 ] Completed saving temp checkpoint 473.872 ms, 4.49 s total +[ 2023-09-21 12:23:45 ] Completed replacing temp checkpoint with checkpoint 23.995 ms, 4.51 s total +[ 2023-09-21 12:23:47 ] Completed train batch 1 to device 1,489.385 ms, 6.00 s total +[ 2023-09-21 12:23:47 ] Completed train batch 1 generator forward 39.320 ms, 6.04 s total +[ 2023-09-21 12:23:47 ] Completed train batch 1 generator loss: 1.262 66.269 ms, 6.11 s total +[ 2023-09-21 12:23:48 ] Completed train batch 1 generator backward 1,413.970 ms, 7.52 s total +[ 2023-09-21 12:23:49 ] Completed train batch 1 discriminator loss 0.038 57.051 ms, 7.58 s total +[ 2023-09-21 12:23:49 ] Completed train batch 1 discriminator backward 85.943 ms, 7.67 s total +Epoch [191] Step [1/2], gen_loss: 0.412, disc_loss: 0.013 +[ 2023-09-21 12:23:49 ] Completed train batch 1 metrics update 5.018 ms, 7.67 s total +Saving checkpoint at epoch 191 train batch 1 +[ 2023-09-21 12:23:49 ] Completed saving temp checkpoint 446.736 ms, 8.12 s total +[ 2023-09-21 12:23:49 ] Completed replacing temp checkpoint with checkpoint 15.192 ms, 8.13 s total +Epoch [191] :: gen_loss: 0.411, disc_loss: 0.013 +[ 2023-09-21 12:23:49 ] Completed training generator for epoch 191 57.947 ms, 8.19 s total +[ 2023-09-21 12:23:49 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 191 from eval step 0 + +[ 2023-09-21 12:23:50 ] Completed eval batch 0 to device 1,157.330 ms, 1.16 s total +[ 2023-09-21 12:23:50 ] Completed eval batch 0 forward 31.169 ms, 1.19 s total +[ 2023-09-21 12:23:50 ] Completed eval batch 0 recons_loss 0.293 ms, 1.19 s total +[ 2023-09-21 12:23:50 ] Completed eval batch 0 metrics update 0.539 ms, 1.19 s total +Saving checkpoint at epoch 191 val batch 0 +[ 2023-09-21 12:23:51 ] Completed saving temp checkpoint 412.478 ms, 1.60 s total +[ 2023-09-21 12:23:51 ] Completed replacing temp checkpoint with checkpoint 24.637 ms, 1.63 s total +[ 2023-09-21 12:23:51 ] Completed eval batch 1 to device 563.473 ms, 2.19 s total +[ 2023-09-21 12:23:51 ] Completed eval batch 1 forward 40.032 ms, 2.23 s total +[ 2023-09-21 12:23:51 ] Completed eval batch 1 recons_loss 0.423 ms, 2.23 s total +[ 2023-09-21 12:23:52 ] Completed eval batch 1 metrics update 200.085 ms, 2.43 s total +Saving checkpoint at epoch 191 val batch 1 +Epoch 191 val loss: 0.0552 +[ 2023-09-21 12:23:52 ] Completed saving temp checkpoint 452.547 ms, 2.88 s total +[ 2023-09-21 12:23:52 ] Completed replacing temp checkpoint with checkpoint 25.730 ms, 2.91 s total +[ 2023-09-21 12:23:52 ] Completed evaluating generator for epoch 191 48.637 ms, 2.96 s total + + +EPOCH :: 192 + + +[ 2023-09-21 12:23:52 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 192 from training step 0 + +[ 2023-09-21 12:23:55 ] Completed train batch 0 to device 3,241.744 ms, 3.24 s total +[ 2023-09-21 12:23:55 ] Completed train batch 0 generator forward 25.753 ms, 3.27 s total +[ 2023-09-21 12:23:55 ] Completed train batch 0 generator loss: 1.247 65.247 ms, 3.33 s total +[ 2023-09-21 12:23:56 ] Completed train batch 0 generator backward 547.491 ms, 3.88 s total +[ 2023-09-21 12:23:56 ] Completed train batch 0 discriminator loss 0.038 57.796 ms, 3.94 s total +[ 2023-09-21 12:23:56 ] Completed train batch 0 discriminator backward 99.364 ms, 4.04 s total +Epoch [192] Step [0/2], gen_loss: 0.411, disc_loss: 0.013 +[ 2023-09-21 12:23:56 ] Completed train batch 0 metrics update 7.102 ms, 4.04 s total +Saving checkpoint at epoch 192 train batch 0 +[ 2023-09-21 12:23:57 ] Completed saving temp checkpoint 483.848 ms, 4.53 s total +[ 2023-09-21 12:23:57 ] Completed replacing temp checkpoint with checkpoint 26.575 ms, 4.55 s total +[ 2023-09-21 12:23:58 ] Completed train batch 1 to device 1,691.256 ms, 6.25 s total +[ 2023-09-21 12:23:58 ] Completed train batch 1 generator forward 43.603 ms, 6.29 s total +[ 2023-09-21 12:23:58 ] Completed train batch 1 generator loss: 1.251 67.249 ms, 6.36 s total +[ 2023-09-21 12:23:59 ] Completed train batch 1 generator backward 1,027.922 ms, 7.38 s total +[ 2023-09-21 12:24:00 ] Completed train batch 1 discriminator loss 0.042 58.045 ms, 7.44 s total +[ 2023-09-21 12:24:00 ] Completed train batch 1 discriminator backward 58.829 ms, 7.50 s total +Epoch [192] Step [1/2], gen_loss: 0.415, disc_loss: 0.014 +[ 2023-09-21 12:24:00 ] Completed train batch 1 metrics update 2.288 ms, 7.50 s total +Saving checkpoint at epoch 192 train batch 1 +[ 2023-09-21 12:24:00 ] Completed saving temp checkpoint 665.600 ms, 8.17 s total +[ 2023-09-21 12:24:00 ] Completed replacing temp checkpoint with checkpoint 31.574 ms, 8.20 s total +Epoch [192] :: gen_loss: 0.413, disc_loss: 0.013 +[ 2023-09-21 12:24:00 ] Completed training generator for epoch 192 55.774 ms, 8.26 s total +[ 2023-09-21 12:24:00 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 192 from eval step 0 + +[ 2023-09-21 12:24:01 ] Completed eval batch 0 to device 1,080.124 ms, 1.08 s total +[ 2023-09-21 12:24:01 ] Completed eval batch 0 forward 25.499 ms, 1.11 s total +[ 2023-09-21 12:24:01 ] Completed eval batch 0 recons_loss 0.321 ms, 1.11 s total +[ 2023-09-21 12:24:01 ] Completed eval batch 0 metrics update 0.626 ms, 1.11 s total +Saving checkpoint at epoch 192 val batch 0 +[ 2023-09-21 12:24:02 ] Completed saving temp checkpoint 632.848 ms, 1.74 s total +[ 2023-09-21 12:24:02 ] Completed replacing temp checkpoint with checkpoint 19.759 ms, 1.76 s total +[ 2023-09-21 12:24:02 ] Completed eval batch 1 to device 166.814 ms, 1.93 s total +[ 2023-09-21 12:24:02 ] Completed eval batch 1 forward 21.700 ms, 1.95 s total +[ 2023-09-21 12:24:02 ] Completed eval batch 1 recons_loss 0.379 ms, 1.95 s total +[ 2023-09-21 12:24:02 ] Completed eval batch 1 metrics update 164.777 ms, 2.11 s total +Saving checkpoint at epoch 192 val batch 1 +Epoch 192 val loss: 0.0572 +[ 2023-09-21 12:24:03 ] Completed saving temp checkpoint 665.400 ms, 2.78 s total +[ 2023-09-21 12:24:03 ] Completed replacing temp checkpoint with checkpoint 19.320 ms, 2.80 s total +[ 2023-09-21 12:24:03 ] Completed evaluating generator for epoch 192 48.380 ms, 2.85 s total + + +EPOCH :: 193 + + +[ 2023-09-21 12:24:03 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 193 from training step 0 + +[ 2023-09-21 12:24:06 ] Completed train batch 0 to device 3,119.316 ms, 3.12 s total +[ 2023-09-21 12:24:06 ] Completed train batch 0 generator forward 29.544 ms, 3.15 s total +[ 2023-09-21 12:24:06 ] Completed train batch 0 generator loss: 1.198 67.361 ms, 3.22 s total +[ 2023-09-21 12:24:07 ] Completed train batch 0 generator backward 406.095 ms, 3.62 s total +[ 2023-09-21 12:24:07 ] Completed train batch 0 discriminator loss 0.042 57.192 ms, 3.68 s total +[ 2023-09-21 12:24:07 ] Completed train batch 0 discriminator backward 106.393 ms, 3.79 s total +Epoch [193] Step [0/2], gen_loss: 0.412, disc_loss: 0.014 +[ 2023-09-21 12:24:07 ] Completed train batch 0 metrics update 7.818 ms, 3.79 s total +Saving checkpoint at epoch 193 train batch 0 +[ 2023-09-21 12:24:08 ] Completed saving temp checkpoint 687.953 ms, 4.48 s total +[ 2023-09-21 12:24:08 ] Completed replacing temp checkpoint with checkpoint 34.313 ms, 4.52 s total +[ 2023-09-21 12:24:09 ] Completed train batch 1 to device 1,349.279 ms, 5.87 s total +[ 2023-09-21 12:24:09 ] Completed train batch 1 generator forward 41.304 ms, 5.91 s total +[ 2023-09-21 12:24:09 ] Completed train batch 1 generator loss: 1.263 64.711 ms, 5.97 s total +[ 2023-09-21 12:24:10 ] Completed train batch 1 generator backward 1,148.921 ms, 7.12 s total +[ 2023-09-21 12:24:10 ] Completed train batch 1 discriminator loss 0.042 56.679 ms, 7.18 s total +[ 2023-09-21 12:24:10 ] Completed train batch 1 discriminator backward 76.811 ms, 7.25 s total +Epoch [193] Step [1/2], gen_loss: 0.414, disc_loss: 0.014 +[ 2023-09-21 12:24:10 ] Completed train batch 1 metrics update 3.243 ms, 7.26 s total +Saving checkpoint at epoch 193 train batch 1 +[ 2023-09-21 12:24:11 ] Completed saving temp checkpoint 708.120 ms, 7.97 s total +[ 2023-09-21 12:24:11 ] Completed replacing temp checkpoint with checkpoint 25.528 ms, 7.99 s total +Epoch [193] :: gen_loss: 0.413, disc_loss: 0.014 +[ 2023-09-21 12:24:11 ] Completed training generator for epoch 193 58.442 ms, 8.05 s total +[ 2023-09-21 12:24:11 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 193 from eval step 0 + +[ 2023-09-21 12:24:12 ] Completed eval batch 0 to device 1,056.693 ms, 1.06 s total +[ 2023-09-21 12:24:12 ] Completed eval batch 0 forward 43.483 ms, 1.10 s total +[ 2023-09-21 12:24:12 ] Completed eval batch 0 recons_loss 0.371 ms, 1.10 s total +[ 2023-09-21 12:24:12 ] Completed eval batch 0 metrics update 0.662 ms, 1.10 s total +Saving checkpoint at epoch 193 val batch 0 +[ 2023-09-21 12:24:13 ] Completed saving temp checkpoint 667.856 ms, 1.77 s total +[ 2023-09-21 12:24:13 ] Completed replacing temp checkpoint with checkpoint 40.373 ms, 1.81 s total +[ 2023-09-21 12:24:13 ] Completed eval batch 1 to device 187.863 ms, 2.00 s total +[ 2023-09-21 12:24:13 ] Completed eval batch 1 forward 40.514 ms, 2.04 s total +[ 2023-09-21 12:24:13 ] Completed eval batch 1 recons_loss 0.571 ms, 2.04 s total +[ 2023-09-21 12:24:13 ] Completed eval batch 1 metrics update 0.507 ms, 2.04 s total +Saving checkpoint at epoch 193 val batch 1 +Epoch 193 val loss: 0.0598 +[ 2023-09-21 12:24:14 ] Completed saving temp checkpoint 665.127 ms, 2.70 s total +[ 2023-09-21 12:24:14 ] Completed replacing temp checkpoint with checkpoint 40.411 ms, 2.74 s total +[ 2023-09-21 12:24:14 ] Completed evaluating generator for epoch 193 57.136 ms, 2.80 s total + + +EPOCH :: 194 + + +[ 2023-09-21 12:24:14 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 194 from training step 0 + +[ 2023-09-21 12:24:17 ] Completed train batch 0 to device 2,898.174 ms, 2.90 s total +[ 2023-09-21 12:24:17 ] Completed train batch 0 generator forward 42.895 ms, 2.94 s total +[ 2023-09-21 12:24:17 ] Completed train batch 0 generator loss: 1.141 65.994 ms, 3.01 s total +[ 2023-09-21 12:24:18 ] Completed train batch 0 generator backward 613.431 ms, 3.62 s total +[ 2023-09-21 12:24:18 ] Completed train batch 0 discriminator loss 0.043 57.006 ms, 3.68 s total +[ 2023-09-21 12:24:18 ] Completed train batch 0 discriminator backward 97.453 ms, 3.77 s total +Epoch [194] Step [0/2], gen_loss: 0.412, disc_loss: 0.014 +[ 2023-09-21 12:24:18 ] Completed train batch 0 metrics update 5.236 ms, 3.78 s total +Saving checkpoint at epoch 194 train batch 0 +[ 2023-09-21 12:24:18 ] Completed saving temp checkpoint 639.113 ms, 4.42 s total +[ 2023-09-21 12:24:19 ] Completed replacing temp checkpoint with checkpoint 36.680 ms, 4.46 s total +[ 2023-09-21 12:24:20 ] Completed train batch 1 to device 1,197.758 ms, 5.65 s total +[ 2023-09-21 12:24:20 ] Completed train batch 1 generator forward 26.874 ms, 5.68 s total +[ 2023-09-21 12:24:20 ] Completed train batch 1 generator loss: 1.193 210.179 ms, 5.89 s total +[ 2023-09-21 12:24:21 ] Completed train batch 1 generator backward 1,169.166 ms, 7.06 s total +[ 2023-09-21 12:24:21 ] Completed train batch 1 discriminator loss 0.040 56.419 ms, 7.12 s total +[ 2023-09-21 12:24:21 ] Completed train batch 1 discriminator backward 74.362 ms, 7.19 s total +Epoch [194] Step [1/2], gen_loss: 0.414, disc_loss: 0.014 +[ 2023-09-21 12:24:21 ] Completed train batch 1 metrics update 5.440 ms, 7.20 s total +Saving checkpoint at epoch 194 train batch 1 +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +[ 2023-09-21 12:36:53 ] Completed Start 0.000 ms, 0.00 s total +[ 2023-09-21 12:36:53 ] Completed importing Timer 0.023 ms, 0.00 s total +[ 2023-09-21 12:36:57 ] Completed importing everything else 4,671.341 ms, 4.67 s total +| distributed init (rank 0): env:// +| distributed init (rank 5): env:// +| distributed init (rank 1): env:// +| distributed init (rank 3): env:// +| distributed init (rank 2): env:// +| distributed init (rank 4): env:// +[ 2023-09-21 12:37:05 ] Completed preliminaries 7,707.168 ms, 12.38 s total +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +[ 2023-09-21 12:37:05 ] Completed build datasets 11.271 ms, 12.39 s total +[ 2023-09-21 12:37:05 ] Completed build samplers 0.044 ms, 12.39 s total +[ 2023-09-21 12:37:05 ] Completed build dataloaders 0.130 ms, 12.39 s total +[ 2023-09-21 12:37:05 ] Completed generator to device 91.123 ms, 12.48 s total +[ 2023-09-21 12:37:05 ] Completed discriminator to device 6.993 ms, 12.49 s total +[ 2023-09-21 12:37:06 ] Completed loss functions 394.935 ms, 12.88 s total +[ 2023-09-21 12:37:06 ] Completed models prepped for distribution 71.491 ms, 12.95 s total +[ 2023-09-21 12:37:06 ] Completed optimizers 0.679 ms, 12.96 s total +[ 2023-09-21 12:37:06 ] Completed grad scalers 0.035 ms, 12.96 s total +[ 2023-09-21 12:37:06 ] Completed checkpoint retrieval 502.733 ms, 13.46 s total + + +EPOCH :: 194 + + +[ 2023-09-21 12:37:06 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 194 from training step 1 + +[ 2023-09-21 12:37:10 ] Completed train batch 1 to device 3,623.676 ms, 3.62 s total +[ 2023-09-21 12:37:11 ] Completed train batch 1 generator forward 1,125.412 ms, 4.75 s total +[ 2023-09-21 12:37:11 ] Completed train batch 1 generator loss: 1.270 76.517 ms, 4.83 s total +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +[ 2023-09-21 12:37:12 ] Completed train batch 1 generator backward 1,308.666 ms, 6.13 s total +[ 2023-09-21 12:37:12 ] Completed train batch 1 discriminator loss 0.045 65.729 ms, 6.20 s total +[ 2023-09-21 12:37:12 ] Completed train batch 1 discriminator backward 62.404 ms, 6.26 s total +Epoch [194] Step [1/2], gen_loss: 0.414, disc_loss: 0.014 +[ 2023-09-21 12:37:12 ] Completed train batch 1 metrics update 1.571 ms, 6.26 s total +Saving checkpoint at epoch 194 train batch 1 +[ 2023-09-21 12:37:13 ] Completed saving temp checkpoint 451.428 ms, 6.72 s total +[ 2023-09-21 12:37:13 ] Completed replacing temp checkpoint with checkpoint 141.186 ms, 6.86 s total +Epoch [194] :: gen_loss: 0.413, disc_loss: 0.014 +[ 2023-09-21 12:37:13 ] Completed training generator for epoch 194 56.728 ms, 6.91 s total +[ 2023-09-21 12:37:13 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 194 from eval step 0 + +[ 2023-09-21 12:37:14 ] Completed eval batch 0 to device 1,055.535 ms, 1.06 s total +[ 2023-09-21 12:37:14 ] Completed eval batch 0 forward 26.940 ms, 1.08 s total +[ 2023-09-21 12:37:14 ] Completed eval batch 0 recons_loss 0.374 ms, 1.08 s total +[ 2023-09-21 12:37:14 ] Completed eval batch 0 metrics update 0.803 ms, 1.08 s total +Saving checkpoint at epoch 194 val batch 0 +[ 2023-09-21 12:37:15 ] Completed saving temp checkpoint 424.888 ms, 1.51 s total +[ 2023-09-21 12:37:15 ] Completed replacing temp checkpoint with checkpoint 31.211 ms, 1.54 s total +[ 2023-09-21 12:37:15 ] Completed eval batch 1 to device 472.777 ms, 2.01 s total +[ 2023-09-21 12:37:15 ] Completed eval batch 1 forward 26.040 ms, 2.04 s total +[ 2023-09-21 12:37:15 ] Completed eval batch 1 recons_loss 0.359 ms, 2.04 s total +[ 2023-09-21 12:37:15 ] Completed eval batch 1 metrics update 43.657 ms, 2.08 s total +Saving checkpoint at epoch 194 val batch 1 +Epoch 194 val loss: 0.0665 +[ 2023-09-21 12:37:16 ] Completed saving temp checkpoint 404.610 ms, 2.49 s total +[ 2023-09-21 12:37:16 ] Completed replacing temp checkpoint with checkpoint 30.426 ms, 2.52 s total +[ 2023-09-21 12:37:16 ] Completed evaluating generator for epoch 194 43.911 ms, 2.56 s total + + +EPOCH :: 195 + + +[ 2023-09-21 12:37:16 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 195 from training step 0 + +[ 2023-09-21 12:37:18 ] Completed train batch 0 to device 2,836.128 ms, 2.84 s total +[ 2023-09-21 12:37:19 ] Completed train batch 0 generator forward 25.964 ms, 2.86 s total +[ 2023-09-21 12:37:19 ] Completed train batch 0 generator loss: 1.267 66.155 ms, 2.93 s total +[ 2023-09-21 12:37:20 ] Completed train batch 0 generator backward 1,055.717 ms, 3.98 s total +[ 2023-09-21 12:37:20 ] Completed train batch 0 discriminator loss 0.044 57.146 ms, 4.04 s total +[ 2023-09-21 12:37:20 ] Completed train batch 0 discriminator backward 107.724 ms, 4.15 s total +Epoch [195] Step [0/2], gen_loss: 0.421, disc_loss: 0.014 +[ 2023-09-21 12:37:20 ] Completed train batch 0 metrics update 8.808 ms, 4.16 s total +Saving checkpoint at epoch 195 train batch 0 +[ 2023-09-21 12:37:20 ] Completed saving temp checkpoint 431.353 ms, 4.59 s total +[ 2023-09-21 12:37:20 ] Completed replacing temp checkpoint with checkpoint 29.621 ms, 4.62 s total +[ 2023-09-21 12:37:21 ] Completed train batch 1 to device 1,183.230 ms, 5.80 s total +[ 2023-09-21 12:37:21 ] Completed train batch 1 generator forward 39.904 ms, 5.84 s total +[ 2023-09-21 12:37:22 ] Completed train batch 1 generator loss: 1.188 64.083 ms, 5.91 s total +[ 2023-09-21 12:37:23 ] Completed train batch 1 generator backward 1,649.275 ms, 7.56 s total +[ 2023-09-21 12:37:23 ] Completed train batch 1 discriminator loss 0.041 56.331 ms, 7.61 s total +[ 2023-09-21 12:37:23 ] Completed train batch 1 discriminator backward 72.054 ms, 7.68 s total +Epoch [195] Step [1/2], gen_loss: 0.406, disc_loss: 0.014 +[ 2023-09-21 12:37:23 ] Completed train batch 1 metrics update 2.764 ms, 7.69 s total +Saving checkpoint at epoch 195 train batch 1 +[ 2023-09-21 12:37:24 ] Completed saving temp checkpoint 465.934 ms, 8.15 s total +[ 2023-09-21 12:37:24 ] Completed replacing temp checkpoint with checkpoint 29.662 ms, 8.18 s total +Epoch [195] :: gen_loss: 0.414, disc_loss: 0.014 +[ 2023-09-21 12:37:24 ] Completed training generator for epoch 195 48.553 ms, 8.23 s total +[ 2023-09-21 12:37:24 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 195 from eval step 0 + +[ 2023-09-21 12:37:25 ] Completed eval batch 0 to device 1,129.987 ms, 1.13 s total +[ 2023-09-21 12:37:25 ] Completed eval batch 0 forward 24.808 ms, 1.15 s total +[ 2023-09-21 12:37:25 ] Completed eval batch 0 recons_loss 0.294 ms, 1.16 s total +[ 2023-09-21 12:37:25 ] Completed eval batch 0 metrics update 0.560 ms, 1.16 s total +Saving checkpoint at epoch 195 val batch 0 +[ 2023-09-21 12:37:25 ] Completed saving temp checkpoint 446.513 ms, 1.60 s total +[ 2023-09-21 12:37:26 ] Completed replacing temp checkpoint with checkpoint 28.658 ms, 1.63 s total +[ 2023-09-21 12:37:26 ] Completed eval batch 1 to device 387.610 ms, 2.02 s total +[ 2023-09-21 12:37:26 ] Completed eval batch 1 forward 39.514 ms, 2.06 s total +[ 2023-09-21 12:37:26 ] Completed eval batch 1 recons_loss 0.500 ms, 2.06 s total +[ 2023-09-21 12:37:26 ] Completed eval batch 1 metrics update 196.808 ms, 2.26 s total +Saving checkpoint at epoch 195 val batch 1 +Epoch 195 val loss: 0.0577 +[ 2023-09-21 12:37:27 ] Completed saving temp checkpoint 461.424 ms, 2.72 s total +[ 2023-09-21 12:37:27 ] Completed replacing temp checkpoint with checkpoint 26.860 ms, 2.74 s total +[ 2023-09-21 12:37:27 ] Completed evaluating generator for epoch 195 52.582 ms, 2.80 s total + + +EPOCH :: 196 + + +[ 2023-09-21 12:37:27 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 196 from training step 0 + +[ 2023-09-21 12:37:30 ] Completed train batch 0 to device 3,199.336 ms, 3.20 s total +[ 2023-09-21 12:37:30 ] Completed train batch 0 generator forward 36.014 ms, 3.24 s total +[ 2023-09-21 12:37:30 ] Completed train batch 0 generator loss: 1.224 67.519 ms, 3.30 s total +[ 2023-09-21 12:37:31 ] Completed train batch 0 generator backward 596.916 ms, 3.90 s total +[ 2023-09-21 12:37:31 ] Completed train batch 0 discriminator loss 0.045 57.664 ms, 3.96 s total +[ 2023-09-21 12:37:31 ] Completed train batch 0 discriminator backward 89.635 ms, 4.05 s total +Epoch [196] Step [0/2], gen_loss: 0.412, disc_loss: 0.014 +[ 2023-09-21 12:37:31 ] Completed train batch 0 metrics update 27.127 ms, 4.07 s total +Saving checkpoint at epoch 196 train batch 0 +[ 2023-09-21 12:37:31 ] Completed saving temp checkpoint 474.765 ms, 4.55 s total +[ 2023-09-21 12:37:31 ] Completed replacing temp checkpoint with checkpoint 24.831 ms, 4.57 s total +[ 2023-09-21 12:37:33 ] Completed train batch 1 to device 1,820.142 ms, 6.39 s total +[ 2023-09-21 12:37:33 ] Completed train batch 1 generator forward 41.231 ms, 6.44 s total +[ 2023-09-21 12:37:33 ] Completed train batch 1 generator loss: 1.299 64.701 ms, 6.50 s total +[ 2023-09-21 12:37:34 ] Completed train batch 1 generator backward 919.217 ms, 7.42 s total +[ 2023-09-21 12:37:34 ] Completed train batch 1 discriminator loss 0.044 56.335 ms, 7.48 s total +[ 2023-09-21 12:37:34 ] Completed train batch 1 discriminator backward 70.745 ms, 7.55 s total +Epoch [196] Step [1/2], gen_loss: 0.407, disc_loss: 0.014 +[ 2023-09-21 12:37:34 ] Completed train batch 1 metrics update 4.616 ms, 7.55 s total +Saving checkpoint at epoch 196 train batch 1 +[ 2023-09-21 12:37:35 ] Completed saving temp checkpoint 475.563 ms, 8.03 s total +[ 2023-09-21 12:37:35 ] Completed replacing temp checkpoint with checkpoint 24.597 ms, 8.05 s total +Epoch [196] :: gen_loss: 0.410, disc_loss: 0.014 +[ 2023-09-21 12:37:35 ] Completed training generator for epoch 196 55.430 ms, 8.11 s total +[ 2023-09-21 12:37:35 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 196 from eval step 0 + +[ 2023-09-21 12:37:36 ] Completed eval batch 0 to device 1,112.970 ms, 1.11 s total +[ 2023-09-21 12:37:36 ] Completed eval batch 0 forward 25.194 ms, 1.14 s total +[ 2023-09-21 12:37:36 ] Completed eval batch 0 recons_loss 0.294 ms, 1.14 s total +[ 2023-09-21 12:37:36 ] Completed eval batch 0 metrics update 0.626 ms, 1.14 s total +Saving checkpoint at epoch 196 val batch 0 +[ 2023-09-21 12:37:36 ] Completed saving temp checkpoint 460.657 ms, 1.60 s total +[ 2023-09-21 12:37:36 ] Completed replacing temp checkpoint with checkpoint 33.225 ms, 1.63 s total +[ 2023-09-21 12:37:37 ] Completed eval batch 1 to device 452.853 ms, 2.09 s total +[ 2023-09-21 12:37:37 ] Completed eval batch 1 forward 22.050 ms, 2.11 s total +[ 2023-09-21 12:37:37 ] Completed eval batch 1 recons_loss 0.354 ms, 2.11 s total +[ 2023-09-21 12:37:37 ] Completed eval batch 1 metrics update 247.851 ms, 2.36 s total +Saving checkpoint at epoch 196 val batch 1 +Epoch 196 val loss: 0.0568 +[ 2023-09-21 12:37:38 ] Completed saving temp checkpoint 508.984 ms, 2.87 s total +[ 2023-09-21 12:37:38 ] Completed replacing temp checkpoint with checkpoint 24.996 ms, 2.89 s total +[ 2023-09-21 12:37:38 ] Completed evaluating generator for epoch 196 52.235 ms, 2.94 s total + + +EPOCH :: 197 + + +[ 2023-09-21 12:37:38 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 197 from training step 0 + +[ 2023-09-21 12:37:41 ] Completed train batch 0 to device 3,177.233 ms, 3.18 s total +[ 2023-09-21 12:37:41 ] Completed train batch 0 generator forward 34.093 ms, 3.21 s total +[ 2023-09-21 12:37:41 ] Completed train batch 0 generator loss: 1.335 76.146 ms, 3.29 s total +[ 2023-09-21 12:37:41 ] Completed train batch 0 generator backward 452.451 ms, 3.74 s total +[ 2023-09-21 12:37:42 ] Completed train batch 0 discriminator loss 0.041 64.362 ms, 3.80 s total +[ 2023-09-21 12:37:42 ] Completed train batch 0 discriminator backward 92.707 ms, 3.90 s total +Epoch [197] Step [0/2], gen_loss: 0.418, disc_loss: 0.014 +[ 2023-09-21 12:37:42 ] Completed train batch 0 metrics update 3.511 ms, 3.90 s total +Saving checkpoint at epoch 197 train batch 0 +[ 2023-09-21 12:37:42 ] Completed saving temp checkpoint 488.428 ms, 4.39 s total +[ 2023-09-21 12:37:42 ] Completed replacing temp checkpoint with checkpoint 33.013 ms, 4.42 s total +[ 2023-09-21 12:37:44 ] Completed train batch 1 to device 1,877.048 ms, 6.30 s total +[ 2023-09-21 12:37:44 ] Completed train batch 1 generator forward 22.691 ms, 6.32 s total +[ 2023-09-21 12:37:44 ] Completed train batch 1 generator loss: 1.281 64.011 ms, 6.39 s total +[ 2023-09-21 12:37:45 ] Completed train batch 1 generator backward 966.175 ms, 7.35 s total +[ 2023-09-21 12:37:45 ] Completed train batch 1 discriminator loss 0.041 56.787 ms, 7.41 s total +[ 2023-09-21 12:37:45 ] Completed train batch 1 discriminator backward 75.567 ms, 7.48 s total +Epoch [197] Step [1/2], gen_loss: 0.411, disc_loss: 0.013 +[ 2023-09-21 12:37:45 ] Completed train batch 1 metrics update 3.615 ms, 7.49 s total +Saving checkpoint at epoch 197 train batch 1 +[ 2023-09-21 12:37:46 ] Completed saving temp checkpoint 476.774 ms, 7.96 s total +[ 2023-09-21 12:37:46 ] Completed replacing temp checkpoint with checkpoint 23.169 ms, 7.99 s total +Epoch [197] :: gen_loss: 0.415, disc_loss: 0.014 +[ 2023-09-21 12:37:46 ] Completed training generator for epoch 197 53.490 ms, 8.04 s total +[ 2023-09-21 12:37:46 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 197 from eval step 0 + +[ 2023-09-21 12:37:47 ] Completed eval batch 0 to device 1,179.252 ms, 1.18 s total +[ 2023-09-21 12:37:47 ] Completed eval batch 0 forward 24.216 ms, 1.20 s total +[ 2023-09-21 12:37:47 ] Completed eval batch 0 recons_loss 0.250 ms, 1.20 s total +[ 2023-09-21 12:37:47 ] Completed eval batch 0 metrics update 0.569 ms, 1.20 s total +Saving checkpoint at epoch 197 val batch 0 +[ 2023-09-21 12:37:47 ] Completed saving temp checkpoint 442.414 ms, 1.65 s total +[ 2023-09-21 12:37:47 ] Completed replacing temp checkpoint with checkpoint 29.570 ms, 1.68 s total +[ 2023-09-21 12:37:48 ] Completed eval batch 1 to device 542.401 ms, 2.22 s total +[ 2023-09-21 12:37:48 ] Completed eval batch 1 forward 22.293 ms, 2.24 s total +[ 2023-09-21 12:37:48 ] Completed eval batch 1 recons_loss 0.406 ms, 2.24 s total +[ 2023-09-21 12:37:48 ] Completed eval batch 1 metrics update 4.930 ms, 2.25 s total +Saving checkpoint at epoch 197 val batch 1 +Epoch 197 val loss: 0.0523 +[ 2023-09-21 12:37:49 ] Completed saving temp checkpoint 515.372 ms, 2.76 s total +[ 2023-09-21 12:37:49 ] Completed replacing temp checkpoint with checkpoint 25.815 ms, 2.79 s total +[ 2023-09-21 12:37:49 ] Completed evaluating generator for epoch 197 50.537 ms, 2.84 s total + + +EPOCH :: 198 + + +[ 2023-09-21 12:37:49 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 198 from training step 0 + +[ 2023-09-21 12:37:52 ] Completed train batch 0 to device 3,136.246 ms, 3.14 s total +[ 2023-09-21 12:37:52 ] Completed train batch 0 generator forward 27.287 ms, 3.16 s total +[ 2023-09-21 12:37:52 ] Completed train batch 0 generator loss: 1.288 66.925 ms, 3.23 s total +[ 2023-09-21 12:37:52 ] Completed train batch 0 generator backward 581.281 ms, 3.81 s total +[ 2023-09-21 12:37:52 ] Completed train batch 0 discriminator loss 0.042 57.231 ms, 3.87 s total +[ 2023-09-21 12:37:53 ] Completed train batch 0 discriminator backward 102.083 ms, 3.97 s total +Epoch [198] Step [0/2], gen_loss: 0.413, disc_loss: 0.013 +[ 2023-09-21 12:37:53 ] Completed train batch 0 metrics update 6.490 ms, 3.98 s total +Saving checkpoint at epoch 198 train batch 0 +[ 2023-09-21 12:37:53 ] Completed saving temp checkpoint 460.335 ms, 4.44 s total +[ 2023-09-21 12:37:53 ] Completed replacing temp checkpoint with checkpoint 27.580 ms, 4.47 s total +[ 2023-09-21 12:37:54 ] Completed train batch 1 to device 1,417.674 ms, 5.88 s total +[ 2023-09-21 12:37:55 ] Completed train batch 1 generator forward 33.211 ms, 5.92 s total +[ 2023-09-21 12:37:55 ] Completed train batch 1 generator loss: 1.257 65.584 ms, 5.98 s total +[ 2023-09-21 12:37:56 ] Completed train batch 1 generator backward 1,292.290 ms, 7.27 s total +[ 2023-09-21 12:37:56 ] Completed train batch 1 discriminator loss 0.038 56.155 ms, 7.33 s total +[ 2023-09-21 12:37:56 ] Completed train batch 1 discriminator backward 78.001 ms, 7.41 s total +Epoch [198] Step [1/2], gen_loss: 0.411, disc_loss: 0.013 +[ 2023-09-21 12:37:56 ] Completed train batch 1 metrics update 0.878 ms, 7.41 s total +Saving checkpoint at epoch 198 train batch 1 +[ 2023-09-21 12:37:57 ] Completed saving temp checkpoint 490.681 ms, 7.90 s total +[ 2023-09-21 12:37:57 ] Completed replacing temp checkpoint with checkpoint 27.750 ms, 7.93 s total +Epoch [198] :: gen_loss: 0.412, disc_loss: 0.013 +[ 2023-09-21 12:37:57 ] Completed training generator for epoch 198 49.332 ms, 7.98 s total +[ 2023-09-21 12:37:57 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 198 from eval step 0 + +[ 2023-09-21 12:37:58 ] Completed eval batch 0 to device 1,173.745 ms, 1.17 s total +[ 2023-09-21 12:37:58 ] Completed eval batch 0 forward 42.868 ms, 1.22 s total +[ 2023-09-21 12:37:58 ] Completed eval batch 0 recons_loss 0.382 ms, 1.22 s total +[ 2023-09-21 12:37:58 ] Completed eval batch 0 metrics update 0.835 ms, 1.22 s total +Saving checkpoint at epoch 198 val batch 0 +[ 2023-09-21 12:37:58 ] Completed saving temp checkpoint 418.189 ms, 1.64 s total +[ 2023-09-21 12:37:58 ] Completed replacing temp checkpoint with checkpoint 24.562 ms, 1.66 s total +[ 2023-09-21 12:37:59 ] Completed eval batch 1 to device 403.744 ms, 2.06 s total +[ 2023-09-21 12:37:59 ] Completed eval batch 1 forward 39.488 ms, 2.10 s total +[ 2023-09-21 12:37:59 ] Completed eval batch 1 recons_loss 0.564 ms, 2.10 s total +[ 2023-09-21 12:37:59 ] Completed eval batch 1 metrics update 117.865 ms, 2.22 s total +Saving checkpoint at epoch 198 val batch 1 +Epoch 198 val loss: 0.0553 +[ 2023-09-21 12:37:59 ] Completed saving temp checkpoint 415.808 ms, 2.64 s total +[ 2023-09-21 12:37:59 ] Completed replacing temp checkpoint with checkpoint 29.471 ms, 2.67 s total +[ 2023-09-21 12:37:59 ] Completed evaluating generator for epoch 198 48.223 ms, 2.72 s total + + +EPOCH :: 199 + + +[ 2023-09-21 12:37:59 ] Completed Start training 0.000 ms, 0.00 s total + +Training / resuming epoch 199 from training step 0 + +[ 2023-09-21 12:38:02 ] Completed train batch 0 to device 2,974.733 ms, 2.97 s total +[ 2023-09-21 12:38:02 ] Completed train batch 0 generator forward 43.949 ms, 3.02 s total +[ 2023-09-21 12:38:02 ] Completed train batch 0 generator loss: 1.228 66.572 ms, 3.09 s total +[ 2023-09-21 12:38:03 ] Completed train batch 0 generator backward 849.776 ms, 3.94 s total +[ 2023-09-21 12:38:03 ] Completed train batch 0 discriminator loss 0.041 58.974 ms, 3.99 s total +[ 2023-09-21 12:38:03 ] Completed train batch 0 discriminator backward 105.970 ms, 4.10 s total +Epoch [199] Step [0/2], gen_loss: 0.413, disc_loss: 0.013 +[ 2023-09-21 12:38:03 ] Completed train batch 0 metrics update 8.685 ms, 4.11 s total +Saving checkpoint at epoch 199 train batch 0 +[ 2023-09-21 12:38:04 ] Completed saving temp checkpoint 440.545 ms, 4.55 s total +[ 2023-09-21 12:38:04 ] Completed replacing temp checkpoint with checkpoint 28.927 ms, 4.58 s total +[ 2023-09-21 12:38:05 ] Completed train batch 1 to device 1,080.842 ms, 5.66 s total +[ 2023-09-21 12:38:05 ] Completed train batch 1 generator forward 41.359 ms, 5.70 s total +[ 2023-09-21 12:38:05 ] Completed train batch 1 generator loss: 1.261 62.661 ms, 5.76 s total +[ 2023-09-21 12:38:07 ] Completed train batch 1 generator backward 1,982.879 ms, 7.75 s total +[ 2023-09-21 12:38:07 ] Completed train batch 1 discriminator loss 0.040 56.241 ms, 7.80 s total +[ 2023-09-21 12:38:07 ] Completed train batch 1 discriminator backward 90.145 ms, 7.89 s total +Epoch [199] Step [1/2], gen_loss: 0.410, disc_loss: 0.013 +[ 2023-09-21 12:38:07 ] Completed train batch 1 metrics update 5.615 ms, 7.90 s total +Saving checkpoint at epoch 199 train batch 1 +[ 2023-09-21 12:38:08 ] Completed saving temp checkpoint 489.762 ms, 8.39 s total +[ 2023-09-21 12:38:08 ] Completed replacing temp checkpoint with checkpoint 22.297 ms, 8.41 s total +Epoch [199] :: gen_loss: 0.411, disc_loss: 0.013 +[ 2023-09-21 12:38:08 ] Completed training generator for epoch 199 57.412 ms, 8.47 s total +[ 2023-09-21 12:38:08 ] Completed Start evaluation 0.000 ms, 0.00 s total + +Evaluating / resuming epoch 199 from eval step 0 + +[ 2023-09-21 12:38:09 ] Completed eval batch 0 to device 1,121.779 ms, 1.12 s total +[ 2023-09-21 12:38:09 ] Completed eval batch 0 forward 25.215 ms, 1.15 s total +[ 2023-09-21 12:38:09 ] Completed eval batch 0 recons_loss 0.346 ms, 1.15 s total +[ 2023-09-21 12:38:09 ] Completed eval batch 0 metrics update 0.597 ms, 1.15 s total +Saving checkpoint at epoch 199 val batch 0 +[ 2023-09-21 12:38:09 ] Completed saving temp checkpoint 435.496 ms, 1.58 s total +[ 2023-09-21 12:38:09 ] Completed replacing temp checkpoint with checkpoint 32.362 ms, 1.62 s total +[ 2023-09-21 12:38:10 ] Completed eval batch 1 to device 398.376 ms, 2.01 s total +[ 2023-09-21 12:38:10 ] Completed eval batch 1 forward 40.298 ms, 2.05 s total +[ 2023-09-21 12:38:10 ] Completed eval batch 1 recons_loss 0.564 ms, 2.06 s total +[ 2023-09-21 12:38:10 ] Completed eval batch 1 metrics update 236.426 ms, 2.29 s total +Saving checkpoint at epoch 199 val batch 1 +Epoch 199 val loss: 0.0556 +[ 2023-09-21 12:38:10 ] Completed saving temp checkpoint 407.429 ms, 2.70 s total +[ 2023-09-21 12:38:10 ] Completed replacing temp checkpoint with checkpoint 25.343 ms, 2.72 s total +[ 2023-09-21 12:38:11 ] Completed evaluating generator for epoch 199 54.197 ms, 2.78 s total +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 2623573) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python +Traceback (most recent call last): + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in + sys.exit(main()) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + return f(*args, **kwargs) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + run(args) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + elastic_launch( + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +======================================================== +train_cycling_gen.py FAILED +-------------------------------------------------------- +Failures: +[1]: + time : 2023-09-21_12:38:12 + host : sc11 + rank : 1 (local_rank: 1) + exitcode : -6 (pid: 2623574) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2623574 +[2]: + time : 2023-09-21_12:38:12 + host : sc11 + rank : 2 (local_rank: 2) + exitcode : -6 (pid: 2623575) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2623575 +[3]: + time : 2023-09-21_12:38:12 + host : sc11 + rank : 3 (local_rank: 3) + exitcode : -6 (pid: 2623576) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2623576 +[4]: + time : 2023-09-21_12:38:12 + host : sc11 + rank : 4 (local_rank: 4) + exitcode : -6 (pid: 2623577) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2623577 +[5]: + time : 2023-09-21_12:38:12 + host : sc11 + rank : 5 (local_rank: 5) + exitcode : -6 (pid: 2623578) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2623578 +-------------------------------------------------------- +Root Cause (first observed failure): +[0]: + time : 2023-09-21_12:38:12 + host : sc11 + rank : 0 (local_rank: 0) + exitcode : -6 (pid: 2623573) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2623573 +======================================================== diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_1.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_1.txt new file mode 100644 index 00000000..b3cf250b --- /dev/null +++ b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_1.txt @@ -0,0 +1,1310 @@ +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 11): env:// +| distributed init (rank 9): env:// +| distributed init (rank 10): env:// +| distributed init (rank 6): env:// +| distributed init (rank 7): env:// +| distributed init (rank 8): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 11): env:// +| distributed init (rank 6): env:// +| distributed init (rank 8): env:// +| distributed init (rank 9): env:// +| distributed init (rank 10): env:// +| distributed init (rank 7): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 9): env:// +| distributed init (rank 7): env:// +| distributed init (rank 11): env:// +| distributed init (rank 6): env:// +| distributed init (rank 10): env:// +| distributed init (rank 8): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 11): env:// +| distributed init (rank 7): env:// +| distributed init (rank 6): env:// +| distributed init (rank 10): env:// +| distributed init (rank 8): env:// +| distributed init (rank 9): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 11): env:// +| distributed init (rank 7): env:// +| distributed init (rank 9): env:// +| distributed init (rank 6): env:// +| distributed init (rank 10): env:// +| distributed init (rank 8): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 10): env:// +| distributed init (rank 9): env:// +| distributed init (rank 7): env:// +| distributed init (rank 6): env:// +| distributed init (rank 8): env:// +| distributed init (rank 11): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 7): env:// +| distributed init (rank 6): env:// +| distributed init (rank 8): env:// +| distributed init (rank 11): env:// +| distributed init (rank 9): env:// +| distributed init (rank 10): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 6): env:// +| distributed init (rank 7): env:// +| distributed init (rank 8): env:// +| distributed init (rank 11): env:// +| distributed init (rank 9): env:// +| distributed init (rank 10): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 11): env:// +| distributed init (rank 10): env:// +| distributed init (rank 9): env:// +| distributed init (rank 7): env:// +| distributed init (rank 6): env:// +| distributed init (rank 8): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 9): env:// +| distributed init (rank 8): env:// +| distributed init (rank 7): env:// +| distributed init (rank 10): env:// +| distributed init (rank 11): env:// +| distributed init (rank 6): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 11): env:// +| distributed init (rank 9): env:// +| distributed init (rank 7): env:// +| distributed init (rank 10): env:// +| distributed init (rank 6): env:// +| distributed init (rank 8): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 10): env:// +| distributed init (rank 9): env:// +| distributed init (rank 11): env:// +| distributed init (rank 8): env:// +| distributed init (rank 6): env:// +| distributed init (rank 7): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 7): env:// +| distributed init (rank 11): env:// +| distributed init (rank 9): env:// +| distributed init (rank 6): env:// +| distributed init (rank 8): env:// +| distributed init (rank 10): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 9): env:// +| distributed init (rank 6): env:// +| distributed init (rank 11): env:// +| distributed init (rank 7): env:// +| distributed init (rank 8): env:// +| distributed init (rank 10): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 7): env:// +| distributed init (rank 11): env:// +| distributed init (rank 9): env:// +| distributed init (rank 6): env:// +| distributed init (rank 8): env:// +| distributed init (rank 10): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 9): env:// +| distributed init (rank 8): env:// +| distributed init (rank 6): env:// +| distributed init (rank 7): env:// +| distributed init (rank 11): env:// +| distributed init (rank 10): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 7): env:// +| distributed init (rank 8): env:// +| distributed init (rank 9): env:// +| distributed init (rank 10): env:// +| distributed init (rank 6): env:// +| distributed init (rank 11): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 10): env:// +| distributed init (rank 11): env:// +| distributed init (rank 8): env:// +| distributed init (rank 7): env:// +| distributed init (rank 6): env:// +| distributed init (rank 9): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 11): env:// +| distributed init (rank 7): env:// +| distributed init (rank 6): env:// +| distributed init (rank 9): env:// +| distributed init (rank 10): env:// +| distributed init (rank 8): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 11): env:// +| distributed init (rank 6): env:// +| distributed init (rank 8): env:// +| distributed init (rank 7): env:// +| distributed init (rank 9): env:// +| distributed init (rank 10): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 9): env:// +| distributed init (rank 7): env:// +| distributed init (rank 6): env:// +| distributed init (rank 11): env:// +| distributed init (rank 10): env:// +| distributed init (rank 8): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 8): env:// +| distributed init (rank 7): env:// +| distributed init (rank 11): env:// +| distributed init (rank 10): env:// +| distributed init (rank 9): env:// +| distributed init (rank 6): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 6): env:// +| distributed init (rank 11): env:// +| distributed init (rank 7): env:// +| distributed init (rank 9): env:// +| distributed init (rank 8): env:// +| distributed init (rank 10): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 6): env:// +| distributed init (rank 7): env:// +| distributed init (rank 11): env:// +| distributed init (rank 8): env:// +| distributed init (rank 9): env:// +| distributed init (rank 10): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 9): env:// +| distributed init (rank 8): env:// +| distributed init (rank 7): env:// +| distributed init (rank 11): env:// +| distributed init (rank 10): env:// +| distributed init (rank 6): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 6): env:// +| distributed init (rank 7): env:// +| distributed init (rank 10): env:// +| distributed init (rank 9): env:// +| distributed init (rank 11): env:// +| distributed init (rank 8): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 6): env:// +| distributed init (rank 11): env:// +| distributed init (rank 7): env:// +| distributed init (rank 9): env:// +| distributed init (rank 8): env:// +| distributed init (rank 10): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 11): env:// +| distributed init (rank 9): env:// +| distributed init (rank 6): env:// +| distributed init (rank 7): env:// +| distributed init (rank 10): env:// +| distributed init (rank 8): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 7): env:// +| distributed init (rank 9): env:// +| distributed init (rank 6): env:// +| distributed init (rank 11): env:// +| distributed init (rank 8): env:// +| distributed init (rank 10): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 11): env:// +| distributed init (rank 6): env:// +| distributed init (rank 8): env:// +| distributed init (rank 7): env:// +| distributed init (rank 10): env:// +| distributed init (rank 9): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 7): env:// +| distributed init (rank 8): env:// +| distributed init (rank 6): env:// +| distributed init (rank 9): env:// +| distributed init (rank 11): env:// +| distributed init (rank 10): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 2789333) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python +Traceback (most recent call last): + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in + sys.exit(main()) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + return f(*args, **kwargs) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + run(args) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + elastic_launch( + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +======================================================== +train_cycling_gen.py FAILED +-------------------------------------------------------- +Failures: +[1]: + time : 2023-09-21_12:38:12 + host : sc14 + rank : 7 (local_rank: 1) + exitcode : -6 (pid: 2789334) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2789334 +[2]: + time : 2023-09-21_12:38:12 + host : sc14 + rank : 8 (local_rank: 2) + exitcode : -6 (pid: 2789335) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2789335 +[3]: + time : 2023-09-21_12:38:12 + host : sc14 + rank : 9 (local_rank: 3) + exitcode : -6 (pid: 2789336) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2789336 +[4]: + time : 2023-09-21_12:38:12 + host : sc14 + rank : 10 (local_rank: 4) + exitcode : -6 (pid: 2789337) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2789337 +[5]: + time : 2023-09-21_12:38:12 + host : sc14 + rank : 11 (local_rank: 5) + exitcode : -6 (pid: 2789338) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2789338 +-------------------------------------------------------- +Root Cause (first observed failure): +[0]: + time : 2023-09-21_12:38:12 + host : sc14 + rank : 6 (local_rank: 0) + exitcode : -6 (pid: 2789333) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2789333 +======================================================== diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_10.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_10.txt new file mode 100644 index 00000000..23b06cbd --- /dev/null +++ b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_10.txt @@ -0,0 +1,1310 @@ +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 63): env:// +| distributed init (rank 64): env:// +| distributed init (rank 62): env:// +| distributed init (rank 65): env:// +| distributed init (rank 60): env:// +| distributed init (rank 61): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 63): env:// +| distributed init (rank 65): env:// +| distributed init (rank 60): env:// +| distributed init (rank 62): env:// +| distributed init (rank 61): env:// +| distributed init (rank 64): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 62): env:// +| distributed init (rank 64): env:// +| distributed init (rank 63): env:// +| distributed init (rank 65): env:// +| distributed init (rank 60): env:// +| distributed init (rank 61): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 63): env:// +| distributed init (rank 64): env:// +| distributed init (rank 65): env:// +| distributed init (rank 60): env:// +| distributed init (rank 61): env:// +| distributed init (rank 62): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 63): env:// +| distributed init (rank 64): env:// +| distributed init (rank 65): env:// +| distributed init (rank 60): env:// +| distributed init (rank 62): env:// +| distributed init (rank 61): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 61): env:// +| distributed init (rank 60): env:// +| distributed init (rank 64): env:// +| distributed init (rank 63): env:// +| distributed init (rank 62): env:// +| distributed init (rank 65): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 63): env:// +| distributed init (rank 65): env:// +| distributed init (rank 62): env:// +| distributed init (rank 64): env:// +| distributed init (rank 60): env:// +| distributed init (rank 61): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 61): env:// +| distributed init (rank 63): env:// +| distributed init (rank 62): env:// +| distributed init (rank 64): env:// +| distributed init (rank 60): env:// +| distributed init (rank 65): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 61): env:// +| distributed init (rank 62): env:// +| distributed init (rank 65): env:// +| distributed init (rank 64): env:// +| distributed init (rank 63): env:// +| distributed init (rank 60): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 63): env:// +| distributed init (rank 64): env:// +| distributed init (rank 62): env:// +| distributed init (rank 65): env:// +| distributed init (rank 61): env:// +| distributed init (rank 60): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 62): env:// +| distributed init (rank 61): env:// +| distributed init (rank 63): env:// +| distributed init (rank 65): env:// +| distributed init (rank 60): env:// +| distributed init (rank 64): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 62): env:// +| distributed init (rank 64): env:// +| distributed init (rank 61): env:// +| distributed init (rank 63): env:// +| distributed init (rank 60): env:// +| distributed init (rank 65): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 60): env:// +| distributed init (rank 64): env:// +| distributed init (rank 65): env:// +| distributed init (rank 61): env:// +| distributed init (rank 62): env:// +| distributed init (rank 63): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 65): env:// +| distributed init (rank 60): env:// +| distributed init (rank 62): env:// +| distributed init (rank 64): env:// +| distributed init (rank 63): env:// +| distributed init (rank 61): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 60): env:// +| distributed init (rank 65): env:// +| distributed init (rank 64): env:// +| distributed init (rank 61): env:// +| distributed init (rank 62): env:// +| distributed init (rank 63): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 65): env:// +| distributed init (rank 61): env:// +| distributed init (rank 62): env:// +| distributed init (rank 63): env:// +| distributed init (rank 60): env:// +| distributed init (rank 64): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 63): env:// +| distributed init (rank 64): env:// +| distributed init (rank 61): env:// +| distributed init (rank 65): env:// +| distributed init (rank 60): env:// +| distributed init (rank 62): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 65): env:// +| distributed init (rank 64): env:// +| distributed init (rank 63): env:// +| distributed init (rank 60): env:// +| distributed init (rank 62): env:// +| distributed init (rank 61): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 61): env:// +| distributed init (rank 62): env:// +| distributed init (rank 63): env:// +| distributed init (rank 60): env:// +| distributed init (rank 64): env:// +| distributed init (rank 65): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 61): env:// +| distributed init (rank 64): env:// +| distributed init (rank 62): env:// +| distributed init (rank 63): env:// +| distributed init (rank 65): env:// +| distributed init (rank 60): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 63): env:// +| distributed init (rank 64): env:// +| distributed init (rank 62): env:// +| distributed init (rank 61): env:// +| distributed init (rank 65): env:// +| distributed init (rank 60): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 61): env:// +| distributed init (rank 65): env:// +| distributed init (rank 62): env:// +| distributed init (rank 60): env:// +| distributed init (rank 64): env:// +| distributed init (rank 63): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 62): env:// +| distributed init (rank 60): env:// +| distributed init (rank 64): env:// +| distributed init (rank 65): env:// +| distributed init (rank 63): env:// +| distributed init (rank 61): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 62): env:// +| distributed init (rank 60): env:// +| distributed init (rank 65): env:// +| distributed init (rank 61): env:// +| distributed init (rank 64): env:// +| distributed init (rank 63): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 63): env:// +| distributed init (rank 61): env:// +| distributed init (rank 60): env:// +| distributed init (rank 64): env:// +| distributed init (rank 65): env:// +| distributed init (rank 62): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 60): env:// +| distributed init (rank 64): env:// +| distributed init (rank 62): env:// +| distributed init (rank 63): env:// +| distributed init (rank 65): env:// +| distributed init (rank 61): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 64): env:// +| distributed init (rank 60): env:// +| distributed init (rank 65): env:// +| distributed init (rank 63): env:// +| distributed init (rank 61): env:// +| distributed init (rank 62): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 60): env:// +| distributed init (rank 62): env:// +| distributed init (rank 61): env:// +| distributed init (rank 64): env:// +| distributed init (rank 63): env:// +| distributed init (rank 65): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 63): env:// +| distributed init (rank 62): env:// +| distributed init (rank 64): env:// +| distributed init (rank 65): env:// +| distributed init (rank 60): env:// +| distributed init (rank 61): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 65): env:// +| distributed init (rank 62): env:// +| distributed init (rank 63): env:// +| distributed init (rank 64): env:// +| distributed init (rank 61): env:// +| distributed init (rank 60): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 64): env:// +| distributed init (rank 60): env:// +| distributed init (rank 65): env:// +| distributed init (rank 62): env:// +| distributed init (rank 61): env:// +| distributed init (rank 63): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 1663528) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python +Traceback (most recent call last): + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in + sys.exit(main()) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + return f(*args, **kwargs) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + run(args) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + elastic_launch( + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +======================================================== +train_cycling_gen.py FAILED +-------------------------------------------------------- +Failures: +[1]: + time : 2023-09-21_12:38:12 + host : sc20 + rank : 61 (local_rank: 1) + exitcode : -6 (pid: 1663529) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 1663529 +[2]: + time : 2023-09-21_12:38:12 + host : sc20 + rank : 62 (local_rank: 2) + exitcode : -6 (pid: 1663530) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 1663530 +[3]: + time : 2023-09-21_12:38:12 + host : sc20 + rank : 63 (local_rank: 3) + exitcode : -6 (pid: 1663531) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 1663531 +[4]: + time : 2023-09-21_12:38:12 + host : sc20 + rank : 64 (local_rank: 4) + exitcode : -6 (pid: 1663532) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 1663532 +[5]: + time : 2023-09-21_12:38:12 + host : sc20 + rank : 65 (local_rank: 5) + exitcode : -6 (pid: 1663533) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 1663533 +-------------------------------------------------------- +Root Cause (first observed failure): +[0]: + time : 2023-09-21_12:38:12 + host : sc20 + rank : 60 (local_rank: 0) + exitcode : -6 (pid: 1663528) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 1663528 +======================================================== diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_2.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_2.txt new file mode 100644 index 00000000..fa722cbb --- /dev/null +++ b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_2.txt @@ -0,0 +1,1310 @@ +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 14): env:// +| distributed init (rank 12): env:// +| distributed init (rank 15): env:// +| distributed init (rank 17): env:// +| distributed init (rank 13): env:// +| distributed init (rank 16): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 17): env:// +| distributed init (rank 12): env:// +| distributed init (rank 13): env:// +| distributed init (rank 14): env:// +| distributed init (rank 15): env:// +| distributed init (rank 16): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 16): env:// +| distributed init (rank 15): env:// +| distributed init (rank 12): env:// +| distributed init (rank 14): env:// +| distributed init (rank 17): env:// +| distributed init (rank 13): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 16): env:// +| distributed init (rank 15): env:// +| distributed init (rank 17): env:// +| distributed init (rank 12): env:// +| distributed init (rank 14): env:// +| distributed init (rank 13): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 12): env:// +| distributed init (rank 16): env:// +| distributed init (rank 17): env:// +| distributed init (rank 13): env:// +| distributed init (rank 15): env:// +| distributed init (rank 14): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 12): env:// +| distributed init (rank 13): env:// +| distributed init (rank 16): env:// +| distributed init (rank 14): env:// +| distributed init (rank 17): env:// +| distributed init (rank 15): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 17): env:// +| distributed init (rank 15): env:// +| distributed init (rank 14): env:// +| distributed init (rank 16): env:// +| distributed init (rank 12): env:// +| distributed init (rank 13): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 13): env:// +| distributed init (rank 16): env:// +| distributed init (rank 12): env:// +| distributed init (rank 14): env:// +| distributed init (rank 15): env:// +| distributed init (rank 17): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 14): env:// +| distributed init (rank 13): env:// +| distributed init (rank 16): env:// +| distributed init (rank 15): env:// +| distributed init (rank 12): env:// +| distributed init (rank 17): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 17): env:// +| distributed init (rank 14): env:// +| distributed init (rank 15): env:// +| distributed init (rank 12): env:// +| distributed init (rank 16): env:// +| distributed init (rank 13): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 14): env:// +| distributed init (rank 15): env:// +| distributed init (rank 16): env:// +| distributed init (rank 17): env:// +| distributed init (rank 12): env:// +| distributed init (rank 13): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 15): env:// +| distributed init (rank 13): env:// +| distributed init (rank 16): env:// +| distributed init (rank 12): env:// +| distributed init (rank 14): env:// +| distributed init (rank 17): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 12): env:// +| distributed init (rank 15): env:// +| distributed init (rank 16): env:// +| distributed init (rank 17): env:// +| distributed init (rank 13): env:// +| distributed init (rank 14): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 17): env:// +| distributed init (rank 15): env:// +| distributed init (rank 13): env:// +| distributed init (rank 16): env:// +| distributed init (rank 12): env:// +| distributed init (rank 14): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 17): env:// +| distributed init (rank 13): env:// +| distributed init (rank 12): env:// +| distributed init (rank 14): env:// +| distributed init (rank 15): env:// +| distributed init (rank 16): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 14): env:// +| distributed init (rank 16): env:// +| distributed init (rank 12): env:// +| distributed init (rank 15): env:// +| distributed init (rank 13): env:// +| distributed init (rank 17): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 17): env:// +| distributed init (rank 16): env:// +| distributed init (rank 13): env:// +| distributed init (rank 12): env:// +| distributed init (rank 15): env:// +| distributed init (rank 14): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 17): env:// +| distributed init (rank 14): env:// +| distributed init (rank 15): env:// +| distributed init (rank 13): env:// +| distributed init (rank 16): env:// +| distributed init (rank 12): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 12): env:// +| distributed init (rank 16): env:// +| distributed init (rank 17): env:// +| distributed init (rank 15): env:// +| distributed init (rank 13): env:// +| distributed init (rank 14): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 16): env:// +| distributed init (rank 14): env:// +| distributed init (rank 15): env:// +| distributed init (rank 13): env:// +| distributed init (rank 12): env:// +| distributed init (rank 17): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 12): env:// +| distributed init (rank 15): env:// +| distributed init (rank 14): env:// +| distributed init (rank 16): env:// +| distributed init (rank 13): env:// +| distributed init (rank 17): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 12): env:// +| distributed init (rank 16): env:// +| distributed init (rank 17): env:// +| distributed init (rank 14): env:// +| distributed init (rank 13): env:// +| distributed init (rank 15): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 16): env:// +| distributed init (rank 17): env:// +| distributed init (rank 12): env:// +| distributed init (rank 14): env:// +| distributed init (rank 15): env:// +| distributed init (rank 13): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 13): env:// +| distributed init (rank 16): env:// +| distributed init (rank 15): env:// +| distributed init (rank 12): env:// +| distributed init (rank 14): env:// +| distributed init (rank 17): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 17): env:// +| distributed init (rank 16): env:// +| distributed init (rank 12): env:// +| distributed init (rank 15): env:// +| distributed init (rank 13): env:// +| distributed init (rank 14): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 15): env:// +| distributed init (rank 14): env:// +| distributed init (rank 17): env:// +| distributed init (rank 16): env:// +| distributed init (rank 12): env:// +| distributed init (rank 13): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 12): env:// +| distributed init (rank 16): env:// +| distributed init (rank 14): env:// +| distributed init (rank 13): env:// +| distributed init (rank 17): env:// +| distributed init (rank 15): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 15): env:// +| distributed init (rank 14): env:// +| distributed init (rank 12): env:// +| distributed init (rank 16): env:// +| distributed init (rank 13): env:// +| distributed init (rank 17): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 15): env:// +| distributed init (rank 12): env:// +| distributed init (rank 14): env:// +| distributed init (rank 13): env:// +| distributed init (rank 16): env:// +| distributed init (rank 17): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 16): env:// +| distributed init (rank 13): env:// +| distributed init (rank 14): env:// +| distributed init (rank 15): env:// +| distributed init (rank 12): env:// +| distributed init (rank 17): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 13): env:// +| distributed init (rank 15): env:// +| distributed init (rank 14): env:// +| distributed init (rank 16): env:// +| distributed init (rank 17): env:// +| distributed init (rank 12): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 598797) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python +Traceback (most recent call last): + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in + sys.exit(main()) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + return f(*args, **kwargs) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + run(args) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + elastic_launch( + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +======================================================= +train_cycling_gen.py FAILED +------------------------------------------------------- +Failures: +[1]: + time : 2023-09-21_12:38:12 + host : sc12 + rank : 13 (local_rank: 1) + exitcode : -6 (pid: 598798) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 598798 +[2]: + time : 2023-09-21_12:38:12 + host : sc12 + rank : 14 (local_rank: 2) + exitcode : -6 (pid: 598799) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 598799 +[3]: + time : 2023-09-21_12:38:12 + host : sc12 + rank : 15 (local_rank: 3) + exitcode : -6 (pid: 598800) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 598800 +[4]: + time : 2023-09-21_12:38:12 + host : sc12 + rank : 16 (local_rank: 4) + exitcode : -6 (pid: 598801) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 598801 +[5]: + time : 2023-09-21_12:38:12 + host : sc12 + rank : 17 (local_rank: 5) + exitcode : -6 (pid: 598802) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 598802 +------------------------------------------------------- +Root Cause (first observed failure): +[0]: + time : 2023-09-21_12:38:12 + host : sc12 + rank : 12 (local_rank: 0) + exitcode : -6 (pid: 598797) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 598797 +======================================================= diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_3.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_3.txt new file mode 100644 index 00000000..90e93581 --- /dev/null +++ b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_3.txt @@ -0,0 +1,1310 @@ +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 20): env:// +| distributed init (rank 22): env:// +| distributed init (rank 18): env:// +| distributed init (rank 23): env:// +| distributed init (rank 21): env:// +| distributed init (rank 19): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 18): env:// +| distributed init (rank 19): env:// +| distributed init (rank 20): env:// +| distributed init (rank 21): env:// +| distributed init (rank 23): env:// +| distributed init (rank 22): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 18): env:// +| distributed init (rank 23): env:// +| distributed init (rank 20): env:// +| distributed init (rank 22): env:// +| distributed init (rank 19): env:// +| distributed init (rank 21): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 23): env:// +| distributed init (rank 20): env:// +| distributed init (rank 22): env:// +| distributed init (rank 21): env:// +| distributed init (rank 18): env:// +| distributed init (rank 19): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 20): env:// +| distributed init (rank 18): env:// +| distributed init (rank 21): env:// +| distributed init (rank 22): env:// +| distributed init (rank 19): env:// +| distributed init (rank 23): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 19): env:// +| distributed init (rank 21): env:// +| distributed init (rank 18): env:// +| distributed init (rank 22): env:// +| distributed init (rank 23): env:// +| distributed init (rank 20): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 20): env:// +| distributed init (rank 19): env:// +| distributed init (rank 21): env:// +| distributed init (rank 18): env:// +| distributed init (rank 23): env:// +| distributed init (rank 22): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 19): env:// +| distributed init (rank 21): env:// +| distributed init (rank 18): env:// +| distributed init (rank 22): env:// +| distributed init (rank 23): env:// +| distributed init (rank 20): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 21): env:// +| distributed init (rank 23): env:// +| distributed init (rank 20): env:// +| distributed init (rank 22): env:// +| distributed init (rank 19): env:// +| distributed init (rank 18): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 19): env:// +| distributed init (rank 22): env:// +| distributed init (rank 20): env:// +| distributed init (rank 18): env:// +| distributed init (rank 23): env:// +| distributed init (rank 21): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 23): env:// +| distributed init (rank 19): env:// +| distributed init (rank 21): env:// +| distributed init (rank 22): env:// +| distributed init (rank 20): env:// +| distributed init (rank 18): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 22): env:// +| distributed init (rank 18): env:// +| distributed init (rank 19): env:// +| distributed init (rank 20): env:// +| distributed init (rank 21): env:// +| distributed init (rank 23): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 20): env:// +| distributed init (rank 21): env:// +| distributed init (rank 19): env:// +| distributed init (rank 22): env:// +| distributed init (rank 18): env:// +| distributed init (rank 23): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 18): env:// +| distributed init (rank 23): env:// +| distributed init (rank 19): env:// +| distributed init (rank 20): env:// +| distributed init (rank 22): env:// +| distributed init (rank 21): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 21): env:// +| distributed init (rank 18): env:// +| distributed init (rank 19): env:// +| distributed init (rank 23): env:// +| distributed init (rank 22): env:// +| distributed init (rank 20): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 18): env:// +| distributed init (rank 19): env:// +| distributed init (rank 21): env:// +| distributed init (rank 23): env:// +| distributed init (rank 20): env:// +| distributed init (rank 22): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 19): env:// +| distributed init (rank 23): env:// +| distributed init (rank 22): env:// +| distributed init (rank 21): env:// +| distributed init (rank 18): env:// +| distributed init (rank 20): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 21): env:// +| distributed init (rank 18): env:// +| distributed init (rank 19): env:// +| distributed init (rank 23): env:// +| distributed init (rank 22): env:// +| distributed init (rank 20): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 20): env:// +| distributed init (rank 22): env:// +| distributed init (rank 18): env:// +| distributed init (rank 21): env:// +| distributed init (rank 23): env:// +| distributed init (rank 19): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 22): env:// +| distributed init (rank 21): env:// +| distributed init (rank 23): env:// +| distributed init (rank 18): env:// +| distributed init (rank 20): env:// +| distributed init (rank 19): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 19): env:// +| distributed init (rank 20): env:// +| distributed init (rank 18): env:// +| distributed init (rank 22): env:// +| distributed init (rank 23): env:// +| distributed init (rank 21): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 18): env:// +| distributed init (rank 23): env:// +| distributed init (rank 22): env:// +| distributed init (rank 19): env:// +| distributed init (rank 20): env:// +| distributed init (rank 21): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 18): env:// +| distributed init (rank 20): env:// +| distributed init (rank 22): env:// +| distributed init (rank 21): env:// +| distributed init (rank 23): env:// +| distributed init (rank 19): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 22): env:// +| distributed init (rank 21): env:// +| distributed init (rank 18): env:// +| distributed init (rank 23): env:// +| distributed init (rank 19): env:// +| distributed init (rank 20): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 18): env:// +| distributed init (rank 23): env:// +| distributed init (rank 22): env:// +| distributed init (rank 20): env:// +| distributed init (rank 21): env:// +| distributed init (rank 19): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 20): env:// +| distributed init (rank 23): env:// +| distributed init (rank 22): env:// +| distributed init (rank 19): env:// +| distributed init (rank 21): env:// +| distributed init (rank 18): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 21): env:// +| distributed init (rank 22): env:// +| distributed init (rank 23): env:// +| distributed init (rank 20): env:// +| distributed init (rank 18): env:// +| distributed init (rank 19): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 22): env:// +| distributed init (rank 19): env:// +| distributed init (rank 21): env:// +| distributed init (rank 18): env:// +| distributed init (rank 23): env:// +| distributed init (rank 20): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 19): env:// +| distributed init (rank 21): env:// +| distributed init (rank 23): env:// +| distributed init (rank 18): env:// +| distributed init (rank 22): env:// +| distributed init (rank 20): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 20): env:// +| distributed init (rank 18): env:// +| distributed init (rank 21): env:// +| distributed init (rank 19): env:// +| distributed init (rank 22): env:// +| distributed init (rank 23): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 21): env:// +| distributed init (rank 19): env:// +| distributed init (rank 22): env:// +| distributed init (rank 20): env:// +| distributed init (rank 18): env:// +| distributed init (rank 23): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 436434) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python +Traceback (most recent call last): + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in + sys.exit(main()) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + return f(*args, **kwargs) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + run(args) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + elastic_launch( + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +======================================================= +train_cycling_gen.py FAILED +------------------------------------------------------- +Failures: +[1]: + time : 2023-09-21_12:38:12 + host : sc18.strongc.ai + rank : 19 (local_rank: 1) + exitcode : -6 (pid: 436435) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 436435 +[2]: + time : 2023-09-21_12:38:12 + host : sc18.strongc.ai + rank : 20 (local_rank: 2) + exitcode : -6 (pid: 436436) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 436436 +[3]: + time : 2023-09-21_12:38:12 + host : sc18.strongc.ai + rank : 21 (local_rank: 3) + exitcode : -6 (pid: 436437) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 436437 +[4]: + time : 2023-09-21_12:38:12 + host : sc18.strongc.ai + rank : 22 (local_rank: 4) + exitcode : -6 (pid: 436438) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 436438 +[5]: + time : 2023-09-21_12:38:12 + host : sc18.strongc.ai + rank : 23 (local_rank: 5) + exitcode : -6 (pid: 436439) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 436439 +------------------------------------------------------- +Root Cause (first observed failure): +[0]: + time : 2023-09-21_12:38:12 + host : sc18.strongc.ai + rank : 18 (local_rank: 0) + exitcode : -6 (pid: 436434) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 436434 +======================================================= diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_4.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_4.txt new file mode 100644 index 00000000..a4626165 --- /dev/null +++ b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_4.txt @@ -0,0 +1,1310 @@ +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 29): env:// +| distributed init (rank 25): env:// +| distributed init (rank 26): env:// +| distributed init (rank 24): env:// +| distributed init (rank 28): env:// +| distributed init (rank 27): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 28): env:// +| distributed init (rank 26): env:// +| distributed init (rank 24): env:// +| distributed init (rank 29): env:// +| distributed init (rank 27): env:// +| distributed init (rank 25): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 28): env:// +| distributed init (rank 29): env:// +| distributed init (rank 26): env:// +| distributed init (rank 27): env:// +| distributed init (rank 24): env:// +| distributed init (rank 25): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 26): env:// +| distributed init (rank 27): env:// +| distributed init (rank 28): env:// +| distributed init (rank 29): env:// +| distributed init (rank 24): env:// +| distributed init (rank 25): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 29): env:// +| distributed init (rank 24): env:// +| distributed init (rank 28): env:// +| distributed init (rank 26): env:// +| distributed init (rank 25): env:// +| distributed init (rank 27): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 25): env:// +| distributed init (rank 27): env:// +| distributed init (rank 24): env:// +| distributed init (rank 29): env:// +| distributed init (rank 26): env:// +| distributed init (rank 28): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 26): env:// +| distributed init (rank 27): env:// +| distributed init (rank 24): env:// +| distributed init (rank 29): env:// +| distributed init (rank 28): env:// +| distributed init (rank 25): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 28): env:// +| distributed init (rank 26): env:// +| distributed init (rank 27): env:// +| distributed init (rank 25): env:// +| distributed init (rank 29): env:// +| distributed init (rank 24): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 28): env:// +| distributed init (rank 24): env:// +| distributed init (rank 26): env:// +| distributed init (rank 27): env:// +| distributed init (rank 29): env:// +| distributed init (rank 25): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 26): env:// +| distributed init (rank 29): env:// +| distributed init (rank 28): env:// +| distributed init (rank 25): env:// +| distributed init (rank 27): env:// +| distributed init (rank 24): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 27): env:// +| distributed init (rank 25): env:// +| distributed init (rank 26): env:// +| distributed init (rank 28): env:// +| distributed init (rank 24): env:// +| distributed init (rank 29): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 24): env:// +| distributed init (rank 25): env:// +| distributed init (rank 29): env:// +| distributed init (rank 27): env:// +| distributed init (rank 26): env:// +| distributed init (rank 28): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 24): env:// +| distributed init (rank 26): env:// +| distributed init (rank 28): env:// +| distributed init (rank 29): env:// +| distributed init (rank 27): env:// +| distributed init (rank 25): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 29): env:// +| distributed init (rank 28): env:// +| distributed init (rank 27): env:// +| distributed init (rank 25): env:// +| distributed init (rank 26): env:// +| distributed init (rank 24): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 26): env:// +| distributed init (rank 25): env:// +| distributed init (rank 28): env:// +| distributed init (rank 24): env:// +| distributed init (rank 27): env:// +| distributed init (rank 29): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 24): env:// +| distributed init (rank 26): env:// +| distributed init (rank 25): env:// +| distributed init (rank 28): env:// +| distributed init (rank 29): env:// +| distributed init (rank 27): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 28): env:// +| distributed init (rank 24): env:// +| distributed init (rank 25): env:// +| distributed init (rank 29): env:// +| distributed init (rank 26): env:// +| distributed init (rank 27): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 29): env:// +| distributed init (rank 24): env:// +| distributed init (rank 27): env:// +| distributed init (rank 26): env:// +| distributed init (rank 25): env:// +| distributed init (rank 28): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 25): env:// +| distributed init (rank 28): env:// +| distributed init (rank 27): env:// +| distributed init (rank 24): env:// +| distributed init (rank 29): env:// +| distributed init (rank 26): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 25): env:// +| distributed init (rank 27): env:// +| distributed init (rank 26): env:// +| distributed init (rank 24): env:// +| distributed init (rank 28): env:// +| distributed init (rank 29): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 24): env:// +| distributed init (rank 25): env:// +| distributed init (rank 29): env:// +| distributed init (rank 27): env:// +| distributed init (rank 26): env:// +| distributed init (rank 28): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 29): env:// +| distributed init (rank 24): env:// +| distributed init (rank 25): env:// +| distributed init (rank 26): env:// +| distributed init (rank 27): env:// +| distributed init (rank 28): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 28): env:// +| distributed init (rank 26): env:// +| distributed init (rank 29): env:// +| distributed init (rank 27): env:// +| distributed init (rank 24): env:// +| distributed init (rank 25): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 29): env:// +| distributed init (rank 24): env:// +| distributed init (rank 25): env:// +| distributed init (rank 26): env:// +| distributed init (rank 27): env:// +| distributed init (rank 28): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 24): env:// +| distributed init (rank 29): env:// +| distributed init (rank 25): env:// +| distributed init (rank 28): env:// +| distributed init (rank 27): env:// +| distributed init (rank 26): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 29): env:// +| distributed init (rank 28): env:// +| distributed init (rank 26): env:// +| distributed init (rank 25): env:// +| distributed init (rank 24): env:// +| distributed init (rank 27): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 25): env:// +| distributed init (rank 27): env:// +| distributed init (rank 24): env:// +| distributed init (rank 28): env:// +| distributed init (rank 29): env:// +| distributed init (rank 26): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 27): env:// +| distributed init (rank 29): env:// +| distributed init (rank 26): env:// +| distributed init (rank 24): env:// +| distributed init (rank 25): env:// +| distributed init (rank 28): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 24): env:// +| distributed init (rank 29): env:// +| distributed init (rank 26): env:// +| distributed init (rank 27): env:// +| distributed init (rank 25): env:// +| distributed init (rank 28): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 25): env:// +| distributed init (rank 26): env:// +| distributed init (rank 24): env:// +| distributed init (rank 27): env:// +| distributed init (rank 28): env:// +| distributed init (rank 29): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 26): env:// +| distributed init (rank 25): env:// +| distributed init (rank 27): env:// +| distributed init (rank 29): env:// +| distributed init (rank 28): env:// +| distributed init (rank 24): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 2749012) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python +Traceback (most recent call last): + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in + sys.exit(main()) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + return f(*args, **kwargs) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + run(args) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + elastic_launch( + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +======================================================== +train_cycling_gen.py FAILED +-------------------------------------------------------- +Failures: +[1]: + time : 2023-09-21_12:38:12 + host : sc13 + rank : 25 (local_rank: 1) + exitcode : -6 (pid: 2749013) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2749013 +[2]: + time : 2023-09-21_12:38:12 + host : sc13 + rank : 26 (local_rank: 2) + exitcode : -6 (pid: 2749014) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2749014 +[3]: + time : 2023-09-21_12:38:12 + host : sc13 + rank : 27 (local_rank: 3) + exitcode : -6 (pid: 2749015) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2749015 +[4]: + time : 2023-09-21_12:38:12 + host : sc13 + rank : 28 (local_rank: 4) + exitcode : -6 (pid: 2749016) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2749016 +[5]: + time : 2023-09-21_12:38:12 + host : sc13 + rank : 29 (local_rank: 5) + exitcode : -6 (pid: 2749017) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2749017 +-------------------------------------------------------- +Root Cause (first observed failure): +[0]: + time : 2023-09-21_12:38:12 + host : sc13 + rank : 24 (local_rank: 0) + exitcode : -6 (pid: 2749012) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2749012 +======================================================== diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_5.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_5.txt new file mode 100644 index 00000000..68a974e8 --- /dev/null +++ b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_5.txt @@ -0,0 +1,1310 @@ +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 34): env:// +| distributed init (rank 35): env:// +| distributed init (rank 31): env:// +| distributed init (rank 32): env:// +| distributed init (rank 33): env:// +| distributed init (rank 30): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 34): env:// +| distributed init (rank 32): env:// +| distributed init (rank 31): env:// +| distributed init (rank 35): env:// +| distributed init (rank 30): env:// +| distributed init (rank 33): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 30): env:// +| distributed init (rank 33): env:// +| distributed init (rank 34): env:// +| distributed init (rank 32): env:// +| distributed init (rank 31): env:// +| distributed init (rank 35): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 35): env:// +| distributed init (rank 33): env:// +| distributed init (rank 30): env:// +| distributed init (rank 32): env:// +| distributed init (rank 31): env:// +| distributed init (rank 34): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 30): env:// +| distributed init (rank 35): env:// +| distributed init (rank 31): env:// +| distributed init (rank 34): env:// +| distributed init (rank 33): env:// +| distributed init (rank 32): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 32): env:// +| distributed init (rank 33): env:// +| distributed init (rank 31): env:// +| distributed init (rank 34): env:// +| distributed init (rank 35): env:// +| distributed init (rank 30): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 32): env:// +| distributed init (rank 33): env:// +| distributed init (rank 34): env:// +| distributed init (rank 35): env:// +| distributed init (rank 31): env:// +| distributed init (rank 30): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 35): env:// +| distributed init (rank 31): env:// +| distributed init (rank 34): env:// +| distributed init (rank 30): env:// +| distributed init (rank 33): env:// +| distributed init (rank 32): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 32): env:// +| distributed init (rank 31): env:// +| distributed init (rank 30): env:// +| distributed init (rank 34): env:// +| distributed init (rank 35): env:// +| distributed init (rank 33): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 31): env:// +| distributed init (rank 34): env:// +| distributed init (rank 35): env:// +| distributed init (rank 33): env:// +| distributed init (rank 32): env:// +| distributed init (rank 30): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 35): env:// +| distributed init (rank 31): env:// +| distributed init (rank 30): env:// +| distributed init (rank 34): env:// +| distributed init (rank 32): env:// +| distributed init (rank 33): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 32): env:// +| distributed init (rank 34): env:// +| distributed init (rank 35): env:// +| distributed init (rank 33): env:// +| distributed init (rank 31): env:// +| distributed init (rank 30): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 35): env:// +| distributed init (rank 34): env:// +| distributed init (rank 32): env:// +| distributed init (rank 31): env:// +| distributed init (rank 33): env:// +| distributed init (rank 30): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 34): env:// +| distributed init (rank 31): env:// +| distributed init (rank 30): env:// +| distributed init (rank 33): env:// +| distributed init (rank 32): env:// +| distributed init (rank 35): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 34): env:// +| distributed init (rank 31): env:// +| distributed init (rank 35): env:// +| distributed init (rank 33): env:// +| distributed init (rank 30): env:// +| distributed init (rank 32): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 31): env:// +| distributed init (rank 30): env:// +| distributed init (rank 34): env:// +| distributed init (rank 32): env:// +| distributed init (rank 35): env:// +| distributed init (rank 33): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 34): env:// +| distributed init (rank 30): env:// +| distributed init (rank 35): env:// +| distributed init (rank 32): env:// +| distributed init (rank 33): env:// +| distributed init (rank 31): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 31): env:// +| distributed init (rank 35): env:// +| distributed init (rank 34): env:// +| distributed init (rank 32): env:// +| distributed init (rank 30): env:// +| distributed init (rank 33): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 31): env:// +| distributed init (rank 30): env:// +| distributed init (rank 34): env:// +| distributed init (rank 35): env:// +| distributed init (rank 32): env:// +| distributed init (rank 33): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 34): env:// +| distributed init (rank 31): env:// +| distributed init (rank 32): env:// +| distributed init (rank 33): env:// +| distributed init (rank 30): env:// +| distributed init (rank 35): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 32): env:// +| distributed init (rank 35): env:// +| distributed init (rank 31): env:// +| distributed init (rank 33): env:// +| distributed init (rank 34): env:// +| distributed init (rank 30): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 30): env:// +| distributed init (rank 32): env:// +| distributed init (rank 35): env:// +| distributed init (rank 34): env:// +| distributed init (rank 31): env:// +| distributed init (rank 33): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 32): env:// +| distributed init (rank 35): env:// +| distributed init (rank 30): env:// +| distributed init (rank 34): env:// +| distributed init (rank 31): env:// +| distributed init (rank 33): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 35): env:// +| distributed init (rank 34): env:// +| distributed init (rank 32): env:// +| distributed init (rank 30): env:// +| distributed init (rank 33): env:// +| distributed init (rank 31): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 30): env:// +| distributed init (rank 34): env:// +| distributed init (rank 32): env:// +| distributed init (rank 33): env:// +| distributed init (rank 31): env:// +| distributed init (rank 35): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 32): env:// +| distributed init (rank 34): env:// +| distributed init (rank 35): env:// +| distributed init (rank 30): env:// +| distributed init (rank 33): env:// +| distributed init (rank 31): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 34): env:// +| distributed init (rank 31): env:// +| distributed init (rank 35): env:// +| distributed init (rank 30): env:// +| distributed init (rank 32): env:// +| distributed init (rank 33): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 30): env:// +| distributed init (rank 32): env:// +| distributed init (rank 33): env:// +| distributed init (rank 35): env:// +| distributed init (rank 31): env:// +| distributed init (rank 34): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 30): env:// +| distributed init (rank 35): env:// +| distributed init (rank 32): env:// +| distributed init (rank 33): env:// +| distributed init (rank 31): env:// +| distributed init (rank 34): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 34): env:// +| distributed init (rank 32): env:// +| distributed init (rank 31): env:// +| distributed init (rank 35): env:// +| distributed init (rank 30): env:// +| distributed init (rank 33): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 30): env:// +| distributed init (rank 34): env:// +| distributed init (rank 35): env:// +| distributed init (rank 31): env:// +| distributed init (rank 33): env:// +| distributed init (rank 32): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 3421449) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python +Traceback (most recent call last): + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in + sys.exit(main()) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + return f(*args, **kwargs) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + run(args) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + elastic_launch( + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +======================================================== +train_cycling_gen.py FAILED +-------------------------------------------------------- +Failures: +[1]: + time : 2023-09-21_12:38:12 + host : sc17 + rank : 31 (local_rank: 1) + exitcode : -6 (pid: 3421450) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 3421450 +[2]: + time : 2023-09-21_12:38:12 + host : sc17 + rank : 32 (local_rank: 2) + exitcode : -6 (pid: 3421451) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 3421451 +[3]: + time : 2023-09-21_12:38:12 + host : sc17 + rank : 33 (local_rank: 3) + exitcode : -6 (pid: 3421452) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 3421452 +[4]: + time : 2023-09-21_12:38:12 + host : sc17 + rank : 34 (local_rank: 4) + exitcode : -6 (pid: 3421453) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 3421453 +[5]: + time : 2023-09-21_12:38:12 + host : sc17 + rank : 35 (local_rank: 5) + exitcode : -6 (pid: 3421454) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 3421454 +-------------------------------------------------------- +Root Cause (first observed failure): +[0]: + time : 2023-09-21_12:38:12 + host : sc17 + rank : 30 (local_rank: 0) + exitcode : -6 (pid: 3421449) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 3421449 +======================================================== diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_6.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_6.txt new file mode 100644 index 00000000..2882a604 --- /dev/null +++ b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_6.txt @@ -0,0 +1,1310 @@ +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 39): env:// +| distributed init (rank 40): env:// +| distributed init (rank 37): env:// +| distributed init (rank 38): env:// +| distributed init (rank 36): env:// +| distributed init (rank 41): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 38): env:// +| distributed init (rank 40): env:// +| distributed init (rank 37): env:// +| distributed init (rank 39): env:// +| distributed init (rank 36): env:// +| distributed init (rank 41): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 37): env:// +| distributed init (rank 41): env:// +| distributed init (rank 38): env:// +| distributed init (rank 36): env:// +| distributed init (rank 39): env:// +| distributed init (rank 40): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 37): env:// +| distributed init (rank 36): env:// +| distributed init (rank 40): env:// +| distributed init (rank 38): env:// +| distributed init (rank 39): env:// +| distributed init (rank 41): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 39): env:// +| distributed init (rank 37): env:// +| distributed init (rank 38): env:// +| distributed init (rank 36): env:// +| distributed init (rank 41): env:// +| distributed init (rank 40): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 36): env:// +| distributed init (rank 38): env:// +| distributed init (rank 39): env:// +| distributed init (rank 40): env:// +| distributed init (rank 41): env:// +| distributed init (rank 37): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 39): env:// +| distributed init (rank 38): env:// +| distributed init (rank 41): env:// +| distributed init (rank 37): env:// +| distributed init (rank 40): env:// +| distributed init (rank 36): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 37): env:// +| distributed init (rank 41): env:// +| distributed init (rank 39): env:// +| distributed init (rank 36): env:// +| distributed init (rank 40): env:// +| distributed init (rank 38): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 40): env:// +| distributed init (rank 39): env:// +| distributed init (rank 38): env:// +| distributed init (rank 41): env:// +| distributed init (rank 36): env:// +| distributed init (rank 37): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 38): env:// +| distributed init (rank 37): env:// +| distributed init (rank 39): env:// +| distributed init (rank 40): env:// +| distributed init (rank 41): env:// +| distributed init (rank 36): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 40): env:// +| distributed init (rank 37): env:// +| distributed init (rank 41): env:// +| distributed init (rank 38): env:// +| distributed init (rank 39): env:// +| distributed init (rank 36): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 39): env:// +| distributed init (rank 37): env:// +| distributed init (rank 38): env:// +| distributed init (rank 40): env:// +| distributed init (rank 41): env:// +| distributed init (rank 36): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 38): env:// +| distributed init (rank 36): env:// +| distributed init (rank 37): env:// +| distributed init (rank 41): env:// +| distributed init (rank 39): env:// +| distributed init (rank 40): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 37): env:// +| distributed init (rank 40): env:// +| distributed init (rank 38): env:// +| distributed init (rank 36): env:// +| distributed init (rank 39): env:// +| distributed init (rank 41): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 41): env:// +| distributed init (rank 38): env:// +| distributed init (rank 40): env:// +| distributed init (rank 39): env:// +| distributed init (rank 36): env:// +| distributed init (rank 37): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 40): env:// +| distributed init (rank 38): env:// +| distributed init (rank 36): env:// +| distributed init (rank 41): env:// +| distributed init (rank 39): env:// +| distributed init (rank 37): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 36): env:// +| distributed init (rank 41): env:// +| distributed init (rank 40): env:// +| distributed init (rank 38): env:// +| distributed init (rank 37): env:// +| distributed init (rank 39): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 40): env:// +| distributed init (rank 39): env:// +| distributed init (rank 38): env:// +| distributed init (rank 36): env:// +| distributed init (rank 37): env:// +| distributed init (rank 41): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 40): env:// +| distributed init (rank 39): env:// +| distributed init (rank 38): env:// +| distributed init (rank 37): env:// +| distributed init (rank 36): env:// +| distributed init (rank 41): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 36): env:// +| distributed init (rank 37): env:// +| distributed init (rank 40): env:// +| distributed init (rank 39): env:// +| distributed init (rank 41): env:// +| distributed init (rank 38): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 36): env:// +| distributed init (rank 38): env:// +| distributed init (rank 39): env:// +| distributed init (rank 40): env:// +| distributed init (rank 37): env:// +| distributed init (rank 41): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 41): env:// +| distributed init (rank 40): env:// +| distributed init (rank 38): env:// +| distributed init (rank 36): env:// +| distributed init (rank 37): env:// +| distributed init (rank 39): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 41): env:// +| distributed init (rank 39): env:// +| distributed init (rank 38): env:// +| distributed init (rank 40): env:// +| distributed init (rank 37): env:// +| distributed init (rank 36): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 37): env:// +| distributed init (rank 39): env:// +| distributed init (rank 40): env:// +| distributed init (rank 36): env:// +| distributed init (rank 38): env://| distributed init (rank 41): env:// + +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 38): env:// +| distributed init (rank 39): env:// +| distributed init (rank 41): env:// +| distributed init (rank 36): env:// +| distributed init (rank 37): env:// +| distributed init (rank 40): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 39): env:// +| distributed init (rank 38): env:// +| distributed init (rank 40): env:// +| distributed init (rank 41): env:// +| distributed init (rank 36): env:// +| distributed init (rank 37): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 38): env:// +| distributed init (rank 37): env:// +| distributed init (rank 36): env:// +| distributed init (rank 40): env:// +| distributed init (rank 41): env:// +| distributed init (rank 39): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 36): env:// +| distributed init (rank 37): env:// +| distributed init (rank 38): env:// +| distributed init (rank 41): env:// +| distributed init (rank 40): env:// +| distributed init (rank 39): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 37): env:// +| distributed init (rank 41): env:// +| distributed init (rank 39): env:// +| distributed init (rank 36): env:// +| distributed init (rank 38): env:// +| distributed init (rank 40): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 37): env:// +| distributed init (rank 38): env:// +| distributed init (rank 40): env:// +| distributed init (rank 41): env:// +| distributed init (rank 39): env:// +| distributed init (rank 36): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 39): env:// +| distributed init (rank 38): env:// +| distributed init (rank 36): env:// +| distributed init (rank 40): env:// +| distributed init (rank 37): env:// +| distributed init (rank 41): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 1500785) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python +Traceback (most recent call last): + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in + sys.exit(main()) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + return f(*args, **kwargs) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + run(args) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + elastic_launch( + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +======================================================== +train_cycling_gen.py FAILED +-------------------------------------------------------- +Failures: +[1]: + time : 2023-09-21_12:38:12 + host : sc9 + rank : 37 (local_rank: 1) + exitcode : -6 (pid: 1500786) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 1500786 +[2]: + time : 2023-09-21_12:38:12 + host : sc9 + rank : 38 (local_rank: 2) + exitcode : -6 (pid: 1500787) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 1500787 +[3]: + time : 2023-09-21_12:38:12 + host : sc9 + rank : 39 (local_rank: 3) + exitcode : -6 (pid: 1500788) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 1500788 +[4]: + time : 2023-09-21_12:38:12 + host : sc9 + rank : 40 (local_rank: 4) + exitcode : -6 (pid: 1500789) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 1500789 +[5]: + time : 2023-09-21_12:38:12 + host : sc9 + rank : 41 (local_rank: 5) + exitcode : -6 (pid: 1500790) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 1500790 +-------------------------------------------------------- +Root Cause (first observed failure): +[0]: + time : 2023-09-21_12:38:12 + host : sc9 + rank : 36 (local_rank: 0) + exitcode : -6 (pid: 1500785) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 1500785 +======================================================== diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_7.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_7.txt new file mode 100644 index 00000000..fb26dabd --- /dev/null +++ b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_7.txt @@ -0,0 +1,1310 @@ +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 42): env:// +| distributed init (rank 47): env:// +| distributed init (rank 43): env:// +| distributed init (rank 45): env:// +| distributed init (rank 44): env:// +| distributed init (rank 46): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 43): env:// +| distributed init (rank 44): env:// +| distributed init (rank 46): env:// +| distributed init (rank 42): env:// +| distributed init (rank 47): env:// +| distributed init (rank 45): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 47): env:// +| distributed init (rank 44): env:// +| distributed init (rank 42): env:// +| distributed init (rank 46): env:// +| distributed init (rank 43): env:// +| distributed init (rank 45): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 44): env:// +| distributed init (rank 45): env:// +| distributed init (rank 43): env:// +| distributed init (rank 46): env:// +| distributed init (rank 47): env:// +| distributed init (rank 42): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 46): env:// +| distributed init (rank 47): env:// +| distributed init (rank 44): env:// +| distributed init (rank 43): env:// +| distributed init (rank 42): env:// +| distributed init (rank 45): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 42): env:// +| distributed init (rank 44): env:// +| distributed init (rank 46): env:// +| distributed init (rank 45): env:// +| distributed init (rank 47): env:// +| distributed init (rank 43): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 45): env:// +| distributed init (rank 47): env:// +| distributed init (rank 42): env:// +| distributed init (rank 44): env:// +| distributed init (rank 46): env:// +| distributed init (rank 43): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 44): env:// +| distributed init (rank 43): env:// +| distributed init (rank 47): env:// +| distributed init (rank 45): env:// +| distributed init (rank 42): env:// +| distributed init (rank 46): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 46): env:// +| distributed init (rank 44): env:// +| distributed init (rank 42): env:// +| distributed init (rank 45): env:// +| distributed init (rank 47): env:// +| distributed init (rank 43): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 46): env:// +| distributed init (rank 45): env:// +| distributed init (rank 47): env:// +| distributed init (rank 44): env:// +| distributed init (rank 42): env:// +| distributed init (rank 43): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 43): env:// +| distributed init (rank 45): env:// +| distributed init (rank 42): env:// +| distributed init (rank 47): env:// +| distributed init (rank 44): env:// +| distributed init (rank 46): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 45): env:// +| distributed init (rank 44): env:// +| distributed init (rank 47): env:// +| distributed init (rank 42): env:// +| distributed init (rank 46): env:// +| distributed init (rank 43): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 45): env:// +| distributed init (rank 47): env:// +| distributed init (rank 46): env:// +| distributed init (rank 42): env:// +| distributed init (rank 44): env:// +| distributed init (rank 43): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 47): env:// +| distributed init (rank 43): env:// +| distributed init (rank 44): env:// +| distributed init (rank 45): env:// +| distributed init (rank 42): env:// +| distributed init (rank 46): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 44): env:// +| distributed init (rank 43): env:// +| distributed init (rank 46): env:// +| distributed init (rank 47): env:// +| distributed init (rank 42): env:// +| distributed init (rank 45): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 42): env:// +| distributed init (rank 47): env:// +| distributed init (rank 44): env:// +| distributed init (rank 46): env:// +| distributed init (rank 45): env:// +| distributed init (rank 43): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 45): env:// +| distributed init (rank 43): env:// +| distributed init (rank 46): env:// +| distributed init (rank 44): env:// +| distributed init (rank 42): env:// +| distributed init (rank 47): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 44): env:// +| distributed init (rank 45): env:// +| distributed init (rank 46): env:// +| distributed init (rank 42): env:// +| distributed init (rank 47): env:// +| distributed init (rank 43): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 42): env:// +| distributed init (rank 47): env:// +| distributed init (rank 45): env:// +| distributed init (rank 43): env:// +| distributed init (rank 44): env:// +| distributed init (rank 46): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 46): env:// +| distributed init (rank 44): env:// +| distributed init (rank 42): env:// +| distributed init (rank 47): env:// +| distributed init (rank 43): env:// +| distributed init (rank 45): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 46): env:// +| distributed init (rank 43): env:// +| distributed init (rank 45): env:// +| distributed init (rank 47): env:// +| distributed init (rank 44): env:// +| distributed init (rank 42): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 44): env:// +| distributed init (rank 43): env:// +| distributed init (rank 45): env:// +| distributed init (rank 42): env:// +| distributed init (rank 47): env:// +| distributed init (rank 46): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 43): env:// +| distributed init (rank 47): env:// +| distributed init (rank 45): env:// +| distributed init (rank 42): env:// +| distributed init (rank 44): env:// +| distributed init (rank 46): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 43): env:// +| distributed init (rank 46): env:// +| distributed init (rank 47): env:// +| distributed init (rank 44): env:// +| distributed init (rank 42): env:// +| distributed init (rank 45): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 44): env:// +| distributed init (rank 47): env:// +| distributed init (rank 43): env:// +| distributed init (rank 45): env:// +| distributed init (rank 46): env:// +| distributed init (rank 42): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 47): env:// +| distributed init (rank 42): env:// +| distributed init (rank 45): env:// +| distributed init (rank 46): env:// +| distributed init (rank 44): env:// +| distributed init (rank 43): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 47): env:// +| distributed init (rank 46): env:// +| distributed init (rank 44): env:// +| distributed init (rank 43): env:// +| distributed init (rank 45): env:// +| distributed init (rank 42): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 47): env:// +| distributed init (rank 43): env:// +| distributed init (rank 42): env:// +| distributed init (rank 46): env:// +| distributed init (rank 45): env:// +| distributed init (rank 44): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 42): env:// +| distributed init (rank 43): env:// +| distributed init (rank 45): env:// +| distributed init (rank 44): env:// +| distributed init (rank 46): env:// +| distributed init (rank 47): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 43): env:// +| distributed init (rank 47): env:// +| distributed init (rank 45): env:// +| distributed init (rank 46): env:// +| distributed init (rank 42): env:// +| distributed init (rank 44): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 42): env:// +| distributed init (rank 43): env:// +| distributed init (rank 45): env:// +| distributed init (rank 44): env:// +| distributed init (rank 46): env:// +| distributed init (rank 47): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 3214610) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python +Traceback (most recent call last): + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in + sys.exit(main()) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + return f(*args, **kwargs) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + run(args) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + elastic_launch( + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +======================================================== +train_cycling_gen.py FAILED +-------------------------------------------------------- +Failures: +[1]: + time : 2023-09-21_12:38:12 + host : sc19 + rank : 43 (local_rank: 1) + exitcode : -6 (pid: 3214611) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 3214611 +[2]: + time : 2023-09-21_12:38:12 + host : sc19 + rank : 44 (local_rank: 2) + exitcode : -6 (pid: 3214612) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 3214612 +[3]: + time : 2023-09-21_12:38:12 + host : sc19 + rank : 45 (local_rank: 3) + exitcode : -6 (pid: 3214613) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 3214613 +[4]: + time : 2023-09-21_12:38:12 + host : sc19 + rank : 46 (local_rank: 4) + exitcode : -6 (pid: 3214614) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 3214614 +[5]: + time : 2023-09-21_12:38:12 + host : sc19 + rank : 47 (local_rank: 5) + exitcode : -6 (pid: 3214615) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 3214615 +-------------------------------------------------------- +Root Cause (first observed failure): +[0]: + time : 2023-09-21_12:38:12 + host : sc19 + rank : 42 (local_rank: 0) + exitcode : -6 (pid: 3214610) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 3214610 +======================================================== diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_8.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_8.txt new file mode 100644 index 00000000..3461fba9 --- /dev/null +++ b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_8.txt @@ -0,0 +1,1310 @@ +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 53): env:// +| distributed init (rank 48): env:// +| distributed init (rank 49): env:// +| distributed init (rank 52): env:// +| distributed init (rank 50): env:// +| distributed init (rank 51): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 53): env:// +| distributed init (rank 51): env:// +| distributed init (rank 48): env:// +| distributed init (rank 50): env:// +| distributed init (rank 49): env:// +| distributed init (rank 52): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 50): env:// +| distributed init (rank 52): env:// +| distributed init (rank 48): env:// +| distributed init (rank 49): env:// +| distributed init (rank 53): env:// +| distributed init (rank 51): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 53): env:// +| distributed init (rank 49): env:// +| distributed init (rank 52): env:// +| distributed init (rank 50): env:// +| distributed init (rank 51): env:// +| distributed init (rank 48): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 48): env:// +| distributed init (rank 50): env:// +| distributed init (rank 52): env:// +| distributed init (rank 53): env:// +| distributed init (rank 51): env:// +| distributed init (rank 49): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 51): env:// +| distributed init (rank 52): env:// +| distributed init (rank 48): env:// +| distributed init (rank 53): env:// +| distributed init (rank 49): env:// +| distributed init (rank 50): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 52): env:// +| distributed init (rank 48): env:// +| distributed init (rank 51): env:// +| distributed init (rank 49): env:// +| distributed init (rank 50): env:// +| distributed init (rank 53): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 52): env:// +| distributed init (rank 50): env:// +| distributed init (rank 48): env:// +| distributed init (rank 53): env:// +| distributed init (rank 49): env:// +| distributed init (rank 51): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 52): env:// +| distributed init (rank 48): env:// +| distributed init (rank 49): env:// +| distributed init (rank 50): env:// +| distributed init (rank 51): env:// +| distributed init (rank 53): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 48): env:// +| distributed init (rank 49): env:// +| distributed init (rank 52): env:// +| distributed init (rank 50): env:// +| distributed init (rank 51): env:// +| distributed init (rank 53): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 50): env:// +| distributed init (rank 53): env:// +| distributed init (rank 52): env:// +| distributed init (rank 49): env:// +| distributed init (rank 51): env:// +| distributed init (rank 48): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 50): env:// +| distributed init (rank 51): env:// +| distributed init (rank 48): env:// +| distributed init (rank 53): env:// +| distributed init (rank 52): env:// +| distributed init (rank 49): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 48): env:// +| distributed init (rank 53): env:// +| distributed init (rank 52): env:// +| distributed init (rank 51): env:// +| distributed init (rank 49): env:// +| distributed init (rank 50): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 52): env:// +| distributed init (rank 53): env:// +| distributed init (rank 48): env:// +| distributed init (rank 49): env:// +| distributed init (rank 51): env:// +| distributed init (rank 50): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 50): env:// +| distributed init (rank 53): env:// +| distributed init (rank 51): env:// +| distributed init (rank 52): env:// +| distributed init (rank 49): env:// +| distributed init (rank 48): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 51): env:// +| distributed init (rank 52): env:// +| distributed init (rank 50): env:// +| distributed init (rank 48): env:// +| distributed init (rank 49): env:// +| distributed init (rank 53): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 52): env:// +| distributed init (rank 53): env:// +| distributed init (rank 51): env:// +| distributed init (rank 49): env:// +| distributed init (rank 48): env:// +| distributed init (rank 50): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 50): env:// +| distributed init (rank 51): env:// +| distributed init (rank 49): env:// +| distributed init (rank 48): env:// +| distributed init (rank 53): env:// +| distributed init (rank 52): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 51): env:// +| distributed init (rank 49): env:// +| distributed init (rank 52): env:// +| distributed init (rank 53): env:// +| distributed init (rank 50): env:// +| distributed init (rank 48): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 53): env:// +| distributed init (rank 48): env:// +| distributed init (rank 51): env:// +| distributed init (rank 52): env:// +| distributed init (rank 50): env:// +| distributed init (rank 49): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 53): env:// +| distributed init (rank 48): env:// +| distributed init (rank 49): env:// +| distributed init (rank 52): env:// +| distributed init (rank 50): env:// +| distributed init (rank 51): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 52): env:// +| distributed init (rank 50): env:// +| distributed init (rank 53): env:// +| distributed init (rank 49): env:// +| distributed init (rank 51): env:// +| distributed init (rank 48): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 51): env:// +| distributed init (rank 53): env:// +| distributed init (rank 52): env:// +| distributed init (rank 50): env:// +| distributed init (rank 48): env:// +| distributed init (rank 49): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 50): env:// +| distributed init (rank 52): env:// +| distributed init (rank 49): env:// +| distributed init (rank 53): env:// +| distributed init (rank 48): env:// +| distributed init (rank 51): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 48): env:// +| distributed init (rank 49): env:// +| distributed init (rank 52): env:// +| distributed init (rank 50): env:// +| distributed init (rank 53): env:// +| distributed init (rank 51): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 49): env:// +| distributed init (rank 48): env:// +| distributed init (rank 53): env:// +| distributed init (rank 50): env:// +| distributed init (rank 51): env:// +| distributed init (rank 52): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 53): env:// +| distributed init (rank 51): env:// +| distributed init (rank 50): env:// +| distributed init (rank 48): env:// +| distributed init (rank 52): env:// +| distributed init (rank 49): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 53): env:// +| distributed init (rank 52): env:// +| distributed init (rank 49): env:// +| distributed init (rank 48): env:// +| distributed init (rank 50): env:// +| distributed init (rank 51): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 49): env:// +| distributed init (rank 51): env:// +| distributed init (rank 50): env:// +| distributed init (rank 48): env:// +| distributed init (rank 52): env:// +| distributed init (rank 53): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 51): env:// +| distributed init (rank 53): env:// +| distributed init (rank 50): env:// +| distributed init (rank 48): env:// +| distributed init (rank 49): env:// +| distributed init (rank 52): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 53): env:// +| distributed init (rank 52): env:// +| distributed init (rank 48): env:// +| distributed init (rank 51): env:// +| distributed init (rank 49): env:// +| distributed init (rank 50): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 2065295) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python +Traceback (most recent call last): + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in + sys.exit(main()) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + return f(*args, **kwargs) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + run(args) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + elastic_launch( + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +======================================================== +train_cycling_gen.py FAILED +-------------------------------------------------------- +Failures: +[1]: + time : 2023-09-21_12:38:12 + host : sc10 + rank : 49 (local_rank: 1) + exitcode : -6 (pid: 2065296) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2065296 +[2]: + time : 2023-09-21_12:38:12 + host : sc10 + rank : 50 (local_rank: 2) + exitcode : -6 (pid: 2065297) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2065297 +[3]: + time : 2023-09-21_12:38:12 + host : sc10 + rank : 51 (local_rank: 3) + exitcode : -6 (pid: 2065298) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2065298 +[4]: + time : 2023-09-21_12:38:12 + host : sc10 + rank : 52 (local_rank: 4) + exitcode : -6 (pid: 2065299) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2065299 +[5]: + time : 2023-09-21_12:38:12 + host : sc10 + rank : 53 (local_rank: 5) + exitcode : -6 (pid: 2065300) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2065300 +-------------------------------------------------------- +Root Cause (first observed failure): +[0]: + time : 2023-09-21_12:38:12 + host : sc10 + rank : 48 (local_rank: 0) + exitcode : -6 (pid: 2065295) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2065295 +======================================================== diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_9.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_9.txt new file mode 100644 index 00000000..b16f4d1c --- /dev/null +++ b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_9.txt @@ -0,0 +1,1310 @@ +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 54): env:// +| distributed init (rank 56): env:// +| distributed init (rank 57): env:// +| distributed init (rank 55): env:// +| distributed init (rank 59): env:// +| distributed init (rank 58): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 55): env:// +| distributed init (rank 59): env:// +| distributed init (rank 54): env:// +| distributed init (rank 56): env:// +| distributed init (rank 57): env:// +| distributed init (rank 58): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 55): env:// +| distributed init (rank 59): env:// +| distributed init (rank 57): env:// +| distributed init (rank 54): env:// +| distributed init (rank 56): env:// +| distributed init (rank 58): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 56): env:// +| distributed init (rank 58): env:// +| distributed init (rank 59): env:// +| distributed init (rank 55): env:// +| distributed init (rank 57): env:// +| distributed init (rank 54): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 55): env:// +| distributed init (rank 54): env:// +| distributed init (rank 58): env:// +| distributed init (rank 57): env:// +| distributed init (rank 56): env:// +| distributed init (rank 59): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 59): env:// +| distributed init (rank 57): env:// +| distributed init (rank 58): env:// +| distributed init (rank 55): env:// +| distributed init (rank 56): env:// +| distributed init (rank 54): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 54): env:// +| distributed init (rank 59): env:// +| distributed init (rank 58): env:// +| distributed init (rank 57): env:// +| distributed init (rank 55): env:// +| distributed init (rank 56): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 59): env:// +| distributed init (rank 58): env:// +| distributed init (rank 57): env:// +| distributed init (rank 55): env:// +| distributed init (rank 56): env:// +| distributed init (rank 54): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 58): env:// +| distributed init (rank 56): env:// +| distributed init (rank 55): env:// +| distributed init (rank 59): env:// +| distributed init (rank 54): env:// +| distributed init (rank 57): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 57): env:// +| distributed init (rank 56): env:// +| distributed init (rank 58): env:// +| distributed init (rank 54): env:// +| distributed init (rank 55): env:// +| distributed init (rank 59): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 59): env:// +| distributed init (rank 56): env:// +| distributed init (rank 57): env:// +| distributed init (rank 55): env:// +| distributed init (rank 54): env:// +| distributed init (rank 58): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 54): env:// +| distributed init (rank 58): env:// +| distributed init (rank 59): env:// +| distributed init (rank 55): env:// +| distributed init (rank 56): env:// +| distributed init (rank 57): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 58): env:// +| distributed init (rank 56): env:// +| distributed init (rank 55): env:// +| distributed init (rank 59): env:// +| distributed init (rank 54): env:// +| distributed init (rank 57): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 58): env:// +| distributed init (rank 55): env:// +| distributed init (rank 57): env:// +| distributed init (rank 54): env:// +| distributed init (rank 56): env:// +| distributed init (rank 59): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 56): env:// +| distributed init (rank 58): env:// +| distributed init (rank 59): env:// +| distributed init (rank 55): env:// +| distributed init (rank 57): env:// +| distributed init (rank 54): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 56): env:// +| distributed init (rank 57): env:// +| distributed init (rank 59): env:// +| distributed init (rank 58): env:// +| distributed init (rank 55): env:// +| distributed init (rank 54): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 54): env:// +| distributed init (rank 56): env:// +| distributed init (rank 55): env:// +| distributed init (rank 57): env:// +| distributed init (rank 58): env:// +| distributed init (rank 59): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 59): env:// +| distributed init (rank 56): env:// +| distributed init (rank 55): env:// +| distributed init (rank 54): env:// +| distributed init (rank 58): env:// +| distributed init (rank 57): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 55): env:// +| distributed init (rank 56): env:// +| distributed init (rank 59): env:// +| distributed init (rank 57): env:// +| distributed init (rank 54): env:// +| distributed init (rank 58): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 55): env:// +| distributed init (rank 58): env:// +| distributed init (rank 57): env:// +| distributed init (rank 56): env:// +| distributed init (rank 54): env:// +| distributed init (rank 59): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 55): env:// +| distributed init (rank 57): env:// +| distributed init (rank 56): env:// +| distributed init (rank 59): env:// +| distributed init (rank 54): env:// +| distributed init (rank 58): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 56): env:// +| distributed init (rank 59): env:// +| distributed init (rank 58): env:// +| distributed init (rank 57): env:// +| distributed init (rank 55): env:// +| distributed init (rank 54): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 54): env:// +| distributed init (rank 57): env:// +| distributed init (rank 58): env:// +| distributed init (rank 59): env:// +| distributed init (rank 56): env:// +| distributed init (rank 55): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 58): env:// +| distributed init (rank 57): env:// +| distributed init (rank 59): env:// +| distributed init (rank 56): env:// +| distributed init (rank 54): env:// +| distributed init (rank 55): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 59): env:// +| distributed init (rank 58): env:// +| distributed init (rank 56): env:// +| distributed init (rank 54): env:// +| distributed init (rank 57): env:// +| distributed init (rank 55): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 57): env:// +| distributed init (rank 54): env:// +| distributed init (rank 55): env:// +| distributed init (rank 59): env:// +| distributed init (rank 58): env:// +| distributed init (rank 56): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 57): env:// +| distributed init (rank 56): env:// +| distributed init (rank 58): env:// +| distributed init (rank 59): env:// +| distributed init (rank 54): env:// +| distributed init (rank 55): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 58): env:// +| distributed init (rank 55): env:// +| distributed init (rank 56): env:// +| distributed init (rank 57): env:// +| distributed init (rank 59): env:// +| distributed init (rank 54): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 57): env:// +| distributed init (rank 58): env:// +| distributed init (rank 59): env:// +| distributed init (rank 56): env:// +| distributed init (rank 54): env:// +| distributed init (rank 55): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 57): env:// +| distributed init (rank 54): env:// +| distributed init (rank 58): env:// +| distributed init (rank 59): env:// +| distributed init (rank 55): env:// +| distributed init (rank 56): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +WARNING:torch.distributed.run: +***************************************** +Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +***************************************** +| distributed init (rank 54): env:// +| distributed init (rank 58): env:// +| distributed init (rank 56): env:// +| distributed init (rank 59): env:// +| distributed init (rank 57): env:// +| distributed init (rank 55): env:// +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. +grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] +bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +free(): invalid pointer +ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 2917693) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python +Traceback (most recent call last): + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in + sys.exit(main()) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper + return f(*args, **kwargs) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main + run(args) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run + elastic_launch( + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +======================================================== +train_cycling_gen.py FAILED +-------------------------------------------------------- +Failures: +[1]: + time : 2023-09-21_12:38:12 + host : sc16 + rank : 55 (local_rank: 1) + exitcode : -6 (pid: 2917694) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2917694 +[2]: + time : 2023-09-21_12:38:12 + host : sc16 + rank : 56 (local_rank: 2) + exitcode : -6 (pid: 2917695) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2917695 +[3]: + time : 2023-09-21_12:38:12 + host : sc16 + rank : 57 (local_rank: 3) + exitcode : -6 (pid: 2917696) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2917696 +[4]: + time : 2023-09-21_12:38:12 + host : sc16 + rank : 58 (local_rank: 4) + exitcode : -6 (pid: 2917697) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2917697 +[5]: + time : 2023-09-21_12:38:12 + host : sc16 + rank : 59 (local_rank: 5) + exitcode : -6 (pid: 2917698) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2917698 +-------------------------------------------------------- +Root Cause (first observed failure): +[0]: + time : 2023-09-21_12:38:12 + host : sc16 + rank : 54 (local_rank: 0) + exitcode : -6 (pid: 2917693) + error_file: + traceback : Signal 6 (SIGABRT) received by PID 2917693 +======================================================== diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275223.sc14.2163737.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275223.sc14.2163737.0 new file mode 100644 index 0000000000000000000000000000000000000000..040bf46aca2d09c27264c109779502f2bcbc466d GIT binary patch literal 193 zcmeZZfPjCKJmzv5Bs@3$=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(s5s9g$mSMC+!Ab&dj3_a|IZ+#JNPd_(DLY z>!+vY#pmP~7mG3+2)5sUH{4Jas^`@nv(KET5qc!J#JKn&dQvirlcAa%UfSN3a_t2G DzyLwZ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275225.sc14.2163737.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275225.sc14.2163737.1 new file mode 100644 index 0000000000000000000000000000000000000000..e581300283f6e964e9ba4ddca7e4a690a5876dab GIT binary patch literal 193 zcmeZZfPjCKJmzxl2#h!T=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@-$PLK?Q2AlXinIXYp2uxdKdD;#{I!d?6sy z_0v=H;&bwgi$xh4{Osi~?-y2u>Up)t>@(*%gdPblF)n_Ho|MeuWT+;F7TZO6?WzFs Cu0DML literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275228.sc14.2163737.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275228.sc14.2163737.2 new file mode 100644 index 0000000000000000000000000000000000000000..06d9dc5853e0eed150ac06e9521e28edc3445b10 GIT binary patch literal 132 zcmeZZfPjCKJmzxxi*lHMbGqp$#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZF$?udm4TXj;^0(S&d5@Txmto;{9GJii8=Z? R`NhSe9P;1nSQfIX008!UDdqqG literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275232.sc14.2163737.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275232.sc14.2163737.3 new file mode 100644 index 0000000000000000000000000000000000000000..f88073939410d3fe6dda4ee238539f53d9f6a53e GIT binary patch literal 193 zcmeZZfPjCKJmzvHn8;dubGqp$#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z*%^DQO$BPMlXinI=c}m@a|M{S#JNPd_(DLY z>!+vY#pmP~7mG4DgxOaVG%Qzz>Up)t>@#QdG`JoKE-@~Ch@O|u Dd<#C$ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275238.sc14.2163737.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275238.sc14.2163737.5 new file mode 100644 index 0000000000000000000000000000000000000000..e674ec3582a516a63457190a6b1cd71f875f4309 GIT binary patch literal 134 zcmeZZfPjCKJmzw86fLy;=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-62rggq#V@Tn|FkaIKN+km@B}jCCJ6k#Sxa6 Tqo0#sTrB#q`J~-7O>tHL=N~Q! literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275242.sc14.2163737.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275242.sc14.2163737.6 new file mode 100644 index 0000000000000000000000000000000000000000..3d6496d920a7affb58e6216ca869f77f44d47c63 GIT binary patch literal 193 zcmeZZfPjCKJmzw~D%x)K&FQA26mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWFCj@RTZeYPTCE=oN2r}z~&0DYKe1+a`A_zC2;1c8Fhv-SkEKY`MTEJ!dbotg6 E0R7EELI3~& literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275249.sc14.2163737.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275249.sc14.2163737.8 new file mode 100644 index 0000000000000000000000000000000000000000..30b175c21f9fa4f4c88889202876340fd9851691 GIT binary patch literal 134 zcmeZZfPjCKJmzw~^Yyd&=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^42J^Tn=jP%{xLyoE{VW!R87uX$f-ib8&NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZxxiWLssc6FNxQ+9^IHnUTmg11aV}9Vz7UY< z`st~8@j3a$#i9%k{@0g!Ym}-&^}O0+_Lrk&FQA26mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#OhJPY!#@vPTCE=oQVp)U~>gHw8XhYx%fgr zrt7Dt=Edja7Z-~%JosO)wEIw{Dpb#_J!YRdA0zZgaEWp8L-eF%7AHeBEx2gIxAfIY E0DV?LTL1t6 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275259.sc14.2163737.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275259.sc14.2163737.11 new file mode 100644 index 0000000000000000000000000000000000000000..c81b730e48981f8508854fe891c1481d596549c4 GIT binary patch literal 134 zcmeZZfPjCKJmzv%=>N0(=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;=AC{6FI24H}41;aXwuGF;{?DOOT77iz6&C UM?WXOxLEXI^HIC$pBKge09bP{_W%F@ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275264.sc14.2163737.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275264.sc14.2163737.12 new file mode 100644 index 0000000000000000000000000000000000000000..1622a6e86d60bf0c7407ec540347aed899a2be1b GIT binary patch literal 193 zcmeZZfPjCKJmzw~my>q*=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a%urXr3%zsC+!AbPNn$}a|JlH#JNPd_(DLY z>!+vY#pmP~7mG4H_+P)#Y2hzbsGe7Q%szAOMd*>>664~B=t;>ePKIhau*N2!==EFx DIu1cy literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275267.sc14.2163737.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275267.sc14.2163737.13 new file mode 100644 index 0000000000000000000000000000000000000000..6911450c440a67a8cba6bbbbf69c89a87692e4a1 GIT binary patch literal 193 zcmeZZfPjCKJmzv*3f^}3=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!X17qUj=HelXinI=j{%NxdL2T;#{I!d?6sy z_0v=H;&bwgi$xh8{I7SpwK!B2s^`@nv(KE4op3!8Tw+}O5IrfG#mP`j2O4b}kIk6_ E0EinxMgRZ+ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275270.sc14.2163737.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275270.sc14.2163737.14 new file mode 100644 index 0000000000000000000000000000000000000000..5adc0af9062672b2ae64dd3631d531359a58ce85 GIT binary patch literal 134 zcmeZZfPjCKJmzwW2K76BbGqp$#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZdF(GNAO|(~<{cp;&fhyB<_fTA33BmsafBu2 T=;!1Y7mJo$+;1oJBO(9*yF)Fj literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275274.sc14.2163737.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275274.sc14.2163737.15 new file mode 100644 index 0000000000000000000000000000000000000000..802b5ea4c1d67e9a61e2f492f2156d2dda1fbb0f GIT binary patch literal 193 zcmeZZfPjCKJmzxtY+LU1&FQA26mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWLcl@Srw?cPTCE=oLiMvfXx-)))MCu<>Ctg znXaFnnirpwUtBE8@Zf)awypk0Rj8gNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZNf*BLNCj%HlXinI=Yeg_U~>g{w8XhYx%fgr zrt7Dt=Edja7Z-~%JosNfZS9>!s!%^4Bf%xc#ShVwl3AP#)zsi@Q^Apc F7XZKxLBIe2 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275280.sc14.2163737.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275280.sc14.2163737.17 new file mode 100644 index 0000000000000000000000000000000000000000..722e5a56c328830b94bff2acaf0642a0a7605918 GIT binary patch literal 134 zcmeZZfPjCKJmzvX&9rp+=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GU=n`d^xDOH}41;ahi%*fz1_Q)e_|5=i&%U U%+b%uFD@3f`?1ST=RoCR0O8^-ng9R* literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275285.sc14.2163737.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275285.sc14.2163737.18 new file mode 100644 index 0000000000000000000000000000000000000000..240ef112ac7e28c9be85a705f053cbe95047b8e7 GIT binary patch literal 193 zcmeZZfPjCKJmzw0t_*Vh=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-QmE9ct^zgJNxQ+9Q(Z0?Y_0&WmN=Iv7hed- zbp7idw8XhYx%fgr zrt7Dt=Edja7Z-~%D1_SwKZsnd3f1#!kJ)EVMFF@T2`(`%eu$ow%;IFIrUULaAJ+<+ F005j6J;eY3 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275975.sc12.3473404.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275975.sc12.3473404.0 new file mode 100644 index 0000000000000000000000000000000000000000..a70d87afcfdffe4070b2f2b132cb1839d7680142 GIT binary patch literal 134 zcmeZZfPjCKJmzvXZ2Iu=o6}84Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$Oofq$#PJ0Z{86y;=CX85p1pio0cFKKNm+> VVvc@JesQtr>Q!xaJzV>y0RYdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vwe@4 E012c)-~a#s literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275983.sc12.3473404.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275983.sc12.3473404.2 new file mode 100644 index 0000000000000000000000000000000000000000..defc00125ca2bd9f1a7a163c249c142e22e5fec2 GIT binary patch literal 193 zcmeZZfPjCKJmzxdrTzQ-&FQA26mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL~>!FlnT^bC+!Ab&Z0XIa|Hyn#JNPd_(DLY z>!+vY#pmP~7mG4Hu(w~j?dw8SsGe7Q%sz8ILFkd-664~B=t;>ePKIhaaKcva^qo}z D(=kQ- literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275986.sc12.3473404.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275986.sc12.3473404.3 new file mode 100644 index 0000000000000000000000000000000000000000..c347673b66b2a6a8005a2fed3121dcd40a6e84e1 GIT binary patch literal 134 zcmeZZfPjCKJmzv9wcqjOo6}84Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$c4b9m2yyXZ{86y;+(e#Vy*zYmLL~D7e`oP Tj($#lak1#^oH{!;<)0S;njJ7G literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275991.sc12.3473404.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275991.sc12.3473404.4 new file mode 100644 index 0000000000000000000000000000000000000000..6e9f780b521f759ad424bc31690064d16d26dc20 GIT binary patch literal 193 zcmeZZfPjCKJmzv{{J!(`o6}84Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th~xM3-YQUYowOT#Ij`s31)D1%s3p!N%EcD~ zGF?ACH7`CVzqnYG;enO?1~Ip{s!%C>J;)m!-$t+HWYC5peHstE{ FE&w5?MJWIP literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275995.sc12.3473404.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275995.sc12.3473404.5 new file mode 100644 index 0000000000000000000000000000000000000000..b0fc9296c722bea47b4e602f9696725246f36f87 GIT binary patch literal 193 zcmeZZfPjCKJmzvvv^eqYo6}84Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th|ROIZz@o8owOT#IZIYR%oPyQ66X@-;tK(p zuAiQo7oU?~TrA4q;9>v7ePKIh~XtO<`)w~n{ D)0;%D literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275997.sc12.3473404.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275997.sc12.3473404.6 new file mode 100644 index 0000000000000000000000000000000000000000..0852b3bc5e380ca5bbc453cfe8b1f03b4876e4cc GIT binary patch literal 134 zcmeZZfPjCKJmzx#dQt!Vo6}84Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNL1OYNI9svH}41;aefPH0Glhop(V)0&&3gz Un4_PQUtBD@det2}6Ylcq0OL3?l>h($ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276002.sc12.3473404.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276002.sc12.3473404.7 new file mode 100644 index 0000000000000000000000000000000000000000..a101cabba3835239a5fd70feacd87bc2f43557bd GIT binary patch literal 193 zcmeZZfPjCKJmzxR-dXK*QfC|)HC+!AbPFIf=U~>h8wZyqZx%fgr zrt7Dt=Edja7Z-~%D0tgnUCv~t3f1#!kJ)F=3kW?DTw+}O5IrfG#mP`j4w<%lylQ#@ DrOiM? literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276006.sc12.3473404.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276006.sc12.3473404.8 new file mode 100644 index 0000000000000000000000000000000000000000..1f6d6d52033a9eac04c377cd4ac66091829522e7 GIT binary patch literal 193 zcmeZZfPjCKJmzw8==A^m=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;a|J}S#JNPd_(DLY z>!+vY#pmP~7mG3|c-#MUo;X7ls^`@nv(KCd5PBrI#JKn&dQvirlcAap1lro4GWP=j D;L1VO literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276009.sc12.3473404.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276009.sc12.3473404.9 new file mode 100644 index 0000000000000000000000000000000000000000..a2298a98b2fbb0b82770b7c6447eb49b4e468ad9 GIT binary patch literal 134 zcmeZZfPjCKJmzxVQgi(E&FQA26mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#Al|(FFB~WH}41;aUPlkF;{?7OOT77iz6&C TM?WXOxL9;{&IG%c$2|lASEVm& literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276013.sc12.3473404.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276013.sc12.3473404.10 new file mode 100644 index 0000000000000000000000000000000000000000..62cba7a56b270b1e529cb229cc1c0691e19e41da GIT binary patch literal 193 zcmeZZfPjCKJmzw4mrD8l&FQA26mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq&fXspbFGnC+!AbPM-x3a|J}T#JNPd_(DLY z>!+vY#pmP~7mG4D*xP4(K6PCcs^`@nv(KD25PBrI#JKn&dQvirlcAazqHLx3-x>k{ D9$`Ur literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276016.sc12.3473404.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276016.sc12.3473404.11 new file mode 100644 index 0000000000000000000000000000000000000000..e5846474fc8bf8e67ac5642a5fa38a791d0eda6a GIT binary patch literal 193 zcmeZZfPjCKJmzw~op1K%o6}84Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th~$k= literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276019.sc12.3473404.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276019.sc12.3473404.12 new file mode 100644 index 0000000000000000000000000000000000000000..e7016000a765230de8095f3b1925c69de60e1e21 GIT binary patch literal 134 zcmeZZfPjCKJmzwm{CWB3o6}84Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNR+bX1v#j>H}41;ajpn|1vXcJOG}W8pNk_b UF-JcqzqnY`?njbcz{@wH0H<0oEdT%j literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276024.sc12.3473404.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276024.sc12.3473404.13 new file mode 100644 index 0000000000000000000000000000000000000000..7361d3ba11b89a9d159fa6b008333c8c1973908d GIT binary patch literal 193 zcmeZZfPjCKJmzw0XY%~}=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!V+<%Lj`KClXinI=e8pda|Oh;#JNPd_(DLY z>!+vY#pmP~7mG4D*xPFaFQ`_9>Up)t>@%nDQMeunE-@~Ch@OdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vgyZB3K^3UEPTCE=oTYCe<_bt?iF1i^@r8g) z*H2H)i_gg~E*522U|=sYIYCkts^`@nv(KDw5PBrI#JKn&dQvirlcAaz#BApX-G~JM DsG>rk literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276030.sc12.3473404.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276030.sc12.3473404.15 new file mode 100644 index 0000000000000000000000000000000000000000..66e8dae95f2f35527163949b317f5950219c8efc GIT binary patch literal 134 zcmeZZfPjCKJmzxFzP{l9H>aD9QoKn;iJ5tNu4SotC00g3dR#gssd>fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjw5%UwFb#hR1Z{86y;uQB<2sT%MTT76OpNk_b UF-JcqzqnX5wxrzddVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL_TGqk_yyZC+!AbPTA8Ca|I-|#JNPd_(DLY z>!+vY#pmP~7mG4D7}=Y?ow`yLs^`@nv(KCp5PBrI#JKn&dQvirlcAapsM$t@ObiDA D{e?dU literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276038.sc12.3473404.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276038.sc12.3473404.17 new file mode 100644 index 0000000000000000000000000000000000000000..1fba33e94f25bdff4671abe139bc2543500671ad GIT binary patch literal 193 zcmeZZfPjCKJmzwq-!_f$yVFfaDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T2w(o8KPpgjowOT#Ii+oCtg znXaFnnirpwUtBE8aKPC9)B3~4s!%<{9 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276041.sc12.3473404.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276041.sc12.3473404.18 new file mode 100644 index 0000000000000000000000000000000000000000..d8368e46d2dd88b19e9ef6e4b72629b7e23a90f7 GIT binary patch literal 134 zcmeZZfPjCKJmzxln&rdv-RY*I6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vgjY!^Uk+;S%{xLyoIkpJ!R89^XbE!hb8&dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWOm5;W)-NpPTCE=oY%ty!R87`Yl(A-a`AdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#Hj7rE)}S`PTCE=oTgzwZyqZx%fgr zrt7Dt=Edja7Z-~%BuLv&nvoKt3f1#!kJ)F=I|w}zTw+}O5IrfG#mP`j2V!iGGlkj# E000R<=l}o! literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276769.sc13.1777137.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276769.sc13.1777137.4 new file mode 100644 index 0000000000000000000000000000000000000000..73d7945622111aa025f468c9b92ac79f00c85477 GIT binary patch literal 193 zcmeZZfPjCKJmzxVdK9|jyVFfaDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNdM%W3ss=zI%zlfa=N>Pfz1_=(-P+r<>Ctg znXaFnnirpwUtBE8z#wV=%BMbD6{_de9<$G!ClGohxWu^lA$n3Wi<6<68e(jNJ#yy& E02~HDs{jB1 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276772.sc13.1777137.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276772.sc13.1777137.5 new file mode 100644 index 0000000000000000000000000000000000000000..2d085310e12e8335f23a35cff7b7733c0218e00e GIT binary patch literal 134 zcmeZZfPjCKJmzw~f6BJ=yVFfaDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$c;PSv*n=X-n=7Z#95ib4mMYSPfL)CpNk_b UF-JcqzqnX5cT=X_e-D!c0F^*5ng9R* literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276776.sc13.1777137.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276776.sc13.1777137.6 new file mode 100644 index 0000000000000000000000000000000000000000..5ccba276188ea821e69f947be380f4e8b32ddcb4 GIT binary patch literal 193 zcmeZZfPjCKJmzxdtyJ6f-RY*I6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vB(+$sMFncElXinI=T&`mu(<;ATH;)yTznxQ z)AiF+^Wt;zi;G1W6m;!%6!I!mp?Y5JG5gHvZ2;FJ!6nAU57Cp7S)2^jw7}n%>*bd+ E0Qf;Zi2wiq literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276780.sc13.1777137.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276780.sc13.1777137.7 new file mode 100644 index 0000000000000000000000000000000000000000..bc1933dc35a49cf825248efa6ceafee21b28de2e GIT binary patch literal 193 zcmeZZfPjCKJmzwKzazN&yVFfaDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNRPxkZWXAxPTCE=oc5VQU~>f&w8XhYx%fgr zrt7Dt=Edja7Z-~%DCpWJHcF_fLiN1bWA>SIK0=QKmlzj6L{Ca)aWYg>gQe}WST{QW DLeoAl literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276783.sc13.1777137.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276783.sc13.1777137.8 new file mode 100644 index 0000000000000000000000000000000000000000..28c8b764e8b01b12b61c6782f3a48f21b110cbd3 GIT binary patch literal 134 zcmeZZfPjCKJmzvrBp%-V-RY*I6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vB)G_aq8!xRn|FkaIPWh$0ybBGUrUgSpNk_b UF-JcqzqnZRW=f--Y^lHw04vxomjD0& literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276787.sc13.1777137.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276787.sc13.1777137.9 new file mode 100644 index 0000000000000000000000000000000000000000..f0875e7e0af7b8ebcdfb50f0493c615a2df8d99b GIT binary patch literal 193 zcmeZZfPjCKJmzxF`TBIvcc+_fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwk*QlF-BqCGI%zlfa{eiK1~ykfQA?all#4F} zWV(KOYF>O!esQrVLxYvQz1oils!%C>J;)m!-$t+HWY0|T0UB_hz E0O>?RQvd(} literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276791.sc13.1777137.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276791.sc13.1777137.10 new file mode 100644 index 0000000000000000000000000000000000000000..1438796be89b77d7246db1b97f6be4044ba27879 GIT binary patch literal 193 zcmeZZfPjCKJmzw~d318`cc+_fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwk*Y4^Y!#@vPTCE=oT>h&z~%}lX^C@*a`AdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq>t;)L^-IrH}41;asHnIF;_r9OOT77iz6&C UM?WXOxL9;?_aeIuVh{EJ0BI{QMF0Q* literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276798.sc13.1777137.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276798.sc13.1777137.12 new file mode 100644 index 0000000000000000000000000000000000000000..087afcb87f8c1a67027ea140aed26a82a2f44fc6 GIT binary patch literal 193 zcmeZZfPjCKJmzv{&ziaayVFfaDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNSsl_1Qn>cPTCE=oZHxDfz1_A))MCu<>Ctg znXaFnnirpwUtBE8aKO!8!SX?lDpb#_J!YRdli1;UB)G)5_#t{yGK-U;nih!ITJCMS F006RDK#u?b literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276801.sc13.1777137.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276801.sc13.1777137.13 new file mode 100644 index 0000000000000000000000000000000000000000..3831e675cda7e8450f766baa8cf97182f6668b87 GIT binary patch literal 193 zcmeZZfPjCKJmzv9Dkwkj-RY*I6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vr2BhUtqRm!C+!Ab&P%@`<_f52iF1i^@r8g) z*H2H)i_gg~E*52IaIt5t6FH&^)$?kP*=J7mKX5$~Tw+}O5IrfG#mP`j3wUiS4$V;q E0Bcf0F8}}l literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276804.sc13.1777137.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276804.sc13.1777137.14 new file mode 100644 index 0000000000000000000000000000000000000000..3956faaba01ea53b67f1df7f1b56c9f04ed3f556 GIT binary patch literal 134 zcmeZZfPjCKJmzv*s%akl?sU^piZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^3K0lP7Z4B%{xLyoc|X>%oPyS66E6N;s{I3 T(a*^*E*6!qnPI2fd`=Jm)xIrP literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276809.sc13.1777137.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276809.sc13.1777137.15 new file mode 100644 index 0000000000000000000000000000000000000000..a7d1ba887a0d9020078ce93700b3f9889b46b891 GIT binary patch literal 193 zcmeZZfPjCKJmzvvEAu$?-RY*I6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vgp-jWP6cYNlXinI=S>bzu(<-NTH;)yTznxQ z)AiF+^Wt;zi;G1W9@yCb&3Toj3f1#!kJ)EVcTTt-2`(`%eu$ow%;IF2CMnxBAx|y? E04Lo*X8-^I literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276812.sc13.1777137.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276812.sc13.1777137.16 new file mode 100644 index 0000000000000000000000000000000000000000..6a10508a050dcde2a495a71a09c6448331086b4e GIT binary patch literal 193 zcmeZZfPjCKJmzwqIVFDhyVFfaDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$f@6D1u9T;owOT#IS)rmfXx+9(-P+r<>Ctg znXaFnnirpwUtBE8u)xCp^TH43RH1ra?J@hzX%++5Bf%xc#ShVwl3AP#)ufn+a literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276819.sc13.1777137.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276819.sc13.1777137.18 new file mode 100644 index 0000000000000000000000000000000000000000..798956e2ec95d026400ebf79e59eacf3578afe08 GIT binary patch literal 193 zcmeZZfPjCKJmzxV+wttkcc+_fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwksYTW*Q!9xb<%F|<+Rs(4mMXnT}zxxl#4F} zWV(KOYF>O!esQrVgMyL0Z0Lu_s!%B<^TWy literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276822.sc13.1777137.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276822.sc13.1777137.19 new file mode 100644 index 0000000000000000000000000000000000000000..111c084925b95f5f368ce3d213687601d949ea7a GIT binary patch literal 193 zcmeZZfPjCKJmzvvY1(u2yVFfaDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th=l2NV-={mPTCE=oO$;k<_c(NiF1i^@r8g) z*H2H)i_gg~E*51-(6ZnAx+zE%s^`@nv(KED5qc!J#JKn&dQvirlVO@dY>lr3i2wje C)j~1= literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277599.sc12.4183255.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277599.sc12.4183255.0 new file mode 100644 index 0000000000000000000000000000000000000000..d14004f38543e266d1ae2c2693f725e2a122455d GIT binary patch literal 134 zcmeZZfPjCKJmzvb82)Pi;dIkciZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vhmyd4mqf~H}41;ahhC#m@6QxCCJ6k#Sxa6 Tqo0#sTr9e{ThA^?%;6^hWD+lq literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277604.sc12.4183255.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277604.sc12.4183255.1 new file mode 100644 index 0000000000000000000000000000000000000000..0fa8f4a3fb3b7780a75d42124a903e8d9dca7879 GIT binary patch literal 193 zcmeZZfPjCKJmzvvF4FA$;dIkciZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@>KRguL{&$C+!Ab&b${8a|JZD#JNPd_(DLY z>!+vY#pmP~7mG49$k?x{U;j%Ls^`@nv(KD65qc!J#JKn&dQvirlcAapB-)<)`+PG1 D-5x`j literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277608.sc12.4183255.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277608.sc12.4183255.2 new file mode 100644 index 0000000000000000000000000000000000000000..4d1a10823e5a6bb80ec9099293b8111423731837 GIT binary patch literal 193 zcmeZZfPjCKJmzw8Rd96uaJuOz#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZNxG;ipaM15NxQ+9)9ydSTmdaDaV}9Vz7UY< z`st~8@j3a$#i9%jJoXAV7s#qY^}O0+_L=hdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL@wdZKNYCCPTCE=oB@n4z~%~QYl(A-a`AdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#J;@goC?%jC+!AbP8G&WU~>g@w8XhYx%fgr zrt7Dt=Edja7Z-~%ByihH7j&Oeh3a{=$LurbG=v@rE-@~Ch@OdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vBw22=w;a^mn|FkaIQz{z!R88xY6)`jb8&dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWKPPwG8L$~PTCE=oJt{GU~>iZw8XhYx%fgr zrt7Dt=Edja7Z-~%G;rH{2J|SI2SSeomlzj6L{Ca)aWYJkz3qyE9|-^- C4M9=> literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277632.sc12.4183255.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277632.sc12.4183255.9 new file mode 100644 index 0000000000000000000000000000000000000000..c66473406a5b8f8c4e80ae75b00e9a896219b5f3 GIT binary patch literal 134 zcmeZZfPjCKJmzwKOt+cv!|A4@6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq%dvbUpc6`H}41;ajv;&3pQ6kOiPf9pNk_b UF-JcqzqnX5H1w~n$-@)L09Au8v;Y7A literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277637.sc12.4183255.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277637.sc12.4183255.10 new file mode 100644 index 0000000000000000000000000000000000000000..261a1970a3617963d4017014339133c6a7e2916c GIT binary patch literal 193 zcmeZZfPjCKJmzu=q$W@N;dIkciZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GF#(ahzitPC+!Ab&X1ufU~>iZwZyqZx%fgr zrt7Dt=Edja7Z-~%FmT(SykqCD3f1#!kJ)F=A7OAk5?o?j{181UnZ?O4O-{BP37ang E06lv@hX4Qo literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277641.sc12.4183255.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277641.sc12.4183255.11 new file mode 100644 index 0000000000000000000000000000000000000000..d8bf3abfac2f7acbbc9c1f9c7cf90192ec4ecd3d GIT binary patch literal 193 zcmeZZfPjCKJmzwS1^Z6=;dIkciZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(vX}XpaM15NxQ+9bNUQFu(<*TTH;)yTznxQ z)AiF+^Wt;zi;G1W4zSq^T>bV@6{_de9<$G!jx*tUB)G)5_#t{yGK-U;njYBND#m3` F0{}QHKnwr? literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277644.sc12.4183255.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277644.sc12.4183255.12 new file mode 100644 index 0000000000000000000000000000000000000000..65a1fcd0b664c46f9826abfca88a6ef926abc433 GIT binary patch literal 134 zcmeZZfPjCKJmzvvyTLN~hto|*Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$SvLPeR5E9Z{86y;=JR|1~ykfTuYFPpNk_b UF-JcqzqnX**9!r=q}gv+0Y6tR=>Px# literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277648.sc12.4183255.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277648.sc12.4183255.13 new file mode 100644 index 0000000000000000000000000000000000000000..cf809ea3135577598819b470148aa6996cf2c6d4 GIT binary patch literal 193 zcmeZZfPjCKJmzv9jntd+!|A4@6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#H2mYO$BPMlXinIr^#%6u(<+;TH;)yTznxQ z)AiF+^Wt;zi;G1W4oKRUa3t?jh3a{=$Lurb0fZh2E-@~Ch@OCtg znXaFnnirpwUtBE8z#wAZB)p?Y6{_de9<$G!5+!gw5?o?j{181UnZ?O4O-8oI5+k1i E081M_o&W#< literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277655.sc12.4183255.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277655.sc12.4183255.15 new file mode 100644 index 0000000000000000000000000000000000000000..0fae31448eea83aeb67896ee172a879dbb2b0b7c GIT binary patch literal 134 zcmeZZfPjCKJmzwSZoD=1hto|*Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNS`{>S2?J;H}41;ao*i@8*HwCgq9!|KNm+> UVvc@JesQtrt`}T(pEZ-?0dHw9rvLx| literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277659.sc12.4183255.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277659.sc12.4183255.16 new file mode 100644 index 0000000000000000000000000000000000000000..2d94dfba14ec149c2c1acee620cc92c045c98161 GIT binary patch literal 193 zcmeZZfPjCKJmzxBt^POdhto|*Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th~WRmz*tM1OO%T* z1Z28?dTL&LPJVH*D1!sHeWB5sv#L-%ulAUI=FHNC>yh9RdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#IJVlKRKwmH}41;aTdwt<8 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278445.sc20.758883.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278445.sc20.758883.0 new file mode 100644 index 0000000000000000000000000000000000000000..83e43f53e1f5248bcd76ac00713aa0c02a180155 GIT binary patch literal 193 zcmeZZfPjCKJmzv<{#OdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq{ZfalnT^bC+!Ab&Z5K`u(<-JTH;)yTznxQ z)AiF+^Wt;zi;G1W61eOe&*?8xh3a{=$Lurbb%Y)XE-@~Ch@OdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq#(h|LIrBBlXinIXZ8KxU~>h`w8XhYx%fgr zrt7Dt=Edja7Z-~%FmT#`Ie5}p6{_de9<$G!ClGohxWu^lA$n3Wi<6<660~d?8ji0A E09}4UU;qFB literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278450.sc20.758883.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278450.sc20.758883.2 new file mode 100644 index 0000000000000000000000000000000000000000..07ac8c2a98b50d5197f0af43a1edd55bd911187d GIT binary patch literal 134 zcmeZZfPjCKJmzwC{ag|A)9I$86mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq)({vupHFfn|FkaI0M$M1e+@$r6tJ4&&3gz Un4_PQUtBEeCjQj+aOuVc0AWurH~;_u literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278455.sc20.758883.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278455.sc20.758883.3 new file mode 100644 index 0000000000000000000000000000000000000000..1a5c1e2348fc87b80c9d87bf4c6e06a45f326a59 GIT binary patch literal 193 zcmeZZfPjCKJmzv{8C;0{>2%XkiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GWF9j4;84nPTCE=oV98f!R88>Yl(A-a`A2%XkiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^5#hJc@?O+PTCE=oD()|1e+^hp(V~G%EcD~ zGF?ACH7`CVzqnYG;Q_n-`huP(s!%Ppz literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278461.sc20.758883.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278461.sc20.758883.5 new file mode 100644 index 0000000000000000000000000000000000000000..6a67c1475d0386bd052e1f951ed782866532b1a2 GIT binary patch literal 134 zcmeZZfPjCKJmzvL7Nx}hbh_y%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z+5L9iM>(juH}41;asDn$1)D1%ttH6C&&3gz Un4_PQUtBC28v4{$SXKE702&r9i~s-t literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278466.sc20.758883.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278466.sc20.758883.6 new file mode 100644 index 0000000000000000000000000000000000000000..6882d44bbb102ccee6455c0acea111aab5ba9738 GIT binary patch literal 193 zcmeZZfPjCKJmzw$i?t^Fbh_y%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zxv`yhsS4CwC+!Ab&J8VXU~>g5wZyqZx%fgr zrt7Dt=Edja7Z-~%FmT$x_dWSY6{_de9<$G!hOKZt5?o?j{181UnZ?OaO$~asYZLT+ E0WXU`h5!Hn literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278469.sc20.758883.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278469.sc20.758883.7 new file mode 100644 index 0000000000000000000000000000000000000000..1e4989ece3547ef4334cae9523af0788c6d8b341 GIT binary patch literal 193 zcmeZZfPjCKJmzvXZB0r1>2%XkiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-V!n8(lnT^bC+!AbPJi|^u(<+OTH;)yTznxQ z)AiF+^Wt;zi;G1W64>o`{oEa)3f1#!kJ)F=R|q{4Tw+}O5IrfG#mP`j3c9w@!hf{@ DIW9iI literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278472.sc20.758883.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278472.sc20.758883.8 new file mode 100644 index 0000000000000000000000000000000000000000..c1637fb0d5358ca29c278e70fae065fbaa165b20 GIT binary patch literal 134 zcmeZZfPjCKJmzwiX^ABLbh_y%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZseQ9{gB;Y{n|FkaI1MjC%oUK)66E6N;s{I3 T(a*^*E*9<4dt$p#w^kYe&%Z5N literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278476.sc20.758883.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278476.sc20.758883.9 new file mode 100644 index 0000000000000000000000000000000000000000..bd8c73864d637b45fc5cb715188d33c93262c302 GIT binary patch literal 193 zcmeZZfPjCKJmzvvone{$)9I$86mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#5SW+PX%hOlXinI=PhY#u(<-(TH;)yTznxQ z)AiF+^Wt;zi;G1W5;*MVF8r9T3f1#!kJ)EV9~rnF2`(`%eu$ow%;IFIrUyE<{PSBl E0RE#sJOBUy literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278480.sc20.758883.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278480.sc20.758883.10 new file mode 100644 index 0000000000000000000000000000000000000000..44868675702fe4f26fc7d519d67955c8d9bcb88f GIT binary patch literal 193 zcmeZZfPjCKJmzw0Bnzhebh_y%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zx#0b)S_NvZlXinIr~4iuu(<*@TH;)yTznxQ z)AiF+^Wt;zi;G1W8rbanY_qPbLiN1bWA>SIBSMb^mlzj6L{Ca)aWYg>f`)CJ6oVlE D_Mkoc literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278483.sc20.758883.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278483.sc20.758883.11 new file mode 100644 index 0000000000000000000000000000000000000000..a1a5cc8f3e65000470444b9e0f99c34f312ca91a GIT binary patch literal 134 zcmeZZfPjCKJmzxVO+AwG)9I$86mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vBelCu% U#2o#c{NiHK#oZTd<9oxI0hm}YE&u=k literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278487.sc20.758883.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278487.sc20.758883.12 new file mode 100644 index 0000000000000000000000000000000000000000..6e6eb7f0566981897a2bb99e99b9dc701a727fee GIT binary patch literal 193 zcmeZZfPjCKJmzxd>%U3;>2%XkiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!raK|rUEtBNxQ+9GvzSETmf4xaV}9Vz7UY< z`st~8@j3a$#i9%h9QM39Q)N}5dS2}@`^@2%XkiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;?)}*rvf$CNxQ+9Q=jV`*jxcSEpaYUF1`?u z>H6uZdGR^<#l@lw3~ctdlzUdFLiN1bWA>Tz2||wqmlzj6L{Ca)aWYg>gPN_`Keapn D3UfZ& literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278493.sc20.758883.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278493.sc20.758883.14 new file mode 100644 index 0000000000000000000000000000000000000000..acc36f9d5b90b15a9e07f92d9ef796497786b253 GIT binary patch literal 134 zcmeZZfPjCKJmzwC{i{v?>2%XkiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-64{uTCiJv;?{Mxj4cS UbM$lai;G1k`|h)~x8XDe08IWZlmGw# literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278498.sc20.758883.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278498.sc20.758883.15 new file mode 100644 index 0000000000000000000000000000000000000000..bf638679d3298c41c56137d0e5d9e318a2b78e86 GIT binary patch literal 193 zcmeZZfPjCKJmzwK30j=-)9I$86mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWb3!J8Y)n8owOT#IaSU;%oVWL66X@-;tK(p zuAiQo7oU?~TrA4)fYrWBp*2nws^`@nv(KFK5qc!J#JKn&dQvirlVO_lY=u@fu>k-f Ce?eCO literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278501.sc20.758883.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278501.sc20.758883.16 new file mode 100644 index 0000000000000000000000000000000000000000..7cf8728e09a3bcba1bcba120a6ca1999fda9a85f GIT binary patch literal 193 zcmeZZfPjCKJmzvvKAWHU)9I$86mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq$(#@N(E}JlXinIXQ@>I*jxbzEpaYUF1`?u z>H6uZdGR^<#l@lw4UG1o3njLyLiN1bWA>SIKSGZLmlzj6L{Ca)aWYiXfneMDlWnvB DP{BV! literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278504.sc20.758883.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278504.sc20.758883.17 new file mode 100644 index 0000000000000000000000000000000000000000..156717289aeab9502938524618d25414e8689cbe GIT binary patch literal 134 zcmeZZfPjCKJmzw$Z2%XkiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-lJ_*rLk?>0%{xLyoSP(M!R89cYYB4kb8&2%XkiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a$V%1tP0dzC+!Ab&U6nuu(<+`TH;)yTznxQ z)AiF+^Wt;zi;G2pO6~n(wyCH>^}O0+_L=h_LXQNO7#BZ8PfBKSGE7sat=V+Z835d# BJ$wKF literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278512.sc20.758883.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278512.sc20.758883.19 new file mode 100644 index 0000000000000000000000000000000000000000..20f7790b1d94eafc7cc5e48e76c5cd046a93fc72 GIT binary patch literal 193 zcmeZZfPjCKJmzxBO=ivc>2%XkiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-lH%JwM+IuGlXinIr`tO=u(<+GTH;)yTznxQ z)AiF+^Wt;zi;G1W7W}gFSatM=Dpb#_J!YRd?;-R^aEWp8L-eF%7AHeB9hhXxalCsU E07*STdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v!+vY#pmP~7mG3|{I$F7Tx_HY)$?kP*=NoR2t5*9VqE+XJt>*R$xuxVlWjYHJdy$c D!)8JI literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279213.sc10.1505340.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279213.sc10.1505340.2 new file mode 100644 index 0000000000000000000000000000000000000000..464d18c50bfc5a3eb474dc9bad4d73dde2d120a6 GIT binary patch literal 193 zcmeZZfPjCKJmzv{{_>Ol<#f|giZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;&<-eLKUdFPTCE=oSS3)!R89MXo+))a`A|X<#f|giZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@^4>~stVLxC+!Ab&Z2n`a|K+r#JNPd_(DLY z>!+vY#pmP~7mG3^Fxp!@syU{a=Pg##hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZF%vBLtO7OHNxQ+9bIEszxdLul;#{I!d?6sy z_0v=H;&bwgi$xg{80>2Tv<+3EdS2}@`^+iw1FlDcON@&jq9-M@I2o$xL5;0|u(SgJ Df8RWt literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279227.sc10.1505340.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279227.sc10.1505340.6 new file mode 100644 index 0000000000000000000000000000000000000000..2aab0a06a60bd48fe16171f1035d7912bc4b4c6b GIT binary patch literal 134 zcmeZZfPjCKJmzw`Xr7S!<#f|giZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@{N79wj9*ln|FkaIMdr9<_aij33BmsafBu2 T=;!1Y7mNCt&9yz{@nI(b%Dye& literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279231.sc10.1505340.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279231.sc10.1505340.7 new file mode 100644 index 0000000000000000000000000000000000000000..2a9a3f3af6a9302379846a9ffc019f3eda1419d5 GIT binary patch literal 193 zcmeZZfPjCKJmzvT8NZPK<#f|giZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a@uA8G!>}1PTCE=oHOP?%oT9g66X@-;tK(p zuAiQo7oU?~TrA4az-WK-Aj3mdsGe7Q%sz9P%!TWb;1c8Fhv-SkEKY`MT99L_QGDYN E0CUAbRR910 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279234.sc10.1505340.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279234.sc10.1505340.8 new file mode 100644 index 0000000000000000000000000000000000000000..96a42acf53c14bf9225b364a8d0459a4123842db GIT binary patch literal 193 zcmeZZfPjCKJmzvPJGNWlm(xv0Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$clHBdMZ$JowOT#IWODn0h=q}p(V~G%EcD~ zGF?ACH7`CVzqnYGfq~K9>M!GERj8gdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vBv4beKn`l|%{xLyoM!&{U~>hOwFJ5Nxj4cS UbM$lai;G2fy_jSB=sDvN00p2fI{*Lx literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279242.sc10.1505340.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279242.sc10.1505340.10 new file mode 100644 index 0000000000000000000000000000000000000000..ef846efab0515a381de0fd159f20bf5c46bfdd6b GIT binary patch literal 193 zcmeZZfPjCKJmzw)tm;gLwZyqZx%fgr zrt7Dt=Edja7Z-~%G%(uFn{|Oj6{_de9<$G!hY)%sxWu^lA$n3Wi<6<66vAzJ>y8Tn E07`W}ng9R* literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279245.sc10.1505340.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279245.sc10.1505340.11 new file mode 100644 index 0000000000000000000000000000000000000000..e6db4c024647226cb9917862b1203de2630b8c0a GIT binary patch literal 193 zcmeZZfPjCKJmzxpzQ|Jk<#f|giZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GPlO=lnT^bC+!Ab&h6gWU~>h$w8XhYx%fgr zrt7Dt=Edja7Z-~%9AL2jzGsEMDpb#_J!YRdd41q|B)G)5_#t{yGK-U8ngVT`m$L8! E07D}`I{*Lx literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279248.sc10.1505340.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279248.sc10.1505340.12 new file mode 100644 index 0000000000000000000000000000000000000000..996b06014b2ecf9766ca11105c46ee5eb6a0f2ef GIT binary patch literal 134 zcmeZZfPjCKJmzxVoh7OA%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWZ}FG`EpQmZ{86y;+!EV1vXbeMN5#2pNk_b UF-JcqzqnZRl-gw5?%a9&0Ql!EH2?qr literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279252.sc10.1505340.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279252.sc10.1505340.13 new file mode 100644 index 0000000000000000000000000000000000000000..89cc23b986e32607049df83f6098d97dcea0a693 GIT binary patch literal 193 zcmeZZfPjCKJmzvfY%*8<<#f|giZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(v$zjSOsdXlXinI=coS=a|OJ$#JNPd_(DLY z>!+vY#pmP~7mG4HV6wM7|EEqBs^`@nv(KC*3>ILw0QE?4iE;5m^rU1KCqp$kgxNYA HEiwZDs7pW# literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279255.sc10.1505340.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279255.sc10.1505340.14 new file mode 100644 index 0000000000000000000000000000000000000000..fc539ed40f0cc226e22900e4253d089ecef34e7b GIT binary patch literal 193 zcmeZZfPjCKJmzvPb@-?H%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v*R59-s^`@nv(KCf``~&cxWu^lA$n3Wi<4oR!fcm3x*rPw D0qsG> literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279258.sc10.1505340.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279258.sc10.1505340.15 new file mode 100644 index 0000000000000000000000000000000000000000..1f294ce7b492bc6584a062897c82079b586bfa45 GIT binary patch literal 134 zcmeZZfPjCKJmzv9?%J&O%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vgtz!prySJWn|FkaIQ_S70h=qJswK$9&&3gz Un4_PQUtBEO!rx~5KWgqK05#h#%>V!Z literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279263.sc10.1505340.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279263.sc10.1505340.16 new file mode 100644 index 0000000000000000000000000000000000000000..6ec550057696ab53f30b5f4ce3e9165ef9b463c9 GIT binary patch literal 193 zcmeZZfPjCKJmzw?Y`Cue%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#4cM`UIl8dlXinI=Uv?!U~>h0wZyqZx%fgr zrt7Dt=Edja7Z-~%G%(nE#{W613f1#!kJ)EVPd&IE2`(`%eu$ow%;IFIrU#+6n?C!; F0|01MK#%|c literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279266.sc10.1505340.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279266.sc10.1505340.17 new file mode 100644 index 0000000000000000000000000000000000000000..27813e4f5e00511b800f2780d5a750208ca1881f GIT binary patch literal 193 zcmeZZfPjCKJmzxF&R?nV%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#4L;PlnT^bC+!Ab&N}iF1i^@r8g) z*H2H)i_gg~E*52I_-(g5^q!I`RL`qDW}i8^pTqS?aEWp8L-eF%7AHeBCAizFJ&{%f E0G+WwG5`Po literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279269.sc10.1505340.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279269.sc10.1505340.18 new file mode 100644 index 0000000000000000000000000000000000000000..d6baa2d1c3006a30c40ee10b815098e1166f7c0e GIT binary patch literal 134 zcmeZZfPjCKJmzvPIuWP&%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL~*~oxE$2nn|FkaI9YAu!R896X$f-ib8&dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWR1@1(<)GNowOT#Iirjkz~&10Yl(A-a`AR literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279943.sc18.3937612.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279943.sc18.3937612.0 new file mode 100644 index 0000000000000000000000000000000000000000..f27fa1d7e91e98b07943d19354419b5cdcb43f06 GIT binary patch literal 193 zcmeZZfPjCKJmzw4u6c0rm(xv0Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNW;xj-&COHI%zlfa-RJKF;^f!OPouTi!TIZ zx_)|UUVKh|aj_`F0S5btdU}Ucp?Y5JG5gG^`x~xDf=i5xAEGBEvp5;5$-%?+VqlXw E09^D$lK=n! literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279946.sc18.3937612.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279946.sc18.3937612.1 new file mode 100644 index 0000000000000000000000000000000000000000..b9d08a432a83b7c16656627abd3b17dd9e0ff986 GIT binary patch literal 134 zcmeZZfPjCKJmzvP-P(WYm(xv0Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$kuOOf^txEZ{86y;=E=&0c@^-x|SdpKNm+> UVvc@JesQsAk6w!{#~M3L0F1vblmGw# literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279950.sc18.3937612.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279950.sc18.3937612.2 new file mode 100644 index 0000000000000000000000000000000000000000..635b79adf58c6850d243a6eabbe08315095550ce GIT binary patch literal 193 zcmeZZfPjCKJmzxBr>wdB%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWVXtRyDCs~owOT#IRl?U%oPaK66X@-;tK(p zuAiQo7oU?~TrA3v@ZU~z)1*38sGe7Q%sz8ILFkd-664~B=t;>ePKIiF;9+Z6v*0fP DXvsu( literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279954.sc18.3937612.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279954.sc18.3937612.3 new file mode 100644 index 0000000000000000000000000000000000000000..c42f79bb335a952571f702403179b3b0c8d1bbad GIT binary patch literal 193 zcmeZZfPjCKJmzvTZ*RQv%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#P2{$lnT^bC+!Ab&W~D6U~>h6w8XhYx%fgr zrt7Dt=Edja7Z-~%Jost1y*=)=Dpb#_J!YRdJ+$F^B)G)5_#t{yGK-U;niAY?+2)9= F001=9L6ZOg literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279956.sc18.3937612.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279956.sc18.3937612.4 new file mode 100644 index 0000000000000000000000000000000000000000..febab17fa85bc9cdcd161dbe99095b8efe31b419 GIT binary patch literal 134 zcmeZZfPjCKJmzvdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWSRDbZ7NW6owOT#Ii(DP!R87CYl(A-a`AFFt E037Q=_5c6? literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279964.sc18.3937612.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279964.sc18.3937612.6 new file mode 100644 index 0000000000000000000000000000000000000000..bb72332bba90005c21a220c026a430c8e8996642 GIT binary patch literal 193 zcmeZZfPjCKJmzwie$>DI%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq}n$H6uZdGR^<#l@lw4ZrLXW4TLJp?Y5JG5gH<2BAlSON@&jq9-M@I2o!*!O_;$`&=^s D$SpvI literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279967.sc18.3937612.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279967.sc18.3937612.7 new file mode 100644 index 0000000000000000000000000000000000000000..2b4ebe85da39c70e25558b7cd0d2d65bb5e925eb GIT binary patch literal 134 zcmeZZfPjCKJmzwqocrMVFQ=Q1QoKn;iJ5tNu4SotC00g3dR#gssd>fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwk#J4pb8=8~Z{86y;=C^K5Nxi1rj{TVKNm+> UVvc@JesQs=-H#GmA)$Sr0jP2>%>V!Z literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279971.sc18.3937612.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279971.sc18.3937612.8 new file mode 100644 index 0000000000000000000000000000000000000000..649d05ae87b99c6434508612172b18e99504a5d0 GIT binary patch literal 193 zcmeZZfPjCKJmzw~a{qVZm(xv0Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$P9;P>s6rUI%zlfawZ;zm@5#fCC(+v#TNoH zT|YfFFFq%~xLA~70h9gng(4SKp?Y5JG5gGU9HB>oON@&jq9-M@I2oqN(Du&-Payz6 Cl|@zn literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279975.sc18.3937612.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279975.sc18.3937612.9 new file mode 100644 index 0000000000000000000000000000000000000000..f6ffcff25c15635856c490f9aa34a47f7c4b4bc9 GIT binary patch literal 193 zcmeZZfPjCKJmzwS9DI25m(xv0Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$ai7>_bO0xowOT#IoDb}0-GxkrX|iL%EcD~ zGF?ACH7`CVzqnYG!GY1f=i_H7Rj8gz6qjFGlZ{86y;ymX#8EmeAmX;tFKNm+> VVvc@JesQtr?3{JBO5wLZ0{~PIFPZ=V literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279982.sc18.3937612.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279982.sc18.3937612.11 new file mode 100644 index 0000000000000000000000000000000000000000..a348476937cd00cd789edda0e56a9b81bc1da2a9 GIT binary patch literal 193 zcmeZZfPjCKJmzw0uitR{m(xv0Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th?LFCQWdDVPTCE=oLAL1g3T2O*AnLv<>Ctg znXaFnnirpwUtBE8knq<|Tc`DbDpb#_J!YRdjWytUB)G)5_#t{yGK-U;niR}!1+E{P F4gei7LBIe2 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279985.sc18.3937612.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279985.sc18.3937612.12 new file mode 100644 index 0000000000000000000000000000000000000000..16840a34db1a4deba716ad36c4dbebfc3f20b821 GIT binary patch literal 193 zcmeZZfPjCKJmzw)U0r|Ym(xv0Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Ti1<==6&0wtPTCE=oNweBz~%}>Xo+))a`Aid{P~u3f1#!kJ)F=GI_Wj2`(`%eu$ow%;IFICI>xRf3}^E E0m@B4G5`Po literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279988.sc18.3937612.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279988.sc18.3937612.13 new file mode 100644 index 0000000000000000000000000000000000000000..31b807474043a204203193c7aa59e1bc6110642b GIT binary patch literal 134 zcmeZZfPjCKJmzxFZPvg0%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vMDF@@RynA-H}41;ai01MF;_raOOT77iz6&C TM?WXOxLEYZgT=OHmDjTYO7bud literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279993.sc18.3937612.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279993.sc18.3937612.14 new file mode 100644 index 0000000000000000000000000000000000000000..163a115f77b2c7eac447bc1f3afa1b8e0606a7fc GIT binary patch literal 193 zcmeZZfPjCKJmzv%Uh%*8%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWPM<2mI~BdC+!AbPVSxnu(<+}TH;)yTznxQ z)AiF+^Wt;zi;G1W99Zoav}bgxLiN1bWA>S|6ro3gON@&jq9-M@I2o$xfV%DM^Y@wn D+UG&k literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279996.sc18.3937612.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279996.sc18.3937612.15 new file mode 100644 index 0000000000000000000000000000000000000000..9067dee7fd4b4509bca683bcaeb9115c203a8d7d GIT binary patch literal 193 zcmeZZfPjCKJmzxdNyy#*<#f|giZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-B5W=&MFncElXinIr`;5YxdKsI;#{I!d?6sy z_0v=H;&bwgi$xh8Fxk&*R$xuxSX0}ekH8KGB C5if%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWX1icC2~-6Z{86y;#_y_9N1g|9W6mFelCu% U#2o#c{NiHK7XE3r?^M!70hmTF?EnA( literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280003.sc18.3937612.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280003.sc18.3937612.17 new file mode 100644 index 0000000000000000000000000000000000000000..4f1b0f91353191a20ad97bf6831857b80b6259d4 GIT binary patch literal 193 zcmeZZfPjCKJmzwyaz1dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vaa|NQc#JNPd_(DLY z>!+vY#pmP~7mG49FxubPq;X6Ys^`@nv(KCt5PBrI#JKn&dQvirlcAazENyw&Id%X5 DOR7PN literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280006.sc18.3937612.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280006.sc18.3937612.18 new file mode 100644 index 0000000000000000000000000000000000000000..abac888facdd17ec5b94da3e7390b7d7f562c8a2 GIT binary patch literal 193 zcmeZZfPjCKJmzw)irxS4m(xv0Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Ti2Wsh0~M&bPTCE=oPoC?<_g4UiF1i^@r8g) z*H2H)i_gg~E*52Y@Xzj_?@V4*sGe7Q%sz8&L+Fv<664~B=t;>ePKIhaU}k%yn8O(W Dy0JtF literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280009.sc18.3937612.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280009.sc18.3937612.19 new file mode 100644 index 0000000000000000000000000000000000000000..f347621f0bb33178089389dbdd9f93d89f022a37 GIT binary patch literal 134 zcmeZZfPjCKJmzwKwaIz(%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vBwoxL1#oQ! E0HeA<#sB~S literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280682.sc10.1562555.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280682.sc10.1562555.1 new file mode 100644 index 0000000000000000000000000000000000000000..6c62314224cb662fcdc2c48e6bd6c419e03e9ab4 GIT binary patch literal 193 zcmeZZfPjCKJmzwC`p@h7?R3*oiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-qRsfbTm@>blXinIr{`6OxdL%o;#{I!d?6sy z_0v=H;&bwgi$xh6nCt@-!_rludS2}@`^?R3*oiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;@54^qyjb9NxQ+9Q$4X1Y_34OmN=Iv7hed- zbp7NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIo0PdPX%hOlXinIr_-81u(<*WTH;)yTznxQ z)AiF+^Wt;zi;G1W8vfZ`T`F}$6{_de9<$G!D-e1lxWu^lA$n3Wi<6<67MR%X(&UN< E0GH80DgXcg literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280696.sc10.1562555.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280696.sc10.1562555.5 new file mode 100644 index 0000000000000000000000000000000000000000..c578d9c94694ff0e73ea806dabe1e0921a4ea4e7 GIT binary patch literal 134 zcmeZZfPjCKJmzxxzhUkB?R3*oiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vfZ|3f*jP`n|FkaI4?P|gUuDt*AnF7=i&%U U%+b%uFD@3n=-O_Za$|lO01ON+egFUf literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280700.sc10.1562555.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280700.sc10.1562555.6 new file mode 100644 index 0000000000000000000000000000000000000000..075312964c11ef5b62beb903da344eafbe7b3c92 GIT binary patch literal 193 zcmeZZfPjCKJmzxlI&IMZ+v%pG6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vB!w}_M+IuGlXinIC!@F#*j#}`EpaYUF1`?u z>H6uZdGR^<#l@lw5B}I4h&||}3f1#!kJ)F=7K9!NE-@~Ch@OdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vhMw8XhYx%fgr zrt7Dt=Edja7Z-~%EckEdXnV(86{_de9<$G!=MZ`%xWu^lA$n3Wi<6<64k+4Ap2>3# E09N-w2mk;8 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280707.sc10.1562555.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280707.sc10.1562555.8 new file mode 100644 index 0000000000000000000000000000000000000000..0972361c85788955b98d4ad7dfc502687b263a5a GIT binary patch literal 134 zcmeZZfPjCKJmzxlzkOiBZ>O7%QoKn;iJ5tNu4SotC00g3dR#gssd>fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwk*T*W&dEW|y?IB-h;!bTgJ5$7473Ef__;X3 V5_9x(@{5Z_{mi;;Yi@6i1OUm7Fwp=2 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280711.sc10.1562555.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280711.sc10.1562555.9 new file mode 100644 index 0000000000000000000000000000000000000000..178a49c5c22efb0cb90213d0b30229d5d84c2ca1 GIT binary patch literal 193 zcmeZZfPjCKJmzu=ta&`~x6@5WDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$S)D|L=~vHPTCE=oJ)Ctg znXaFnnirpwUtBE8aDc&H{7B{nRj8gdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vB-{J3yb9D@C+!Ab&Zboma|Keg#JNPd_(DLY z>!+vY#pmP~7mG3^{I!!&dBmd%)$?kP*=Npo2t5*9VqE+XJt>*R$xuxP6l~M^KhFgK D UVvc@JesQs=cx<0-BxlJ?0H>WVga7~l literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280722.sc10.1562555.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280722.sc10.1562555.12 new file mode 100644 index 0000000000000000000000000000000000000000..8911109a68d6d0b0000e17e3b01b5e9fa995e7a5 GIT binary patch literal 193 zcmeZZfPjCKJmzxF`O-1vx6@5WDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNY@#Iy(&<1owOT#IVJPDz~&01YKe1+a`A literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280725.sc10.1562555.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280725.sc10.1562555.13 new file mode 100644 index 0000000000000000000000000000000000000000..14a5eeb33035bc95afc2dd2c11f18c4d8723fde1 GIT binary patch literal 193 zcmeZZfPjCKJmzxFTaz>Ox6@5WDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNYuXBd@4|LowOT#IrEtFz~&01X^C@*a`AdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vgptiTQ3YzQlXinI=lz8)U~>i1wZyqZx%fgr zrt7Dt=Edja7Z-~%DEziF7q#F~h3a{=$Lup_;Uc&m2`(`%eu$ow%;IFIrURU|5ubEk F0{{%#KVAR; literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280736.sc10.1562555.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280736.sc10.1562555.16 new file mode 100644 index 0000000000000000000000000000000000000000..3aabc5519307ed8eb6508d74658462ed6e396052 GIT binary patch literal 193 zcmeZZfPjCKJmzx#f2=s;x6@5WDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNKG%Rp$gPoC+!Ab&S|wuU~>gBw8XhYx%fgr zrt7Dt=Edja7Z-~%9Qb2*JH0ql6{_de9<$G!PZ4?~xWu^lA$n3Wi<6<67I51>x!TO7%QoKn;iJ5tNu4SotC00g3dR#gssd>fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwk*Ujq{p6tL-n=7Z#M#b&8*HwCv6dhgKNm+> UVvc@JesQtrWZw$gCEs(I0f@#gHvj+t literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280743.sc10.1562555.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280743.sc10.1562555.18 new file mode 100644 index 0000000000000000000000000000000000000000..6ff89d311e6faa523dee3037bd980d343e8e5105 GIT binary patch literal 193 zcmeZZfPjCKJmzvP=KM4Bx6@5WDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th!+vY#pmP~7mG3^{I)ykzv#ItRL`qDW}i8iAoNIZiE;5m^rU1KC&M%e*_LejZw>(P CZbF*? literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281501.sc13.2274570.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281501.sc13.2274570.0 new file mode 100644 index 0000000000000000000000000000000000000000..b0aaad48d3470a7f38b11313ca40b6bd7f422ce2 GIT binary patch literal 193 zcmeZZfPjCKJmzw8wiJi|ak}X!#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZNxPWwPz7qPlXinI=b3LMU~>hsw8XhYx%fgr zrt7Dt=Edja7Z-~%9Qa}PoVjbhDpb#_J!YRdv%kajNN|a9@k8{aWELkwH9g?AeN*^M F2mk?6Lreew literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281504.sc13.2274570.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281504.sc13.2274570.1 new file mode 100644 index 0000000000000000000000000000000000000000..27047580ef6ba166e36b1cb4e1b638ae20460c58 GIT binary patch literal 134 zcmeZZfPjCKJmzv*6stu1ak}X!#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZNjv}ji5%42n|FkaIOQKe%oQ-v66E6N;s{I3 T(a*^*E*7o%P;T40|B(*>?WZp% literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281509.sc13.2274570.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281509.sc13.2274570.2 new file mode 100644 index 0000000000000000000000000000000000000000..85ab7c93d8b94d8f980ab310233245c71e10bc1c GIT binary patch literal 193 zcmeZZfPjCKJmzxRT=tFp<8;$eiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-k~V3Qr3%zsC+!Ab&PBm~U~>hswZyqZx%fgr zrt7Dt=Edja7Z-~%Ecj~IzrJOGDpb#_J!YRd*+SrYB)G)5_#t{yGK-U;ni?2wgY>+U E0CQA7YybcN literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281512.sc13.2274570.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281512.sc13.2274570.3 new file mode 100644 index 0000000000000000000000000000000000000000..618b37813050d77c685940af4aaaf2bcaa78dc39 GIT binary patch literal 193 zcmeZZfPjCKJmzxt>Z(Wmak}X!#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Znd$vpTm@>blXinIr}I*XxdJ&_;#{I!d?6sy z_0v=H;&bwgi$xh4e%smg3G7mZ>Up)t>@(+DgdPblF)n_Ho|MeuWT>VG_iRq+_dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWb(vaTq;m=owOT#IhPx7g3T4k)e`3t<>Ctg znXaFnnirpwUtBE8knqVa(5%K<6{_de9<$G!f`)KC5?o?j{181UnZ?O4OdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWRiMvpbFGnC+!AbPM@fcU~>iXw8XhYx%fgr zrt7Dt=Edja7Z-~%9QbB;F?{iTRj8g6en|FkaIPdRW2sT&1OiPf9pNk_b UF-JcqzqnZRVRO4J)1@d~02!Yyy#N3J literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281530.sc13.2274570.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281530.sc13.2274570.8 new file mode 100644 index 0000000000000000000000000000000000000000..6957b181e38adfbb02a18a8020a876d01d649404 GIT binary patch literal 193 zcmeZZfPjCKJmzvvoVh>lkJC*@Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNW>|_D=JWPowOT#Irn`(05(@3UrU@zl#4F} zWV(KOYF>O!esQrV!-G$D&o>#JQ-$hzwa4r;r`Zp<9tkcnE`Erfl+5B}sHOw~+ZTE3 GZvp_rD@6bR literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281534.sc13.2274570.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281534.sc13.2274570.9 new file mode 100644 index 0000000000000000000000000000000000000000..ed115ef93290c0a1ebd01fa6eceb742f9ef8256b GIT binary patch literal 193 zcmeZZfPjCKJmzvbHOz_s<8;$eiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-qITVNmkQKeC+!Ab&baS$!R87SXo+))a`Ae0!p;Dpb#_J!YRd&miCr(} literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281537.sc13.2274570.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281537.sc13.2274570.10 new file mode 100644 index 0000000000000000000000000000000000000000..eca17a5e6054fb954b867e360fc3a3dc8928f116 GIT binary patch literal 134 zcmeZZfPjCKJmzu=&GSq6<8;$eiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-VwoYPDF-$8<{cp;&J!;FU~>h`wFJ5Nxj4cS TbM$lai;G1C>>6zAt@|7SoM0<+ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281541.sc13.2274570.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281541.sc13.2274570.11 new file mode 100644 index 0000000000000000000000000000000000000000..30038c671a70b78a275f946ae27d4191c2223e96 GIT binary patch literal 193 zcmeZZfPjCKJmzvXT*^=U<8;$eiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@>$(FS_NvZlXinIC!1Lz*j#}^EpaYUF1`?u z>H6uZdGR^<#l@lw55C&jpV6JG3f1#!kJ)F=H3&TtTw+}O5IrfG#mP`j2N-ShUIz68 E0GK5}^#A|> literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281545.sc13.2274570.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281545.sc13.2274570.12 new file mode 100644 index 0000000000000000000000000000000000000000..75c05f34ce752e7ab8e9cc6ddad86f5e67b29ba4 GIT binary patch literal 193 zcmeZZfPjCKJmzu+Cj}(^ak}X!#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zarnr4Pz7qPlXinIr`bb@xdKI6;#{I!d?6sy z_0v=H;&bwgi$xh0e6_1Q-?>Q@s^`@nv(KCl5qc!J#JKn&dQvirlVO@Z*|a|CJPQD_ CqeII8 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281548.sc13.2274570.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281548.sc13.2274570.13 new file mode 100644 index 0000000000000000000000000000000000000000..649524917a932c481edf1058148bb9571ac6dde0 GIT binary patch literal 134 zcmeZZfPjCKJmzvPWnxbL<8;$eiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-BGG&JuN>6en|FkaI45|qfXx-K&=Tb0=i&%U U%+b%uFD@3<-_>ASeeIVr0Ji%rt^fc4 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281552.sc13.2274570.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281552.sc13.2274570.14 new file mode 100644 index 0000000000000000000000000000000000000000..5668e28094b385b5e3169ee6731e84a61cf9bb3a GIT binary patch literal 193 zcmeZZfPjCKJmzv{_A90Qak}X!#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZG2IcOqyjb9NxQ+9Q{=8P*j#~PEpaYUF1`?u z>H6uZdGR^<#l@lw2R_+-V9yj+h3a{=$Lup_3qp?smlzj6L{Ca)aWYhs!W|pqY17XF E02mxVQ~&?~ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281555.sc13.2274570.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281555.sc13.2274570.15 new file mode 100644 index 0000000000000000000000000000000000000000..d0ae549115936bafb5c2c475682a387724549d5a GIT binary patch literal 193 zcmeZZfPjCKJmzxlIP@#!kJC*@Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th?KyMDix@?PTCE=oX0HwfXx*s(Guqp<>Ctg znXaFnnirpwUtBE8!0^?MVH?8_Rj8g-as$LXe{6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v_p0H}41;aY{R_0-GyfsU^t8&&3gz Un4_PQUtBDjyQ$Uo>$ALx0AxNd{Qv*} literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281563.sc13.2274570.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281563.sc13.2274570.17 new file mode 100644 index 0000000000000000000000000000000000000000..0f40b4a47b01452ce1695ec8231d99827083f0c3 GIT binary patch literal 193 zcmeZZfPjCKJmzv9T75R{kJC*@Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th`i&R4i%`mPTCE=oHJO?gUuBv)e`3t<>Ctg znXaFnnirpwUtBE8@ZhJN)4{trs!%C>J;)m!-$t+HWYEsx{O!esQrVL&FEVn^jiRRiS!b?J@hzsTK&=Bf%xc#ShVwl3AP#({$4&evb1Z E0M(2`(f|Me literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282323.sc13.2296022.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282323.sc13.2296022.0 new file mode 100644 index 0000000000000000000000000000000000000000..18f728d79d7532db9417515e47796d4e9d943e42 GIT binary patch literal 134 zcmeZZfPjCKJmzxd1w2;y>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;=N~KogCEMn|FkaIG{2LJ#7 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282328.sc13.2296022.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282328.sc13.2296022.1 new file mode 100644 index 0000000000000000000000000000000000000000..ac51c9f4c20c88baa9473985ce4fdc4093bf099a GIT binary patch literal 193 zcmeZZfPjCKJmzw)d?BRv*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL?->5t_sv#C+!AbPD>*Zu(<-|TH;)yTznxQ z)AiF+^Wt;zi;G1W9(=X?pYerD6{_de9<$G!QxJM2xWu^lA$n3Wi<6<66mHqvxmmd! E04&%*!2kdN literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282331.sc13.2296022.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282331.sc13.2296022.2 new file mode 100644 index 0000000000000000000000000000000000000000..55acf9de289b0e89253c00fdf11ca3d6e906c061 GIT binary patch literal 193 zcmeZZfPjCKJmzw~`}k4quhUIODc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$gj-Cr7BQ!owOT#Ip^Dd2AeBTp(V~G%EcD~ zGF?ACH7`CVzqnYG;lNM3oGiBOs!%C>J;)m!-$t+HWYEpP=b13?W FJpjIuLpJ~b literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282334.sc13.2296022.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282334.sc13.2296022.3 new file mode 100644 index 0000000000000000000000000000000000000000..a110068f7bcdb00d75c3200eea580381453ecb9b GIT binary patch literal 134 zcmeZZfPjCKJmzwy99*OR*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vqepjn|FkaIM+^E2R2v0T1$|NpNk_b VF-JcqzqnZR!@-HRf1dXm000}7FNgpD literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282339.sc13.2296022.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282339.sc13.2296022.4 new file mode 100644 index 0000000000000000000000000000000000000000..04d00749a72cb5874a6eb3a0e369150972eeef9c GIT binary patch literal 193 zcmeZZfPjCKJmzxVxO7?LuhUIODc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$nn_^-m5^(b<%F|l3N}}uQcIjml#4F} zWV(KOYF>O!esQrV!-Fq&o9`$0sY3O<+GF;avl*dBf=i5xAEGBEvp5;5iQ$&bQw`%+ E0LH*W(*OVf literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282342.sc13.2296022.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282342.sc13.2296022.5 new file mode 100644 index 0000000000000000000000000000000000000000..65fd4ba7d04b5bdb94c8c9b4dbc24362efae5f49 GIT binary patch literal 193 zcmeZZfPjCKJmzvnaPHRp>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-Qq)mwtpYXINxQ+9)5?Dj*j#}sEpaYUF1`?u z>H6uZdGR^<#l@lw2R_;DePw=46{_de9<$G!vk`hExWu^lA$n3Wi<6<68t&PYTD-~t E08p(#zW@LL literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282345.sc13.2296022.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282345.sc13.2296022.6 new file mode 100644 index 0000000000000000000000000000000000000000..c1339e2c43802263b3fcee8af6fce71d39dbc42e GIT binary patch literal 134 zcmeZZfPjCKJmzw;oy*et>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@;Tf0sT|bYn|FkaIL~Y6fXx-K(Guk1=i&%U U%+b%uFD@3f`_X61Zq)x10Q9IXHvj+t literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282350.sc13.2296022.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282350.sc13.2296022.7 new file mode 100644 index 0000000000000000000000000000000000000000..af8559b86a31561693b20d180a17dedd758df343 GIT binary patch literal 193 zcmeZZfPjCKJmzwK4(QeX>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a%9zt$tqBDowOT#IoF(pm@81NCC(+v#TNoH zT|YfFFFq%~xLB0oz$d#v=}mJ~p?Y5JG5gFZdk(Hgf=i5xAEGBEvp5;5so|vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a#W#wrwY_uC+!Ab&aNMsU~>g(w8XhYx%fgr zrt7Dt=Edja7Z-~%D15Z*TlKO^6{_de9<$G!FA;hqxWu^lA$n3Wi<6<67M!;6EYaKz E0NZCmyZ`_I literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282356.sc13.2296022.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282356.sc13.2296022.9 new file mode 100644 index 0000000000000000000000000000000000000000..f42a128a806c5307b52f13bd50d80248f21b5371 GIT binary patch literal 134 zcmeZZfPjCKJmzxlbCcEm>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-qFA}vP7Z4B%{xLyoZma;z~&0rY6)`jb8&NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zk=|wUO$BPMlXinIr#FWi*j#~HEpaYUF1`?u z>H6uZdGR^<#l@lw4xj99ziWQ03f1#!kJ)F=?Fc;*Tw+}O5IrfG#mP`j3Hxn0kNX<| E04?@F1^@s6 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282364.sc13.2296022.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282364.sc13.2296022.11 new file mode 100644 index 0000000000000000000000000000000000000000..44bbc250ae475cfc6fec3275046802e067d73f94 GIT binary patch literal 193 zcmeZZfPjCKJmzxF^_0^8>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-5~mz`R0V3TlXinIr_fw!u(<+tTH;)yTznxQ z)AiF+^Wt;zi;G1W8a~=xQCzTJ6{_de9<$G!nFu`+Tw+}O5IrfG#mP`j4u@?dr$z1u E0NGYQd;kCd literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282369.sc13.2296022.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282369.sc13.2296022.12 new file mode 100644 index 0000000000000000000000000000000000000000..a0ad77e62192a7e9329818f3f19a66639fee7e36 GIT binary patch literal 134 zcmeZZfPjCKJmzwq-0EoX*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v@WhY`~=0XzCHH~;_u literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282374.sc13.2296022.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282374.sc13.2296022.13 new file mode 100644 index 0000000000000000000000000000000000000000..1274e599a75986b812a038073c307068ed3684e7 GIT binary patch literal 193 zcmeZZfPjCKJmzxp&7W@g*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWV#}&jSAFUC+!Ab&XdeD!R89oYl(A-a`AdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#M=GqdsV2pulAUI=CrVP1)D3-z^En8CCbGY z0y17dJvA>rC%?E@ltJN>oxPj*S~aMqV*kesoYRmrNpOjA@k2DFWELkwbusL+i9Y=8 FC;$`#K(YV; literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282381.sc13.2296022.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282381.sc13.2296022.15 new file mode 100644 index 0000000000000000000000000000000000000000..105a139e593f97b6ba6efe6008d796f6dd06f459 GIT binary patch literal 134 zcmeZZfPjCKJmzvPTk355*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWNq-Ta5<>CH}41;an|#=fXx-K*AnF7=i&%U U%+b%uFD@4KGn;68=IOM>00FfwKmY&$ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282385.sc13.2296022.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282385.sc13.2296022.16 new file mode 100644 index 0000000000000000000000000000000000000000..56b09cb22c46856f541a3ba040e710ee9bbb2eb3 GIT binary patch literal 195 zcmeZZfPjCKJmzvvu*fj^>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vTIjxxGL1#S9{DpbFOL01e+_+$fzaGCCbGY z0y17dJvA>rC%?E@lwrXayLyw-Wol4O#r}^OIPF>ynk2ZyxcDKOQZkE^VY-gm$SACv F001}2K|cTh literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282389.sc13.2296022.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282389.sc13.2296022.17 new file mode 100644 index 0000000000000000000000000000000000000000..c8552c650ce4778972b972a3fc9c9526585a255c GIT binary patch literal 195 zcmeZZfPjCKJmzwKo8oQy*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#5ZcnI#sB-ulAUI=5!JD0h=q(#Hc0CCCbGY z0y17dJvA>rC%?E@l%e6Bopj1R9W|(?V*kesoO6*hNpOjA@k2DFWELkwbt&AiQ7TkY F0ss^fKKB3s literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282391.sc13.2296022.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282391.sc13.2296022.18 new file mode 100644 index 0000000000000000000000000000000000000000..6cf980a4e80607038986a92e05e79cb52e853d9e GIT binary patch literal 134 zcmeZZfPjCKJmzv{`2RQk>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vPPA;SPp9L%{xLyoLmPO%s}P}IA{rS@pEy6 VCFbbofuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjw5s|_pS*lQTU+ppb%(=B?1=w7HW=1V>E>SMN z5Rmcu>8W|~Ir+uKq6`n-*r`l?HAxMsso4K91E)kQLX!lS7#BZ8Q%YuWGECPEo9EFY F4FKiHMWFxy literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283064.sc16.2520987.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283064.sc16.2520987.1 new file mode 100644 index 0000000000000000000000000000000000000000..f1198c1fb2e77db23abca0c8a8cff2cf45c0d623 GIT binary patch literal 195 zcmeZZfPjCKJmzxhJudVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vUwb3MqI@0 FHUQWRLWTeU literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283067.sc16.2520987.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283067.sc16.2520987.2 new file mode 100644 index 0000000000000000000000000000000000000000..a97be4377ad0b76a7aca4e3caf112759a8c81451 GIT binary patch literal 134 zcmeZZfPjCKJmzxNZhd_3uhUIODc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$n%CmV>zg~H}41;aY_n51)D42s3pk7&&3gz Un4_PQUtBEuu({sWPq6X>0H6df#{d8T literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283072.sc16.2520987.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283072.sc16.2520987.3 new file mode 100644 index 0000000000000000000000000000000000000000..c4b095ed3edfd2df3d184d0c89849b99de6a64b2 GIT binary patch literal 195 zcmeZZfPjCKJmzwGbMig->vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-va;C!ohsDaS9{Dpb6R%ugUuCaWz-Vq66N9x z0U58Ko|+e*lV4mc%CO+M-LwhrjcQO$#r}^OIAPom_v%j&9 F3jj{nLM#9P literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283075.sc16.2520987.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283075.sc16.2520987.4 new file mode 100644 index 0000000000000000000000000000000000000000..7a64fc88fab3631ce17448e1e14707d8b8b48583 GIT binary patch literal 195 zcmeZZfPjCKJmzvb9{T>^uhUIODc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th5yumnaus z2*`N-^whlgoc!WqQHF+Bc2a$!J!()*#r}^OIDI`4nk2ZyxcDKOQZkE^p}G=I+K4Tc G;sF4S7(+Aw literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283078.sc16.2520987.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283078.sc16.2520987.5 new file mode 100644 index 0000000000000000000000000000000000000000..c09e0ea1053ec75e0cb3029375dbece32144732e GIT binary patch literal 134 zcmeZZfPjCKJmzw4JG%1WU#FXnQoKn;iJ5tNu4SotC00g3dR#gssd>fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwk>^}GH|3z_-n=7Z#JNdsHP~DMCoMrPelCu% V#2o#c{NiHK9}hZh@5Yab|*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#DpV!mMYZTS9{DpbG})70c@^7JEN93mnaus z2*`N-^whlgoc!WqQHBK%?HH?^AE`k#75hJC;B;7r&?Lbn#>EfOl#*GT4Aphuk4>8N Gg**T!JVavv literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283086.sc16.2520987.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283086.sc16.2520987.7 new file mode 100644 index 0000000000000000000000000000000000000000..3eafaac0c89d27a9ba904e48ac9ed465178aa229 GIT binary patch literal 195 zcmeZZfPjCKJmzw$q%3{>*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq%%KJL=|f8t376)Ij^i)1~ylqgHcPIOO%T* z1Z2E^dTL&LPJVH*C75hJC;0#!c&?Lbn#>EfOl#*GT4AteJZyQ&# GDF^@%DMMob literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283089.sc16.2520987.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283089.sc16.2520987.8 new file mode 100644 index 0000000000000000000000000000000000000000..70f74ea364e1793b8fdc14b4d265e5c390435f9f GIT binary patch literal 134 zcmeZZfPjCKJmzvbmd8E$>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-60q9-oE+5Ln|FkaIJLjWgUuCi))M66=i&%U U%+b%uFD@3{baIOAm6-1@0X?-aw*UYD literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283094.sc16.2520987.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283094.sc16.2520987.9 new file mode 100644 index 0000000000000000000000000000000000000000..6164b2e215f1411e2296ceb2ae3a7cd63e862295 GIT binary patch literal 195 zcmeZZfPjCKJmzxVi)?!O*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vB(_*ATor2Wt376)Idhp>z~%~cGHQu)iE{CU zfQ;8qPtA+Z$uBMzWng$vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-($idUOciSGt376)Im_B(!R88dF=~l(iE{CU zfQ;8qPtA+Z$uBMzWk`5s*ZeTfK@F;@*#9vDCuavjlLVI-7e7Q(N@j5~RF?yv?IOVw F{s56fL8brz literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283100.sc16.2520987.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283100.sc16.2520987.11 new file mode 100644 index 0000000000000000000000000000000000000000..654f5991b8efb7e34dac76b593d21641c3162140 GIT binary patch literal 134 zcmeZZfPjCKJmzvP$>M$f*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vM7QtLF*&HYH}41;aemn@05(^^MN5#2pNk_b UF-JcqzqnY`O}yE*^y`uL09EZUMgRZ+ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283104.sc16.2520987.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283104.sc16.2520987.12 new file mode 100644 index 0000000000000000000000000000000000000000..dcbce7d315ffb17180d4d3d1b458f35708a15d12 GIT binary patch literal 195 zcmeZZfPjCKJmzxpz0-Q}*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWTEV~2dYqWU+ppb%=y7w2W+lDH=~v~mnaus z2*`N-^whlgoc!WqQ3i)6cDAvA?BCCbGY z0y17dJvA>rC%?E@l!4)u-SXaRN@`F|#r}^OI4>e;lHd~K;)iHT$t+HW>RMo68?bSr FHUP~^LGJ(n literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283110.sc16.2520987.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283110.sc16.2520987.14 new file mode 100644 index 0000000000000000000000000000000000000000..862497b4949ab4b630661fc7aa09acc2ef1bb6cb GIT binary patch literal 134 zcmeZZfPjCKJmzxxo!$TPuhUIODc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$oac2o^nugZ{86y;?%b|2sT&1RZEbIpNk_b UF-JcqzqnXbz^={qq`7V?0FT%%rT_o{ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283115.sc16.2520987.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283115.sc16.2520987.15 new file mode 100644 index 0000000000000000000000000000000000000000..3fa9391711a3bf02a51fcc7c59cd979554b95942 GIT binary patch literal 195 zcmeZZfPjCKJmzv<;(7S$uhUIODc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$kAPDYN}9kU+ppb%z0ztBe1ywy^LDoT%ufj zAt2-R(^K=}bMlLeMHw94+6CmAA literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283118.sc16.2520987.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283118.sc16.2520987.16 new file mode 100644 index 0000000000000000000000000000000000000000..84b2dae01e10ead0717275c47ceefda9171a971e GIT binary patch literal 195 zcmeZZfPjCKJmzvfxW4-JU#FXnQoKn;iJ5tNu4SotC00g3dR#gssd>fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjw5mm7ljH*y`U+ppb%-J8h7HqCSAETBymnaus z2*`N-^whlgoc!WqQ3i+ScKEfOl#*GT4Aqq&Vmtrv G_0<5N4n(g2 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283121.sc16.2520987.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283121.sc16.2520987.17 new file mode 100644 index 0000000000000000000000000000000000000000..eda1296c82f45df8d79a601dd09e0edb6e54d7f8 GIT binary patch literal 134 zcmeZZfPjCKJmzv1mc+jK>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-qRMHuLk?>0%{xLyoQ7ZHz~&0LX$f-ib8&dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vBr$Cbt18soS9{Dpb6&nv1vXcppHWMkOO%T* z1Z2E^dTL&LPJVH*C75hJC;B>o-&?Lbn#>EfOl#*GT4ArF|U>k7p G$!-9H2t=U( literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283129.sc16.2520987.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283129.sc16.2520987.19 new file mode 100644 index 0000000000000000000000000000000000000000..69597aaaf049fb15376f8da7de456ca822971f66 GIT binary patch literal 141 zcmeZZfPjCKJmzw)XN!OL*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#4zVcqbk(gS9{DpbKY%A0Glf?fl*7GOO%T* Z1Z2E^dTL&LPJVH*D1*auyV{Pm_W-?PGfMyf literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283892.sc11.2190319.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283892.sc11.2190319.0 new file mode 100644 index 0000000000000000000000000000000000000000..100039dd648445731124bcd78a1e44632811321f GIT binary patch literal 195 zcmeZZfPjCKJmzwGTsK<#&*`S46mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL|kTTtt!;qS9{Dpb4pA#0h=o@fl*7GOO%T* z1Z2E^dTL&LPJVH*D8qvncJE!q*Q!A^75hJC;A}wBB*7)d#ShVxl3AP#)pdZ&Hf7O% FNdRvILBIe2 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283895.sc11.2190319.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283895.sc11.2190319.1 new file mode 100644 index 0000000000000000000000000000000000000000..405a78cf24829bbf049e329cf43ee64842bf08d0 GIT binary patch literal 134 zcmeZZfPjCKJmzvfzxQbEKc}0HQoKn;iJ5tNu4SotC00g3dR#gssd>fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwk?ZvjMdhI8-n=7Z#90&h7;LV9yOtmqKNm+> VVvc@JesQtrMc3uFcer`x0|4sSFpB^H literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283900.sc11.2190319.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283900.sc11.2190319.2 new file mode 100644 index 0000000000000000000000000000000000000000..a0d4fd7e1bf4332c05bd19796fa4df61841ed06d GIT binary patch literal 195 zcmeZZfPjCKJmzxF@#kFs&*`S46mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vLJ{ z$awwq)V%nd{NiF!28Jhg_NiMBs6jOq`#)yj;B*7)d#ShVxl3AP#)z!deo8aO7 F4**?hK_&nI literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283903.sc11.2190319.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283903.sc11.2190319.3 new file mode 100644 index 0000000000000000000000000000000000000000..23f51cc4fb3b03118ecf513f44852353623c5995 GIT binary patch literal 195 zcmeZZfPjCKJmzwSPI|lkpVLi8Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$j0WCv8qsWU+ppb%(+?g9oSrfNsL&!qU+ppb%=s>TC)iwp$&6a!T%ufj zAt2-R(^K=}bMlLeMHv=6vMX4!ae*3CQ?dVJ2F{cWgeD0tF)n_Hrj*R$WT-9%c3b7; G_i_Nh(L`kc literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283914.sc11.2190319.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283914.sc11.2190319.6 new file mode 100644 index 0000000000000000000000000000000000000000..5ae294397f04ddc0eea0fa5e2d50fdc9c325621b GIT binary patch literal 195 zcmeZZfPjCKJmzwqtDd>(pVLi8Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$j`_Z)v8c)U+ppb%-Qq}Vy?gxMlEqJQ7*m^ zkn#HIsd@1^`NhSe3=WU%X3Iqvt3fpt`#)yjyo;nsf=i5xAEGHGvp5;5>jA5+cvCC~ E0LTkOF8}}l literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283917.sc11.2190319.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283917.sc11.2190319.7 new file mode 100644 index 0000000000000000000000000000000000000000..4c150b8e53ccd4f29e01beb09a2de323d12cb223 GIT binary patch literal 134 zcmeZZfPjCKJmzwGO%B-n&*`S46mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vgx4rlMGk82%{xLyoST#a!R88hY6)`jb8&_*Zg!6nAU57Cs8S)2^jwcxi+?!u(I E0GQE4YXATM literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283925.sc11.2190319.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283925.sc11.2190319.9 new file mode 100644 index 0000000000000000000000000000000000000000..7ff19659296befe4e40fcde41e4caa8f18d09b14 GIT binary patch literal 195 zcmeZZfPjCKJmzwiNcnI5=XBFiiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GEe9ZmnziUS9{DpbKdBOm@6=iQA?all#4F} zWW0WQYF>O!esQrV1H%J5rq^q}sX;Xr`#)yjG@O9YB*7)d#ShVxl3AP#)ur&s=1`vU F4*+gXLZAQu literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283927.sc11.2190319.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283927.sc11.2190319.10 new file mode 100644 index 0000000000000000000000000000000000000000..9b7ad02ba0be030e061b40b5601864f9c59fca9f GIT binary patch literal 134 zcmeZZfPjCKJmzwK+V*$rKc}0HQoKn;iJ5tNu4SotC00g3dR#gssd>fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwk%^r4{BlrpZ{86y;uN#}2R2v0OG}W8pNk_b UF-JcqzqnY`>}H#7O~!Q<0Gz`v_5c6? literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283932.sc11.2190319.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283932.sc11.2190319.11 new file mode 100644 index 0000000000000000000000000000000000000000..a6f1f18d7293566520fae180b716f9fa2ff4682a GIT binary patch literal 195 zcmeZZfPjCKJmzxd_b6@u=XBFiiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(k*UxRuyXQt376)Ip4oh2AeA|ol#4iOO%T* z1Z2E^dTL&LPJVH*CKAJm|liv1rmaE81_Xp-O(O!esQrV!-FSw(GNU#szEgs`#)yjWX?folHd~K;)iHT$t+HW>N;@8=J`*# FlK^GSLx%tW literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283938.sc11.2190319.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283938.sc11.2190319.13 new file mode 100644 index 0000000000000000000000000000000000000000..3e170181e8c9b4b1ff56b51fa2e5292292376ddb GIT binary patch literal 134 zcmeZZfPjCKJmzwq?%%WHpVLi8Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$nDp*E99W&-n=7Z#JQS(FW6iGZ!JMCelCu% V#2o#c{NiF!yB{^STUQ+V2mrazF{JfuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjw5gi$&PF1M6ulAUI<_z|~2{u|V~&4-J6 FsQ@0eLs0+# literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283946.sc11.2190319.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283946.sc11.2190319.15 new file mode 100644 index 0000000000000000000000000000000000000000..d0bdf7cc2419390e1ad8da474438c523a4967a8a GIT binary patch literal 195 zcmeZZfPjCKJmzvr3$NSt&*`S46mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v)O!esQrVgTp&Jadl2nHK?Xy|HllRSxA~BxWu^lA(~P$i<4ox?%EtrmX!qn DFIGUE literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283949.sc11.2190319.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283949.sc11.2190319.16 new file mode 100644 index 0000000000000000000000000000000000000000..ab749924505884cd4a2a8e15dd1548e8ab4ac22e GIT binary patch literal 134 zcmeZZfPjCKJmzw~JruwDpVLi8Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$c}Gwx5`1yy?IB-h%+oW0c@^-kCq@8KNm+> VVvc@JesQs=e9a_Vrsu~m006XAFm?a{ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283953.sc11.2190319.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283953.sc11.2190319.17 new file mode 100644 index 0000000000000000000000000000000000000000..ea067c01547388263a094bc0ccb995ee2d0c263b GIT binary patch literal 195 zcmeZZfPjCKJmzwiSl8_N=XBFiiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vQZ}Fsw&jnS9{DpbNWfvfz1_|&8Q{LCCbGY z0y17dJvA>rC%?E@l%e5{-MUKeQ)*C6#r}^OI8P#JlHd~K;)iHT$t+HW>SExvJyJH~ FApoDaLU{lH literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283957.sc11.2190319.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283957.sc11.2190319.18 new file mode 100644 index 0000000000000000000000000000000000000000..94171004779b914dfaff40222ffd7af1a7cf4d6f GIT binary patch literal 195 zcmeZZfPjCKJmzv5>4oh5=XBFiiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vM_y_iYnCHS9{DpbEa>Gm@6=cQA?all#4F} zWW0WQYF>O!esQrVgTf=bD^?m(YEVtZ{*M_rnYJJ_NpOjA@k2DFWELkwbukFpTKr;q F0{|GMKtccj literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283960.sc11.2190319.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283960.sc11.2190319.19 new file mode 100644 index 0000000000000000000000000000000000000000..57cc84743fb53df5293373f66e52c9ee2bde0791 GIT binary patch literal 134 zcmeZZfPjCKJmzu+?qc2d&*`S46mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vO!esQrVL&9yl)8*yI)S#M*{U0-Ms@y|plHd~K;)iHT$t+HW>SFk0QxWA} F3;=kRLAU?_ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284630.sc12.184661.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284630.sc12.184661.1 new file mode 100644 index 0000000000000000000000000000000000000000..68652ed61f481c0b246f45f8f35e65c0d601ba93 GIT binary patch literal 195 zcmeZZfPjCKJmzvX+*p$O-|42K6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#CqRWPgSV7ulAUI=8W-L1~yk<9;22xmnaus z2*`N-^whlgoc!WqQ3i*Hc0w;Fd8Hy`OLRA0& literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284633.sc12.184661.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284633.sc12.184661.2 new file mode 100644 index 0000000000000000000000000000000000000000..e3eb554bebec83aae6aad5b5f4374a3794c3143a GIT binary patch literal 134 zcmeZZfPjCKJmzvvi?2!h?{w2qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-VtrKogB;Y{n|FkaIG3G+m@D9?CCJ6k#Sxa6 Tqo0#sTrAqc-)C#Qv-cqYDbX(f literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284638.sc12.184661.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284638.sc12.184661.3 new file mode 100644 index 0000000000000000000000000000000000000000..38350d3010e53ff9d53f3cb51e8fd2f9852f1b6e GIT binary patch literal 195 zcmeZZfPjCKJmzxdM$AnA?{w2qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a{urxX;rAXulAUI=Bz#gF;`$dqn0?AC>LJ{ z$awwq)V%nd{NiF!h64}nE`QoMPYtT6*#9vD=Qkux5?o?j{18nknZ?OaT@FudHkDnt F2>@`&MxX!y literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284641.sc12.184661.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284641.sc12.184661.4 new file mode 100644 index 0000000000000000000000000000000000000000..4c7975eda3114e5dcf4d55ee7e803ef6820426f2 GIT binary patch literal 195 zcmeZZfPjCKJmzv%yBB5rce?2)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zd8U47mnziUS9{DpbLJn0m@BY=QA?all#4F} zWW0WQYF>O!esQrV1H((ZB_FnbQiEzL_J7R4c@#;L1eX{WKSWbXW^po9*Me&{`#;zv F0RRW3MG*i1 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284645.sc12.184661.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284645.sc12.184661.5 new file mode 100644 index 0000000000000000000000000000000000000000..bf72efc6fe14ebccdc221203a561e9288d7f786e GIT binary patch literal 134 zcmeZZfPjCKJmzwq2=mMQ?{w2qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a{Jf!3^}N|H}41;ah}=@F;~D}OOT77iz6&C TM?WXOxL7oG@>E+*B^^@$T|X|r literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284650.sc12.184661.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284650.sc12.184661.6 new file mode 100644 index 0000000000000000000000000000000000000000..c236b45dcb6eafbaac447d0a38cf6035b7c2c97b GIT binary patch literal 195 zcmeZZfPjCKJmzvPzSE!e-|42K6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#IA)?Tor2Wt376)Ik)*w0GlhYkWovVOO%T* z1Z2E^dTL&LPJVH*C_}NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZiTvCXr3y9o)gH6YoSPaU<_aug)Dq_s<>Ctg z8Lyw7nirpwUtBE8(D2MIp}xpl4XUZw|1krnOcO$r1eX{WKSWbXW^po9m%?rvb@yq; E0NVmT_5c6? literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284656.sc12.184661.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284656.sc12.184661.8 new file mode 100644 index 0000000000000000000000000000000000000000..138d443b8684fd959951bbdb15edd5d40e698686 GIT binary patch literal 134 zcmeZZfPjCKJmzwK4ra^w?{w2qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GIhe!J91ESZ{86y;=FtjVy-}dmLL~D7e`oP Tj($#lak1zj_T{#YwhEpAJK8Qy literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284660.sc12.184661.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284660.sc12.184661.9 new file mode 100644 index 0000000000000000000000000000000000000000..801f2c478f6cc23c6a6d61618e741fb5be3be046 GIT binary patch literal 195 zcmeZZfPjCKJmzvbty9YV?{w2qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^5l7stSZ#pS9{DpbDmI92AeCem{CidOO%T* z1Z2E^dTL&LPJVH*CO!esQrV!-J=GY9S$IYEVtZ{*M_r6OlAYaEWp8Lo}si7AHe>IqbEOdC_+i E00=5XApigX literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284666.sc12.184661.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284666.sc12.184661.11 new file mode 100644 index 0000000000000000000000000000000000000000..f381907f4dc342ee5a673283b4d80c0180c33459 GIT binary patch literal 134 zcmeZZfPjCKJmzxlxV$>=ztc@eDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Ti12xx<#JGSZ{86y; UVvc@JesQtr;_g1%I{9T;0CQt6A^-pY literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284671.sc12.184661.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284671.sc12.184661.12 new file mode 100644 index 0000000000000000000000000000000000000000..b5f686749e8023a5a291cf5483ae591ab94006c9 GIT binary patch literal 195 zcmeZZfPjCKJmzv%?K_tL-|42K6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vUyxtCeomW F9RQjfL4g1O literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284674.sc12.184661.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284674.sc12.184661.13 new file mode 100644 index 0000000000000000000000000000000000000000..bbaeb513e01695c862edfd5211b0b50f70f48755 GIT binary patch literal 195 zcmeZZfPjCKJmzvH)hsRe?{w2qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vf4m5L=|f8t376)Ie8hEfz1_I#;7IECCbGY z0y17dJvA>rC%?E@l;OZ5JH;7l!fH@W#r}^OIHx0NlHd~K;)iHT$t+HW>QdNl^WMeJ F1OWSGKi~iW literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284677.sc12.184661.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284677.sc12.184661.14 new file mode 100644 index 0000000000000000000000000000000000000000..9c4f8aa01f10a4995e779ecdd010e7497396e05a GIT binary patch literal 134 zcmeZZfPjCKJmzvTHis1cce?2)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zx%VRWh8)z~n|FkaIOU#0%oPaI66E6N;s{I3 T(a*^*E*AaqpwTw#NYOC>3mz~L literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284681.sc12.184661.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284681.sc12.184661.15 new file mode 100644 index 0000000000000000000000000000000000000000..486dd711d731876f9947bc74692b2e13c1f0c256 GIT binary patch literal 195 zcmeZZfPjCKJmzwq%gZnN?{w2qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-5;s*nNfm1Dt376)Ir-j0%oSM9s3p!N%EcD~ zGG0GDH7`CVzqnYGLE(kntvCGt)u5V+{U0-M&PLKC!6nAU57Cs8S)2^jwP2f#VCpIf E0MD*M?EnA( literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284684.sc12.184661.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284684.sc12.184661.16 new file mode 100644 index 0000000000000000000000000000000000000000..4c221fa154d389ee5e6e34328ff368c046432d24 GIT binary patch literal 195 zcmeZZfPjCKJmzw;*4P#Qce?2)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZNtfBlt_n5x)gH6YoQJvX!R89AVAK-l66N9x z0U58Ko|+e*lV4mc%HVLvE-gnSM-8f}*#9vD=U*gE5?o?j{18nknZ?OaT@2f7zMuV~ F4FH8nKv@6) literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284687.sc12.184661.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284687.sc12.184661.17 new file mode 100644 index 0000000000000000000000000000000000000000..221aee63048d6486d972f8b61173e18796e4d3b5 GIT binary patch literal 134 zcmeZZfPjCKJmzw)jC@i2-|42K6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vMEcvLH*!#OZ{86y;#6XI2{uHq)$ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284692.sc12.184661.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284692.sc12.184661.18 new file mode 100644 index 0000000000000000000000000000000000000000..0b2e719ada9905e0b1015c33525a2557e7307e74 GIT binary patch literal 195 zcmeZZfPjCKJmzwS*s_-Xce?2)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z(T!bnUlnTZt376)IYSRZ%oSM4s3p!N%EcD~ zGG0GDH7`CVzqnYGf#IHAi2CH6YEVtZ{*M_r_abSM;1c8FhiFR4EKY{$+GLYc`R^J4 D)uTaE literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285450.sc9.1183018.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285450.sc9.1183018.0 new file mode 100644 index 0000000000000000000000000000000000000000..71260a87bcc59ec5c5d4d41ffb4f22db7946cdbb GIT binary patch literal 195 zcmeZZfPjCKJmzxBt=nkC;C$0jiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-lEO7xRuyXQt376)IqMlWgUuCK#i%9DCCbGY z0y17dJvA>rC%?E@lp*1so#IxLJ8Do(#r}^OIIkmVlHd~K;)iHT$t+HW>DpwIobyQw E0Km6B#Q*>R literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285453.sc9.1183018.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285453.sc9.1183018.1 new file mode 100644 index 0000000000000000000000000000000000000000..e63f33de2038169154d4b1e0542a54dac5f992e0 GIT binary patch literal 134 zcmeZZfPjCKJmzwi-H0`2aK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZF+1W~D+e|A<{cp;&Th4Mu(<*uT7q2sTpVGE UIr=&I#l@l*U3+c6SA6~h0CABmC;$Ke literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285457.sc9.1183018.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285457.sc9.1183018.2 new file mode 100644 index 0000000000000000000000000000000000000000..86b171ef074e7d9564b2c785e5041757a77aeefe GIT binary patch literal 195 zcmeZZfPjCKJmzw)bS*PsaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zsr9yMQ-zxQYLD4xPQ`-|a|Kp2YKe1+a`ANT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZaXoy@MHOo9t376)IWw6UyxtW-iYq FO#t?PKB)iz literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285463.sc9.1183018.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285463.sc9.1183018.4 new file mode 100644 index 0000000000000000000000000000000000000000..2ce3d1c9755c5df58c6e4bb8bffc7afd945bef84 GIT binary patch literal 134 zcmeZZfPjCKJmzvdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vrC%?E@lp*1k-J)++g=$bu#r}^OIMbdXG)ZuYaq&YmrDPT-!*p%7$ujz! F3;^buKC=J- literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285471.sc9.1183018.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285471.sc9.1183018.6 new file mode 100644 index 0000000000000000000000000000000000000000..9174e87eefb57285e85e6981fb78c35e6cd3647a GIT binary patch literal 195 zcmeZZfPjCKJmzu=wth8daK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZdFZrqx+>J%S9{Dpa|U&O1Dh+bj!{dTOO%T* z1Z2E^dTL&LPJVH*C_}<^JL#M_4mGHzV*kesoC}aNNpOjA@k2DFWELkwbsgAblfU!F FM*wzhK;HlW literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285474.sc9.1183018.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285474.sc9.1183018.7 new file mode 100644 index 0000000000000000000000000000000000000000..4a69636566dbf881c82e495d09f350ff4fde39d6 GIT binary patch literal 134 zcmeZZfPjCKJmzwi-h+v;?{Mxj4cS UbM$lai;G25Cnwr&TEBGz0Ciq0CjbBd literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285478.sc9.1183018.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285478.sc9.1183018.8 new file mode 100644 index 0000000000000000000000000000000000000000..f08baf63ce29d61f0fd16ee6537fba4b71926687 GIT binary patch literal 195 zcmeZZfPjCKJmzw~dcMh$!TF}66mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vF9;#4tU+ppb%vo)-1#GUsdPXgAE>SMN z5Rmcu>8W|~Ir+uKq6`eT?KnL;EY+Zziv1rma9%*tB*7)d#ShVxl3AP#)pcNxjfs`$ FUI6D=KOz7C literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285482.sc9.1183018.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285482.sc9.1183018.9 new file mode 100644 index 0000000000000000000000000000000000000000..7bb059aa143b90ac896c1fb56c576e499d5df987 GIT binary patch literal 195 zcmeZZfPjCKJmzvv59qOCaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zd6f2WjVjdKS9{Dpb0(jJm@BY>QA?all#4F} zWW0WQYF>O!esQrV!-30omEnbfYEVtZ{*M_r_aSML;1c8FhiFR4EKY{%YB*^lQRaIa E0LL;wo&W#< literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285484.sc9.1183018.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285484.sc9.1183018.10 new file mode 100644 index 0000000000000000000000000000000000000000..ed657a70a46215af13e771f9e57a4120fdc047fd GIT binary patch literal 134 zcmeZZfPjCKJmzv<6}PfxaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZkqijYlY^Rj^Nx@ar$95rT!C;cK`wqSj!JYz*(v1! literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285489.sc9.1183018.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285489.sc9.1183018.11 new file mode 100644 index 0000000000000000000000000000000000000000..ef9ff28bf249dd5a12bbb89b5394ac293ea0e72e GIT binary patch literal 195 zcmeZZfPjCKJmzw)Kjvw};C$0jiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;wNzExhmA$S9{DpbEdd?fz1`z$fzaGCCbGY z0y17dJvA>rC%?E@l!4*8-OcwZKh&U_iv1rma9&5!B*7)d#ShVxl3AP#)#Y&5W=4Q2 FBLHCCK3f0) literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285492.sc9.1183018.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285492.sc9.1183018.12 new file mode 100644 index 0000000000000000000000000000000000000000..8b0d2a03716139ab06b96ed568d896b73ef69104 GIT binary patch literal 195 zcmeZZfPjCKJmzw)KA>RB;C$0jiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a#h?!R~2gRt376)Irqvbfz1`z#Hc0CCCbGY z0y17dJvA>rC%?E@l;OcmyD)Pz5jCi$V*kesoPqKPO%hyUT>KDCDVfE|P+bmtZA6y` GnE(Kk(mSdE literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285495.sc9.1183018.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285495.sc9.1183018.13 new file mode 100644 index 0000000000000000000000000000000000000000..90d392ddf988278b36675140a11d1515c214a7bd GIT binary patch literal 134 zcmeZZfPjCKJmzw~xqr@dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v|Vo&E*o~ FRsiBzK+*sJ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285502.sc9.1183018.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285502.sc9.1183018.15 new file mode 100644 index 0000000000000000000000000000000000000000..9f413d564c8963b58e22568ebfd641c637d8d72f GIT binary patch literal 195 zcmeZZfPjCKJmzxxyxM2a;C$0jiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;>?-Ks0uar)gH6YoPS&ofXx-y!l)(ACCbGY z0y17dJvA>rC%?E@ltJN+-S%tN-fB=y#r}^OI8)sank2ZyxcDKOQZkE^VY)8cC@qj_ F1OPU}J@Ehl literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285505.sc9.1183018.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285505.sc9.1183018.16 new file mode 100644 index 0000000000000000000000000000000000000000..0be0ab24105746022b9439d773f2bbfd59588b78 GIT binary patch literal 134 zcmeZZfPjCKJmzwmeav!TaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIqlD}T@GsQ%{xLyoX?DNz~%}>Y6)`jb8&Qvd(} literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285510.sc9.1183018.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285510.sc9.1183018.17 new file mode 100644 index 0000000000000000000000000000000000000000..19a4f2de07eaad2fdd7d4da4c30b208eef539938 GIT binary patch literal 195 zcmeZZfPjCKJmzvTyEHg5INx-X;!P?_%*@ksElbTSu`&wMhwGHQu)iE{CU zfQ;8qPtA+Z$uBMzWk|SU$Gb?yObx23*#9vD=W`@Y5?o?j{18nknZ?O4U5{;&`R)Y) E0M1Q5EC2ui literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285513.sc9.1183018.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285513.sc9.1183018.18 new file mode 100644 index 0000000000000000000000000000000000000000..decc880d7ce5d5effb0b008837e4456d343773c9 GIT binary patch literal 195 zcmeZZfPjCKJmzvfUli@c;C$0jiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-lC1GdPZetJt376)Iio(tfXx-y#;7IECCbGY z0y17dJvA>rC%?E@l;OcGyLEC8H>g2175hJC;5>??NrFp^iyxvXC9^mgsw?5L%`JxQ Fp8?r3L8$-$ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285516.sc9.1183018.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285516.sc9.1183018.19 new file mode 100644 index 0000000000000000000000000000000000000000..161b723e16d79010b1f5af1829506dc0f75a8aa8 GIT binary patch literal 134 zcmeZZfPjCKJmzvnbFn!yINx-X;!P?_%*@ksElbTSu`&wMhcv;?{Mxj4cS UbM$lai;G2TJ~Y}szM8lh0Qu4^IsgCw literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286186.sc10.1717960.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286186.sc10.1717960.0 new file mode 100644 index 0000000000000000000000000000000000000000..e3e2ebc1ff3a93e9aaa57f82c32f67614cb8c1cb GIT binary patch literal 195 zcmeZZfPjCKJmzxt9&LZc;C$0jiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-V&$jAsR}js)gH6YoErr?!R88VXVens66N9x z0U58Ko|+e*lV4mc%8+o~E;c=My&6dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL^mL^OBHJFt376)Ie%HOfXx-y!Kfw9CCbGY z0y17dJvA>rC%?E@l;OZlyWe|n>!?9B75hJC;H_GGY literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286190.sc10.1717960.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286190.sc10.1717960.2 new file mode 100644 index 0000000000000000000000000000000000000000..6ced3221dc72a6b83a710b95fa6dd2123d52548d GIT binary patch literal 134 zcmeZZfPjCKJmzvH?p^(w!TF}66mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#M{cGL=I~1%{xLyoSWO$g3T3())M66=i&%U U%+b%uFD@3%-89`+;dy-p0OnOKw*UYD literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286195.sc10.1717960.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286195.sc10.1717960.3 new file mode 100644 index 0000000000000000000000000000000000000000..888b9885695acca5932923e27c756d2b3645a569 GIT binary patch literal 195 zcmeZZfPjCKJmzvf$UO9h!TF}66mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vBuO!esQrV!-MN~M@-c&0C3g F4gjIpLjeE) literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286198.sc10.1717960.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286198.sc10.1717960.4 new file mode 100644 index 0000000000000000000000000000000000000000..2b3a1bd4b22adeea6da3e56683c14d87c4136a02 GIT binary patch literal 195 zcmeZZfPjCKJmzwKv!3~u!TF}66mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#N&3zK2@l>ulAUI=G5<=1vXb;7o(Oqmnaus z2*`N-^whlgoc!WqQHBRM?aI^h@2NpG75hJC;GB=7NrFp^iyxvXC9^mgs_Vc88{2Q2 F7y+0iLgfGe literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286201.sc10.1717960.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286201.sc10.1717960.5 new file mode 100644 index 0000000000000000000000000000000000000000..69a23837aff35fa39fabc3fbc1721cf2d72451ce GIT binary patch literal 134 zcmeZZfPjCKJmzv!iJBaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIq;*eUKMKYt376)ISXsk!R89=X4Der66N9x z0U58Ko|+e*lV4mc%HVLrZt=F*7&WM-V*kesoIG_1O%hyUT>KDCDVfE|P+bpJ+1%W* Ga47)enL-Bu literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286209.sc10.1717960.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286209.sc10.1717960.7 new file mode 100644 index 0000000000000000000000000000000000000000..4462e9b6b211a6ce3034523ed9a79d976e5b1ffa GIT binary patch literal 195 zcmeZZfPjCKJmzwqo9+C8!TF}66mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vSMN z5Rmcu>8W|~Ir+uKq6`YR?AX?OajQW!75hJC;QWoGNrFp^iyxvXC9^mgs;gnCjr@6$ FCIC^uKz#rJ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286211.sc10.1717960.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286211.sc10.1717960.8 new file mode 100644 index 0000000000000000000000000000000000000000..2f50d286171512b3000588cc40f0e1a5e22b8a6b GIT binary patch literal 134 zcmeZZfPjCKJmzvn%76I4;C$0jiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vSe<@O*yE!H}41;aenal2sT$BR!fkJpNk_b UF-JcqzqnYGLw=TR`Uj~=00bB=DgXcg literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286216.sc10.1717960.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286216.sc10.1717960.9 new file mode 100644 index 0000000000000000000000000000000000000000..e7d1a79c57fd97f040b0d7d2536885be316ef7bc GIT binary patch literal 195 zcmeZZfPjCKJmzxhXE1+aaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z$^W5Zs0uar)gH6YoPV!D%oW(ns3p!N%EcD~ zGG0GDH7`CVzqnYG!Qr~yjkRHG)S#M*{U0-MI$lR;lHd~K;)iHT$t+HW>QY!>lO)L% F0sxaeKPLbH literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286219.sc10.1717960.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286219.sc10.1717960.10 new file mode 100644 index 0000000000000000000000000000000000000000..ebbface214443507949c421af93b69d312088e1a GIT binary patch literal 195 zcmeZZfPjCKJmzxVO~3t#!TF}66mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWQ*;dBdSnyU+ppb%(-yG9k96q`xv#vxkS16 zLO{mrr>Ew{=j0a`i!w0Wwp04K?zS3KQ?dVJ2F~9|nk2ZyxcDKOQZkE^p}G|2+boaS Gk_7-n9Yjt5 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286222.sc10.1717960.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286222.sc10.1717960.11 new file mode 100644 index 0000000000000000000000000000000000000000..03c4d8f52716895c268d8da147a5abae474bb545 GIT binary patch literal 134 zcmeZZfPjCKJmzw){nYxI!TF}66mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vAEt376)IXmJkz~%}ZWYiMp66N9x z0U58Ko|+e*lV4mc%8+o~?!?k>XVjpYiv1rmaQ;KmB*7)d#ShVxl3AP#)#cDNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zkvq9-l`7QSS9{DpbGFSA0Glgth*3+NOO%T* z1Z2E^dTL&LPJVH*D8qppc1zr9eAJ+tiv1rmaO%uPXp-O(0B5}~7ytkO literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286247.sc10.1717960.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286247.sc10.1717960.18 new file mode 100644 index 0000000000000000000000000000000000000000..a6fa682d503a46da478bfcf0d8af1303830aeb47 GIT binary patch literal 195 zcmeZZfPjCKJmzvvG`#YY!TF}66mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWLfOvqpDDIU+ppb%qcJhVy?hpMlEqJQ7*m^ zkn#HIsd@1^`NhSe3<}rn(zJQDt3fpt`#)yj3`No;!6nAU57Cs8S)2^jl`z@nLEOUU E0Ar;>L;wH) literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286250.sc10.1717960.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286250.sc10.1717960.19 new file mode 100644 index 0000000000000000000000000000000000000000..3af39c699347727dde55563bbfc29d8ecf650681 GIT binary patch literal 195 zcmeZZfPjCKJmzwmhphX>;C$0jiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!ZqXbY*nbaulAUI=Da_3J=k1!-a*Fz literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286253.sc10.1717960.20 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286253.sc10.1717960.20 new file mode 100644 index 0000000000000000000000000000000000000000..0a5a67895fd906edfac90cd2ebf7962765ad84af GIT binary patch literal 134 zcmeZZfPjCKJmzwGmxlaiaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z5nS@6O%7`A%{xLyoZrueg3T33)Dq<4=i&%U U%+b%uFD@26rB-d5r=^|+0MC6bJpcdz literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287096.sc9.1214828.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287096.sc9.1214828.0 new file mode 100644 index 0000000000000000000000000000000000000000..df4e7496f5eddddf047b45cebd9680fe873912b5 GIT binary patch literal 134 zcmeZZfPjCKJmzx#)|ETQ=zP;riZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-lA`?RoE+5Ln|FkaI2U%ygUuC4)Dq<4=i&%U U%+b%uFD@2c++AX8@?+aQ0J>8y9{>OV literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287101.sc9.1214828.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287101.sc9.1214828.1 new file mode 100644 index 0000000000000000000000000000000000000000..d1c32b33bd6d7d656e58ae733c6972056ef01245 GIT binary patch literal 195 zcmeZZfPjCKJmzxVd+l_d(fOvM6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?LJ{ z$awwq)V%nd{NiF!h68u(*t@M}t3fpt`#)yj{DY)Pf=i5xAEGHGvp5;5OJRzQiOHfY E06028JOBUy literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287104.sc9.1214828.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287104.sc9.1214828.2 new file mode 100644 index 0000000000000000000000000000000000000000..0ca8549a5c7422a938efc5c7a401bb715ef60040 GIT binary patch literal 195 zcmeZZfPjCKJmzv*DM?;nbiU~*#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZFhIF=~l(iE{CU zfQ;8qPtA+Z$uBMzWpKD|_c>uU#q6#(l)gH6YoHp^Vz~%}ZXVens66N9x z0U58Ko|+e*lV4mc%JAT(-NwW#GHOsw#r}^OIHw|MlHd~K;)iHT$t+HW>Qb0(!?C1m FBLLkuK*s<8 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287114.sc9.1214828.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287114.sc9.1214828.5 new file mode 100644 index 0000000000000000000000000000000000000000..5fa171c737d4735153ed441f406ff03b901f8faa GIT binary patch literal 195 zcmeZZfPjCKJmzw?@$b6C=zP;riZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vfz#OCsnAqulAUI=3LtfF<0OOqn0?AC>LJ{ z$awwq)V%nd{NiF!28TO#b&P+nszEgs`#)yj)Ney*lHd~K;)iHT$t+HW>6&Bn(<4b4 E0Ag%G!~g&Q literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287117.sc9.1214828.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287117.sc9.1214828.6 new file mode 100644 index 0000000000000000000000000000000000000000..e5bb49218c85a683caaff557644063895273f890 GIT binary patch literal 134 zcmeZZfPjCKJmzvrC%?E@l%e64-5I{gx7DDUiv1rmaIQttB*7)d#ShVxl3AP#)#b3pW_Q&I FSpX9DL5TnW literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287125.sc9.1214828.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287125.sc9.1214828.8 new file mode 100644 index 0000000000000000000000000000000000000000..69d7ac53809ab4743b9198489fce027b5d5d13c1 GIT binary patch literal 195 zcmeZZfPjCKJmzvdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWV@{6PF1M6ulAUI<_!9n3pQ8a6r+|nmnaus z2*`N-^whlgoc!WqQHF*mc4rd>eyTw=75hJC;GBh|NrFp^iyxvXC9^mgrfZE&!yJ`p E0D*QwEC2ui literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287128.sc9.1214828.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287128.sc9.1214828.9 new file mode 100644 index 0000000000000000000000000000000000000000..2b2e931bf0aa9039ed168c137dea2ab3cc911da7 GIT binary patch literal 134 zcmeZZfPjCKJmzxVHkG@^=zP;riZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-Vzf24M-FQ4%{xLyoV-iq!R892XbE!hb8&RPbWrr@JM FD*#FXKH>lX literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287139.sc9.1214828.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287139.sc9.1214828.12 new file mode 100644 index 0000000000000000000000000000000000000000..eac65453214c8cbdd8fb8d79705d0a85f0f1183d GIT binary patch literal 134 zcmeZZfPjCKJmzv*%3i<0=zP;riZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vfnf~UJh#R%{xLyoUdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vB+Id}T@`BXt376)IXkxA2b(K!j!{dTOO%T* z1Z2E^dTL&LPJVH*D8qq!c52$MwA7%Qiv1rmaDGD4B*7)d#ShVxl3AP#)wN)mjh!=p FF94doLMi|N literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287150.sc9.1214828.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287150.sc9.1214828.15 new file mode 100644 index 0000000000000000000000000000000000000000..6499b554d956ca88b7f5ec36e335f98c25d4f69a GIT binary patch literal 134 zcmeZZfPjCKJmzw4x-{W7qw`HiDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNMlDtjU3e6n|FkaI6sC>1e+_6rX|S5&&3gz Un4_PQUtBDDh`rl(Luj-O03cT_%>V!Z literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287154.sc9.1214828.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287154.sc9.1214828.16 new file mode 100644 index 0000000000000000000000000000000000000000..20dfadcad00ef1ca5a7e717c9fca559322716eb5 GIT binary patch literal 195 zcmeZZfPjCKJmzvLT-dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL}7CXn<~`YS9{DpbDAER2{u>Y0;85Vmnaus z2*`N-^whlgoc!WqQ3i+WcJo&ri&2AWD)xWOz}bqVNrFp^iyxvXC9^mgs*B;8O=zRa FVE_?!K>+{& literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287161.sc9.1214828.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287161.sc9.1214828.18 new file mode 100644 index 0000000000000000000000000000000000000000..ee1ab992546b7f5a4db5ac37db367e2de87a32e1 GIT binary patch literal 134 zcmeZZfPjCKJmzvfdhdOY(fOvM6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vBwc^TF*&HYH}41;arTw^g3T33*AnF7=i&%U U%+b%uFD@2c++Ajy(7NL$02_xcl>h($ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287165.sc9.1214828.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287165.sc9.1214828.19 new file mode 100644 index 0000000000000000000000000000000000000000..6b54f82e25dc017af74bc886f80a1ec100925a01 GIT binary patch literal 195 zcmeZZfPjCKJmzxp9?!kc=zP;riZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;yiVqvntfwS9{DpbIz{J2b(K!kx@&WOO%T* z1Z2E^dTL&LPJVH*D8qtVc6TOJ38_If75hJC;1sGtXp-O(NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z3EzL@kt)>OS9{DpbC&;ym@9CRQA?all#4F} zWW0WQYF>O!esQrVgTh@q1s2YWYEVtZ{*M_rA0TOx;1c8FhiFR4EKY{%N?^1-V&OCc E0P@p73;+NC literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287835.sc20.1362611.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287835.sc20.1362611.1 new file mode 100644 index 0000000000000000000000000000000000000000..aeb15a8513dfceaec153ef9801f8ce5aa12e8808 GIT binary patch literal 195 zcmeZZfPjCKJmzwqD|_3<rC%?E@l!4)iof$`#vKmxVvHxQR&H^M&5?o?j{18nknZ?OaT@To8CzRzr F2LPQRKM?=` literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287838.sc20.1362611.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287838.sc20.1362611.2 new file mode 100644 index 0000000000000000000000000000000000000000..6276772fc821fab9f08632a69755f46805c2301d GIT binary patch literal 134 zcmeZZfPjCKJmzwizn#&}NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZNi?)@RfU@SYLD4x&f9w-<_cV9)Dq_s<>Ctg z8Lyw7nirpwUtBE8&~V!>;O&l+YEVtZ{*M_r_4Xk&NpOjA@k2DFWELkwbsacv(|vEf F2>^>NL6ZOg literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287846.sc20.1362611.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287846.sc20.1362611.4 new file mode 100644 index 0000000000000000000000000000000000000000..8c721510c6562dd9d7635352fafd130185363dbe GIT binary patch literal 195 zcmeZZfPjCKJmzxpxJ~F}a=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z(abrlqY5?m)gH6YoB=x~g3T4U!l)(ACCbGY z0y17dJvA>rC%?E@l%e6FT|y7TQ8lQhV*kesoKKK6NpOjA@k2DFWELkwbv@W*Bj6&Y F2mmz%KK%dy literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287849.sc20.1362611.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287849.sc20.1362611.5 new file mode 100644 index 0000000000000000000000000000000000000000..d5c2374d48b0e198e5b1b811ceb3db54284c5068 GIT binary patch literal 134 zcmeZZfPjCKJmzxBR5*1pIp1`Y;!P?_%*@ksElbTSu`&wMfl#r}^OIKLxllHd~K;)iHT$t+HW>SEY#BX#rN FQ2^TzKt%um literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287857.sc20.1362611.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287857.sc20.1362611.7 new file mode 100644 index 0000000000000000000000000000000000000000..18306978f6086596fa352792c50f8f1462c31bf4 GIT binary patch literal 195 zcmeZZfPjCKJmzw8tO@I3a=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zk&d+BSB0AUYLD4x&Q8Gyu(<-)7`4Q?M7j7v zK*sB*r{=}yNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z36rh*CkHk6<{cp;&P_#ZU~>hsv;?{Mxj4cS UbM$lai;G3YW6NxPw^wZi0AU3yx&QzG literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287864.sc20.1362611.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287864.sc20.1362611.9 new file mode 100644 index 0000000000000000000000000000000000000000..5afaf507b060f7557b65859594b504430c116006 GIT binary patch literal 195 zcmeZZfPjCKJmzv9YE$lGa=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZSzj0`tO_;v)gH6YoHJQf!R88FXVens66N9x z0U58Ko|+e*lV4mc%FuAr?#aDX9%@ic#r}^OI0e`cnk2ZyxcDKOQZkE^p}G_{+Z;|T GUjqO=**?zz literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287867.sc20.1362611.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287867.sc20.1362611.10 new file mode 100644 index 0000000000000000000000000000000000000000..ba3227d221cf1389e0450c09d3c3dd225d5eef20 GIT binary patch literal 195 zcmeZZfPjCKJmzw0`Tp!NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z>51OIT@`BXt376)IVWy{m@9CDQA?all#4F} zWW0WQYF>O!esQrVL&7~fj_BX1YEVtZ{*M_rpCW0J;1c8FhiFR4EKY{%IdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vEw{=j0a`i!wa8WoPDUmZk>PRP6tlf%7$zCJ8PvE`Ernl+5B}sIG)nHl?{& F3IV6DLd^gG literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287878.sc20.1362611.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287878.sc20.1362611.13 new file mode 100644 index 0000000000000000000000000000000000000000..cb208c12418f4933f7097d5a9fcf1c53845e9a78 GIT binary patch literal 195 zcmeZZfPjCKJmzx#ZCpK($@!+E6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL@h2uP8DkIt376)IcMEj12$LS7NeFpmnaus z2*`N-^whlgoc!WqQ3i*bcJs}OZPlQfiv1rmaB|#5Xp-O(NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zxwda>nH<#In|FkaI8#4E%oWJd66E6N;s{I3 T(a*^*E*4dv-DNwm;p1umjWsRR literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287886.sc20.1362611.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287886.sc20.1362611.15 new file mode 100644 index 0000000000000000000000000000000000000000..31f0b913265f8e8ebfad0611e04cc7ddb7918808 GIT binary patch literal 195 zcmeZZfPjCKJmzxd_BBjqa=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZaY>QuP=%WNYLD4xPOICEU~>g-Gir%*iE{CU zfQ;8qPtA+Z$uBMzWk|ScH|K@&RW+!lV*kesoZFE!NpOjA@k2DFWELkwbv>AEGky0g FCjh%cL7D&n literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287889.sc20.1362611.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287889.sc20.1362611.16 new file mode 100644 index 0000000000000000000000000000000000000000..996675f5dd6b59c6b34a5a3cd38f375ee84d6042 GIT binary patch literal 195 zcmeZZfPjCKJmzvT*rZQka=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z$^G_sohsDaS9{Dpb8^hi0GlgthfzzMOO%T* z1Z2E^dTL&LPJVH*D8qxhcJn#ce^rBOD)xWOz_}DjlLVI-7e7Q(N@j5~R2Rd1o1JeS Gy#oNTOF~ot literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287892.sc20.1362611.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287892.sc20.1362611.17 new file mode 100644 index 0000000000000000000000000000000000000000..dcf0e8464a0ed69f63d085320d9241c989ebfe63 GIT binary patch literal 134 zcmeZZfPjCKJmzwqWDuLmNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Znb{murV2Ip)gH6YoXIT)U~>iTGHQu)iE{CU zfQ;8qPtA+Z$uBMzWms_CPHusjt{PNRvHxQRPVrWRCJ8PvE`Ernl+5B}s4j&?HZF@= G)&KyyUp=e< literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287900.sc20.1362611.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287900.sc20.1362611.19 new file mode 100644 index 0000000000000000000000000000000000000000..fc94aafbc72dfff745c70eac40a31c02f838cd25 GIT binary patch literal 195 zcmeZZfPjCKJmzvvc)~uN$@!+E6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vM6u|Tnkv-XS9{Dpb51nn1e+^xk5NmUOO%T* z1Z2E^dTL&LPJVH*C_}<+I~&^~Mm4CWV*kesoG+0yNpOjA@k2DFWELkwburAcX%x5o F3;-VWJ%IoK literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288568.sc10.1768728.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288568.sc10.1768728.0 new file mode 100644 index 0000000000000000000000000000000000000000..afcd98fafd9c80bd300a6e46c3f9c84d3b8a3777 GIT binary patch literal 134 zcmeZZfPjCKJmzx#U88Kr?0nNviZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^6rD_c{!-LH}41;aUK*>0h=q3rzOb6&&3gz Un4_PQUtBD@>12X!Ui#$k0KkDQ#Q*>R literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288573.sc10.1768728.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288573.sc10.1768728.1 new file mode 100644 index 0000000000000000000000000000000000000000..394f242887ef9e96c0166871e5b4cd5d60e09095 GIT binary patch literal 195 zcmeZZfPjCKJmzxFE%LQzcE0H-#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIi%4psR}js)gH6YoQ9tv<_g?r)Dq_s<>Ctg z8Lyw7nirpwUtBE8@Zg@^imIg9YEVtZ{*M_rrz2^S;1c8FhiFR4EKY{%Ixxwmn=xM& E08kP?QUCw| literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288576.sc10.1768728.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288576.sc10.1768728.2 new file mode 100644 index 0000000000000000000000000000000000000000..1cf8e78ed9807f8080cc222506ef4132db173b75 GIT binary patch literal 195 zcmeZZfPjCKJmzv%zEgK#cE0H-#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZarnK>MHOo9t376)Ic>u=!R87)VAK-l66N9x z0U58Ko|+e*lV4mc%CO+R-3r}=nQBl?#r}^OI2R&mlHd~K;)iHT$t+HW>N?P8Go^gC F2mnCJKG*;N literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288579.sc10.1768728.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288579.sc10.1768728.3 new file mode 100644 index 0000000000000000000000000000000000000000..8fc2e95b0ae548804cf9cb67968d0e3fd633d923 GIT binary patch literal 134 zcmeZZfPjCKJmzwGeY@|#?0nNviZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GFdKQz8uutn|FkaI1{}eg3T4k*AnF7=i&%U U%+b%uFD@1}yE(@;>dNA)0J;(_hyVZp literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288584.sc10.1768728.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288584.sc10.1768728.4 new file mode 100644 index 0000000000000000000000000000000000000000..4ffdba6683257dd2054c1462a2e41b6a95b34ca1 GIT binary patch literal 195 zcmeZZfPjCKJmzw~m*j9_cE0H-#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z>8q2jRfU@SYLD4x&LayU<_bJy)Dq_s<>Ctg z8Lyw7nirpwUtBE8kZ|8F&H8VN8dOuU|6>MD%S8xH5?o?j{18nknZ?OaT@Af9Yh5qO F0RZ8yJ}m$M literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288587.sc10.1768728.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288587.sc10.1768728.5 new file mode 100644 index 0000000000000000000000000000000000000000..d66db4a411b071af674763866ab716f867329cbf GIT binary patch literal 195 zcmeZZfPjCKJmzvf*!0+m+4-iU6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v}pU=#r}^OIIkgTlHd~K;)iHT$t+HW>Qb0svt;6$ Fd;sE;KhgjI literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288590.sc10.1768728.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288590.sc10.1768728.6 new file mode 100644 index 0000000000000000000000000000000000000000..bc35c64637a546709a6e3f4bb1f1c02cf201c8dc GIT binary patch literal 134 zcmeZZfPjCKJmzxpOrGYdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#ENZlvMSWvS9{DpbH4br8*Hw?V@54;E>SMN z5Rmcu>8W|~Ir+uKq6`PF+ui5er=bSbRP6tlfiw0qLX!lS7#BZ8Q%YuWGE~=tsWzWd HCzt~O|13em literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288598.sc10.1768728.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288598.sc10.1768728.8 new file mode 100644 index 0000000000000000000000000000000000000000..2127253200d422c8e685fab5edb54e64bdea756d GIT binary patch literal 195 zcmeZZfPjCKJmzxt$V_x)cE0H-#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZvG|v>LKSN6t376)Ijh?yfz1_o!l)(ACCbGY z0y17dJvA>rC%?E@lp*1UU7Jpdof=eAvHxQR&L2pcB)G)5_#v87GK-U;x*klou|K1` F2moAwKl}gy literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288601.sc10.1768728.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288601.sc10.1768728.9 new file mode 100644 index 0000000000000000000000000000000000000000..08cd800be07a93642147375af6806271bdc2d8a3 GIT binary patch literal 134 zcmeZZfPjCKJmzwGpLTL%cE0H-#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZSu^#Es~ptan|FkaIPWUEfXx*s)Dq<4=i&%U U%+b%uFD@2cy(+_Y@~@NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZStb5^vntfwS9{DpbL!rA2AeDJj8RLROO%T* z1Z2E^dTL&LPJVH*D1*a8J9YJ=&1z6h#r}^OI2RyklHd~K;)iHT$t+HW>6&M=@q2(c E0EZPnt^fc4 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288611.sc10.1768728.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288611.sc10.1768728.12 new file mode 100644 index 0000000000000000000000000000000000000000..7c4cb8cd6f797841869d497d404766beb99d63dc GIT binary patch literal 134 zcmeZZfPjCKJmzwKKl9Cl+4-iU6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWc6*OJ~^nlH}41;ac;2v4mMYyNK255pNk_b VF-JcqzqnXbeRhuRwynor0st;4FBt#; literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288616.sc10.1768728.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288616.sc10.1768728.13 new file mode 100644 index 0000000000000000000000000000000000000000..4e582ae425b2d21ce6501b0ad3d4f1b695099900 GIT binary patch literal 195 zcmeZZfPjCKJmzxps>*pWJKuDa;!P?_%*@ksElbTSu`&wMNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZS?e0fstPst)gH6YoOz3%gUuCq$*3jHCCbGY z0y17dJvA>rC%?E@lp*1^U3K0SeKn}2V*kesoX?RoNpOjA@k2DFWELkwbsbo2^CDtO F1ps7qK==Rv literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288631.sc10.1768728.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288631.sc10.1768728.17 new file mode 100644 index 0000000000000000000000000000000000000000..e88ae1137a1d846f1a4d32dc12882e63a26cc75f GIT binary patch literal 195 zcmeZZfPjCKJmzx#zjoS}+4-iU6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vSMN z5Rmcu>8W|~Ir+uKq6`W5>TG{czrhOtLt`t| literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289303.sc10.1793187.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289303.sc10.1793187.0 new file mode 100644 index 0000000000000000000000000000000000000000..d37aca89930d7ff95f069a75a02f379fd0a34e95 GIT binary patch literal 195 zcmeZZfPjCKJmzxVHa_*0+4-iU6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vu&**3+I GmJ0zkazrcu literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289305.sc10.1793187.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289305.sc10.1793187.1 new file mode 100644 index 0000000000000000000000000000000000000000..bdfc42161e5897e0e7265a18f71bff1e459e120b GIT binary patch literal 195 zcmeZZfPjCKJmzw$9dr4{?0nNviZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-lKRkA?BCCbGY z0y17dJvA>rC%?E@l)>SW-A3MTo@!7{#r}^OI7J)~nk2ZyxcDKOQZkE^p}G!CvtgKU G>jeOwBtTyP literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289307.sc10.1793187.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289307.sc10.1793187.2 new file mode 100644 index 0000000000000000000000000000000000000000..5a9da0c5ed56962b3d05251c60f2f84a8f78074d GIT binary patch literal 134 zcmeZZfPjCKJmzwiHNXAF?0nNviZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-Qodl$B{`_MH}41;aqjyBF;}2eOOT77iz6&C TM?WXOxL9;{PJ=DKc+Xk@Qgbg$ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289311.sc10.1793187.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289311.sc10.1793187.3 new file mode 100644 index 0000000000000000000000000000000000000000..27d5afc86bfafb752865ef449cb1c4b0cbfe9606 GIT binary patch literal 195 zcmeZZfPjCKJmzvncK`j(?0nNviZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@?&aPjVjdKS9{DpbB1jG2R2vWEu)q=mnaus z2*`N-^whlgoc!WqQ3i#pcD8A)Z`Gihiv1rmaGpTYB*7)d#ShVxl3AP#)wN)n&G-3< Fw*U%LMDPFr literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289315.sc10.1793187.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289315.sc10.1793187.4 new file mode 100644 index 0000000000000000000000000000000000000000..120cf572a2a203e3768915b44572dbf5baa22f9a GIT binary patch literal 195 zcmeZZfPjCKJmzvX&p7>q+4-iU6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vSMN z5Rmcu>8W|~Ir+uKq6`UF>_p!DTA&8iRP6tlfwKiklLVI-7e7Q(N@j5~R9C}f8(H<= Fo&e}ELt_8{ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289317.sc10.1793187.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289317.sc10.1793187.5 new file mode 100644 index 0000000000000000000000000000000000000000..e327c377886fbd90c911937147be311e5b448a94 GIT binary patch literal 134 zcmeZZfPjCKJmzxpH&y;*cE0H-#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIks@2rX1AVn|FkaI3u~L!R88-X$f-ib8&SCB~V{UyS F1OPj>L0td< literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289325.sc10.1793187.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289325.sc10.1793187.7 new file mode 100644 index 0000000000000000000000000000000000000000..4c5535f7ef7992bfb87465a29dd0b51d1df402af GIT binary patch literal 195 zcmeZZfPjCKJmzwmiYEVNcE0H-#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZS=7~Dp$awk)gH6YoaM7qz~%~kVAK-l66N9x z0U58Ko|+e*lV4mc%JAT_U1yg-i5gT>vHxQR&ihE3B)G)5_#v87GK-U8x~AL6e{QS> E0OztoE&u=k literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289328.sc10.1793187.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289328.sc10.1793187.8 new file mode 100644 index 0000000000000000000000000000000000000000..f4c90e0416c6d900087b5010d4c6e4ab51be55a4 GIT binary patch literal 134 zcmeZZfPjCKJmzvPy~z8A+4-iU6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vg!j4p6*;K6H}41;ahmG$gUuBv*AnF7=i&%U U%+b%uFD@1puq(0^+q&Nv0Kttb#Q*>R literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289332.sc10.1793187.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289332.sc10.1793187.9 new file mode 100644 index 0000000000000000000000000000000000000000..5ffb28357a421ef1c559e9ff510261973f8c5f9b GIT binary patch literal 195 zcmeZZfPjCKJmzw~{UrC7+4-iU6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq&GUwRuyXQt376)IcH@kfXx;7$fzaGCCbGY z0y17dJvA>rC%?E@l!4)j-KptIpQu4K75hJC;AF}~Xp-O(Qw+v$w5j0 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289335.sc10.1793187.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289335.sc10.1793187.10 new file mode 100644 index 0000000000000000000000000000000000000000..21bc67a97cf5efee275c94e3230251ac505875c6 GIT binary patch literal 195 zcmeZZfPjCKJmzxdT7Un`?0nNviZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vSibtI#sB-ulAUI=H!|618lCqCq^xCE>SMN z5Rmcu>8W|~Ir+uKq6`UF?YcMLl~99fD)xWOz}bqVNrFp^iyxvXC9^mgs*7Q^P0elh F=K$EoL$3e; literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289338.sc10.1793187.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289338.sc10.1793187.11 new file mode 100644 index 0000000000000000000000000000000000000000..3b3ee129e7abb08edea257902b7bea821bc89579 GIT binary patch literal 134 zcmeZZfPjCKJmzwWyk7E;+4-iU6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vr2ibt6gjB5H}41;asDx11~ylqLQ9Z~pNk_b UF-JcqzqnY`O}yFGxBIaN00S>A!~g&Q literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289342.sc10.1793187.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289342.sc10.1793187.12 new file mode 100644 index 0000000000000000000000000000000000000000..5016b39b649cd5eb5df8e4b7b36a85140c07c5c8 GIT binary patch literal 195 zcmeZZfPjCKJmzv58}0bd?0nNviZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vMev=fGX77S9{DpbM9{52{u>YGozL`mnaus z2*`N-^whlgoc!WqQ3i#pcKvU?`_!PCiv1rmaB8+7G)ZuYaq&YmrDPT-!*tECae1R7 F1OQN8L0

literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289346.sc10.1793187.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289346.sc10.1793187.13 new file mode 100644 index 0000000000000000000000000000000000000000..a1ac76ede28e697dd6bf3a611864751fca5a2f58 GIT binary patch literal 195 zcmeZZfPjCKJmzw$AD_s;;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vaP;orz+IkS9{Dpb50kZ3^rHb3!|1emnaus z2*`N-^whlgoc!WqQHBTC?fehQ%vFPGD)xWO!1)JBlLVI-7e7Q(N@j5~R9C`un}B_~ F!2nW|KNtW2 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289348.sc10.1793187.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289348.sc10.1793187.14 new file mode 100644 index 0000000000000000000000000000000000000000..42ed8e23ef774a0a0ade347bcbf57043255746b5 GIT binary patch literal 134 zcmeZZfPjCKJmzxVUuVO};(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-qCV{nyByTqn|FkaIIY?3z~%~6Y6)`jb8&NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zu~&=zE(bOD<{cp;&d!8;U~>hkv;?{Mxj4cS UbM$lai;G3|cXikNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZseZ4>qzX0n)gH6YoS{r#!R88lXVens66N9x z0U58Ko|+e*lV4mc%HVLtE|)PRUJa_L*#9vD=M5xH5?o?j{18nknZ?OaT?x}{wjbK~ F3;@GKKE?n5 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289367.sc10.1793187.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289367.sc10.1793187.19 new file mode 100644 index 0000000000000000000000000000000000000000..2812414ac9cd9a268768a4bba61147f6925d9189 GIT binary patch literal 195 zcmeZZfPjCKJmzwqusY4k;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-Qo_OfL=|f8t376)IVa3G12$LS2cwoamnaus z2*`N-^whlgoc!WqQHF-AcKM+XE7hQyiv1rmaO%xOXp-O(NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZF^T={BL_A2<{cp;&StMFu(<-&T7q2sTpVGE TIr=&I#l@oHv0b*QuI9l2RnIDJ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290122.sc11.2375140.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290122.sc11.2375140.0 new file mode 100644 index 0000000000000000000000000000000000000000..81b3e46fc284df33f4c5bb4d1d5e53f884b39fea GIT binary patch literal 134 zcmeZZfPjCKJmzvP4xVv{#rdYA6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#KfA}Kox54t376)IWKVU2b(MKlTk~YOO%T* z1Z2E^dTL&LPJVH*Civ1rma9Z&oG)ZuYaq&YmrDPT-!*tEEF&Ecd F3jm*4K3o6* literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290130.sc11.2375140.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290130.sc11.2375140.2 new file mode 100644 index 0000000000000000000000000000000000000000..7f47a46ddcd420a5df5dea9189a849aba31443d7 GIT binary patch literal 195 zcmeZZfPjCKJmzu=uAO^?#rdYA6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWL2q?u`1NuS9{Dpb8b?a2R2vW7o(Oqmnaus z2*`N-^whlgoc!WqQ3i)=c9Q$`9Mqtiiv1rmaQ;EkB*7)d#ShVxl3AP#)uk}Q#(~lI FDFC10KS=-p literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290133.sc11.2375140.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290133.sc11.2375140.3 new file mode 100644 index 0000000000000000000000000000000000000000..403c803f6d6ca08aef6d55506c7dcc265d99f142 GIT binary patch literal 134 zcmeZZfPjCKJmzvPKkI*##rdYA6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL^^2GEjg&UH}41;ac=bt0GlgNqb112&&3gz Un4_PQUtBEuu({dR^Xb1m0O~(3-2eap literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290137.sc11.2375140.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290137.sc11.2375140.4 new file mode 100644 index 0000000000000000000000000000000000000000..73664c2d83f9c369f7c70d105f312b281e48e0c5 GIT binary patch literal 195 zcmeZZfPjCKJmzxR{!2N=;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(v|G|P8DkIt376)IV18?!R8A5X4Der66N9x z0U58Ko|+e*lV4mc%CO*)UC_w|YHCnT#r}^OIKLxllHd~K;)iHT$t+HW>T;N5!}ak) F9{@3gLHz&# literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290140.sc11.2375140.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290140.sc11.2375140.5 new file mode 100644 index 0000000000000000000000000000000000000000..789e36cd10ce57826a3a48fd1f8b2355eeaeb102 GIT binary patch literal 195 zcmeZZfPjCKJmzvB#rdYA6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#3n)Dwkp)zS9{Dpb6zsE2AeDJhfzzMOO%T* z1Z2E^dTL&LPJVH*D8qxxb_-9igsDL_75hJC;M6ffXp-O(yr{ GgRKDm>p?UC literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290143.sc11.2375140.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290143.sc11.2375140.6 new file mode 100644 index 0000000000000000000000000000000000000000..61a95602321badad1c2dc9cd4e3b413c62c5e1a2 GIT binary patch literal 134 zcmeZZfPjCKJmzw`?0s{b#rdYA6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWa1XBGC8QZH}41;aaL%(1)D2St0lO!esQrV!-7k8+jAlps6jOq`#)yjHLltW7t376)Ip^O!esQrVgTpnu9iAeAYEVtZ{*M_r?RO$HNpOjA@k2DFWELkwbv@{_Nxf1& F5dhn{L9_q> literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290154.sc11.2375140.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290154.sc11.2375140.9 new file mode 100644 index 0000000000000000000000000000000000000000..755c3be9badefb0314d4cd2e1d4b4d862f085d38 GIT binary patch literal 134 zcmeZZfPjCKJmzv9Ql5W`#rdYA6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWJZ+uS~;k>H}41;aVD;Tm@81HCCJ6k#Sxa6 Tqo0#sTr8S8xz+Yk;WZZk%kC}E literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290158.sc11.2375140.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290158.sc11.2375140.10 new file mode 100644 index 0000000000000000000000000000000000000000..69efccb2c9df2e650211107a012938366b95e691 GIT binary patch literal 195 zcmeZZfPjCKJmzv5>1;jC;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-((s(~mnziUS9{Dpb8<*+1Dh-GpHWMkOO%T* z1Z2E^dTL&LPJVH*D1*XPyF;=i9coZb#r}^OINOmlNpOjA@k2DFWELmGbamN$KmBPQ E0H`oQJOBUy literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290162.sc11.2375140.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290162.sc11.2375140.11 new file mode 100644 index 0000000000000000000000000000000000000000..c70f29eaa15026a26215b0bd4ff343f8edd16072 GIT binary patch literal 195 zcmeZZfPjCKJmzvXyy!T?;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-5*rZ5r3y9o)gH6YoI2s1U~>f;n6$*XM7j7v zK*sB*r{=}yNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIj-N?EC)6B<{cp;&a<;2<_gqn33BmsafBu2 T=;!1Y7mLc*wAiKyx%vVCabhcu literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290169.sc11.2375140.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290169.sc11.2375140.13 new file mode 100644 index 0000000000000000000000000000000000000000..f4f1f838729c05108f12c713d33ccdbcf6d3479f GIT binary patch literal 195 zcmeZZfPjCKJmzvf;|My(;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-QeMZhUlnTZt376)Id@h-%oS*4(h}zq<>Ctg z8Lyw7nirpwUtBE8@ZhrD8NY9f)u5V+{U0-Ms#hX3NpOjA@k2DFWELkwbuo0?TowCd F0RYd!Kt=!n literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290172.sc11.2375140.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290172.sc11.2375140.14 new file mode 100644 index 0000000000000000000000000000000000000000..d70c3a6325adfdc9ae4e648cfb9de117f846f0f9 GIT binary patch literal 195 zcmeZZfPjCKJmzvP&Qd(j;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-veUCgKox54t376)IfWRMz~%}xF=>f&iE{CU zfQ;8qPtA+Z$uBMzWjJud?$F9_|J0zGiv1rmaE2ghlHd~K;)iHT$t+HW>Pl$0N#4lv F1^`NTKPUhI literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290175.sc11.2375140.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290175.sc11.2375140.15 new file mode 100644 index 0000000000000000000000000000000000000000..bdcc82f0afb10234771be9fef98d9fb6bae0dbec GIT binary patch literal 135 zcmeZZfPjCKJmzw`GhR8*;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a`adCDS4>5f!;0>oXxoqa|Ie0wFJ5Nxj4cS UbM$lai;G29ud1@;I4YD20OH^+EC2ui literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290179.sc11.2375140.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290179.sc11.2375140.16 new file mode 100644 index 0000000000000000000000000000000000000000..1b95dd91eebe4ab3fd03a084b3ed77b1bb33c967 GIT binary patch literal 195 zcmeZZfPjCKJmzv<$@+AG#rdYA6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq~y7tk1EvMS9{Dpb6UQIm@ClCq$SQJ%EcD~ zGG0GDH7`CVzqnYGVZkN4PyUPg)S#M*{U0-Mu0zr!!6nAU57Cs8S)2^jmC$NacQI%? E0ONK;rvLx| literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290183.sc11.2375140.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290183.sc11.2375140.17 new file mode 100644 index 0000000000000000000000000000000000000000..e54c00a3f9d194d75f552406312c8afd6a59bb09 GIT binary patch literal 195 zcmeZZfPjCKJmzwG1)aah;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a)C`#R26FOt376)IYnCtg z8Lyw7nirpwUtBE8u;8kl&X36bYEVtZ{*M_r8<8|gaEWp8Lo}si7AM1Wwb-n@pK1UA D%|k&u literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290185.sc11.2375140.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290185.sc11.2375140.18 new file mode 100644 index 0000000000000000000000000000000000000000..c28d16751b5f52dcdafed7e618e54d8ab6438941 GIT binary patch literal 135 zcmeZZfPjCKJmzw0Ev&i3;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@=9@At~}J-KyMcb&K1nHU~>f;8MOqt__;X3 V5_9x(@{5Z_1?*C6e@XTm006V1D@6bR literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290190.sc11.2375140.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290190.sc11.2375140.19 new file mode 100644 index 0000000000000000000000000000000000000000..4ff69f5c48a205d1e1c92e6ccb9965abedde9f0d GIT binary patch literal 195 zcmeZZfPjCKJmzxlW}A7L#rdYA6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vggIkchbq+ES9{DpbMC5vm@ClAq$SQJ%EcD~ zGG0GDH7`CVzqnYGA>p#!x2l2%YEVtZ{*M_rO=}UFB)G)5_#v87GK-U;x*oLINHHkP F0RZFlKo$T1 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290935.sc12.371669.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290935.sc12.371669.0 new file mode 100644 index 0000000000000000000000000000000000000000..99effbbd1071ed1e8cdf6c3fce1ff55f2c780efa GIT binary patch literal 195 zcmeZZfPjCKJmzvnx_szmb-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z;V|5ip$awk)gH6YoC$kAg3T3ZW6~1m66N9x z0U58Ko|+e*lV4mc%JAT--P)w@i`Af-iv1rmaBf4=B*7)d#ShVxl3AP#)751o5%TpI E0Q{#yUH||9 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290938.sc12.371669.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290938.sc12.371669.1 new file mode 100644 index 0000000000000000000000000000000000000000..f0d5612717277876ec4841d8e85529541d90b9b2 GIT binary patch literal 135 zcmeZZfPjCKJmzxRe4gLO>U`5tiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-k}$g=O&)4)ptp+zr(@Fsu(<+Fj9P+R{9GJi Vi8=Z?`NhSer_@qyTWu!E0076UD|G+> literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290942.sc12.371669.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290942.sc12.371669.2 new file mode 100644 index 0000000000000000000000000000000000000000..53c9f129ea5e768ee2b4d3d479160186a58b43c9 GIT binary patch literal 195 zcmeZZfPjCKJmzvvmD}0R>U`5tiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-5}uUwQ59T2k-d30lG FDFF03LBRk3 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290946.sc12.371669.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290946.sc12.371669.3 new file mode 100644 index 0000000000000000000000000000000000000000..45b79eb508f9135d19067bf179f1176c038163b0 GIT binary patch literal 195 zcmeZZfPjCKJmzxxdiPCWb-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZNveKvS`}*Ut376)IWxBQgUuD_VA2xj66N9x z0U58Ko|+e*lV4mc%JATlU5a_yBQ>a|V*kesoY#>wNpOjA@k2DFWELkwbv5+bB=+$p F0|4RcK&Su! literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290948.sc12.371669.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290948.sc12.371669.4 new file mode 100644 index 0000000000000000000000000000000000000000..b0b90478f230a2b5ee0c56b396672a1ba8df4770 GIT binary patch literal 135 zcmeZZfPjCKJmzwC^qWs)b-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZSy`X{O&)4)ptp+zXU2OAu(<-wj9P+R{9GJi Vi8=Z?`NhSe9}X7VdM}XD0|2;YEMEWs literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290953.sc12.371669.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290953.sc12.371669.5 new file mode 100644 index 0000000000000000000000000000000000000000..3d0f20ac95834b90e9764893a43469a9c9b5b1d7 GIT binary patch literal 195 zcmeZZfPjCKJmzvH^af30b-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZnOgOvP!($Kt376)IiEZZ2AeC;$)qLDCCbGY z0y17dJvA>rC%?E@ltJOD-Ry)m0X3+mV*kesoQY2mnk2ZyxcDKOQZkE^p}H8lZB8@B G8v+2h7e4d= literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290956.sc12.371669.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290956.sc12.371669.6 new file mode 100644 index 0000000000000000000000000000000000000000..c1375149dacf1dea52b5a2921c9680d5fe41e2b1 GIT binary patch literal 195 zcmeZZfPjCKJmzvLDws`Xb-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZVf~-Wq6#(l)gH6YoPjgV!R88dF=>f&iE{CU zfQ;8qPtA+Z$uBMzWpKD+cWP5gvl>)WvHxQR&do@gB)G)5_#v87GK-U;x*FPS_P_YI F767H8Kb8Oh literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290959.sc12.371669.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290959.sc12.371669.7 new file mode 100644 index 0000000000000000000000000000000000000000..9d98938a7f6682c1380e55ea2db158f6a1b9df02 GIT binary patch literal 135 zcmeZZfPjCKJmzxB{CF{$)%m8Q6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#QFR&9(ky_f!;0>oKiDhfz1_YVbl`j;^*QB VOU%*F$uBMzZQ-x7bv_;V8vy2>ERFyG literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290964.sc12.371669.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290964.sc12.371669.8 new file mode 100644 index 0000000000000000000000000000000000000000..b0242234a9d287e3a52f6f13e4474f39fdb10a66 GIT binary patch literal 195 zcmeZZfPjCKJmzxR>atH|b-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zsao%LN)>AEt376)IhB`kfXx-?X3`Sp66N9x z0U58Ko|+e*lV4mc%JAU2UE;|fkJX@>iv1rmaPCFYB*7)d#ShVxl3AP#)75GdcAj+; E03NPDAOHXW literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290967.sc12.371669.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290967.sc12.371669.9 new file mode 100644 index 0000000000000000000000000000000000000000..8de84be76b57b77d183f70c1419289312d88e9e4 GIT binary patch literal 195 zcmeZZfPjCKJmzvnp1w1c)%m8Q6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#6W3#kSf&NS9{Dpb4uCW1Dh+*!=xq7CCbGY z0y17dJvA>rC%?E@l!4)@U4CusO*N>dV*kesoEwldNpOjA@k2DFWELkwbscE3ITLN) F2>_%aKgIw6 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290970.sc12.371669.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290970.sc12.371669.10 new file mode 100644 index 0000000000000000000000000000000000000000..79a544dee3ab0373f43d720fa2061f8a3227b5cd GIT binary patch literal 135 zcmeZZfPjCKJmzw)J=s5v)%m8Q6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWSY6$40))zf!;0>oE$b2!R89IGHMBO@pEy6 VCFbbodVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vEfOl#*GT4Ar%u!{)2< GJaqsbv_TmF literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290978.sc12.371669.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290978.sc12.371669.12 new file mode 100644 index 0000000000000000000000000000000000000000..30cb0dd5723c81ea0bb9064fac9cbe06de51edf7 GIT binary patch literal 195 zcmeZZfPjCKJmzw`xwXzNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZdFNMarV2Ip)gH6YoNlYzz~&0{F=>f&iE{CU zfQ;8qPtA+Z$uBMzWq5GeuEXluPc^8fV*kesoF9=iNpOjA@k2DFWELkwbv1O^_(d%% F2LQ1WK>7dx literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290980.sc12.371669.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290980.sc12.371669.13 new file mode 100644 index 0000000000000000000000000000000000000000..1d99a3c38089876d888528dde2a62e44c06a0a89 GIT binary patch literal 135 zcmeZZfPjCKJmzv{95tWG>U`5tiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(&?I)AP+S+(A!0VvqaAlY_32Xqn02SKNm+> UVvc@JesQsA>f}J%Nv_QA0Dd?sr2qf` literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290985.sc12.371669.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290985.sc12.371669.14 new file mode 100644 index 0000000000000000000000000000000000000000..b60e3fd1ca4f71d0b94a54e661836ca4cd5870ad GIT binary patch literal 195 zcmeZZfPjCKJmzw$UJjYX>U`5tiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-G9x`wP!($Kt376)IeFc}z~&0{Giix)iE{CU zfQ;8qPtA+Z$uBMzWms^<&Ph)3yBbtevHxQR&Nd`X5?o?j{18nknZ?OaT?+j+o?U+e E0OD>vkN^Mx literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290988.sc12.371669.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290988.sc12.371669.15 new file mode 100644 index 0000000000000000000000000000000000000000..b940730e5e6ce5aee6e19801f2aa5db38136432c GIT binary patch literal 195 zcmeZZfPjCKJmzxlO_Z3;>U`5tiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(#d!7yDHS&S9{Dpa|&IM1e+@`fk{i8OO%T* z1Z2E^dTL&LPJVH*C_}>qyN6fVnbe?~iv1rmaLz;0B*7)d#ShVxl3AP#)74|cF2E`S E0B>eJy#N3J literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290991.sc12.371669.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290991.sc12.371669.16 new file mode 100644 index 0000000000000000000000000000000000000000..b1ede97241752cd640f19fd137aeffe48ae7b282 GIT binary patch literal 135 zcmeZZfPjCKJmzwK&O0@m)%m8Q6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vMCnvtkUZ4fKyMcbPHo0BU~>i98MOqt__;X3 V5_9x(@{5Z_53wiP=5f7O1pw%%D`fxx literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290995.sc12.371669.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290995.sc12.371669.17 new file mode 100644 index 0000000000000000000000000000000000000000..c443a23f1c850ef92c99e7c2eca58be4a17d8d3c GIT binary patch literal 195 zcmeZZfPjCKJmzxNt$#9y)%m8Q6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v*33N}|@B9oRlmnaus z2*`N-^whlgoc!WqQ3i#Jc0G4olhvS_iv1rmaK1y*B*7)d#ShVxl3AP#)75QL_2FA9 E06hOez5oCK literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290998.sc12.371669.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290998.sc12.371669.18 new file mode 100644 index 0000000000000000000000000000000000000000..09eeab9c8f3517eaafc24b33a3b788ddd51d6dec GIT binary patch literal 195 zcmeZZfPjCKJmzw?|6DVd)%m8Q6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWYc#3VpXWQulAUI=9KSW2R2t=5|frVmnaus z2*`N-^whlgoc!WqQHBK`CJ8PvE`Ernl+5B}s4j;tn-xbi GjspNxAwj?Z literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291826.sc16.2741235.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291826.sc16.2741235.0 new file mode 100644 index 0000000000000000000000000000000000000000..e76dffbdf5fc9cf29b99f61125592c1a2c27fe5a GIT binary patch literal 135 zcmeZZfPjCKJmzwCPoJH`=6uspiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^7hmDDtV~6f!;0>oEfv{fXx-?VAK-i;^*QB VOU%*F$uBMz4Gqn=-L<=W7XbClExG^z literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291831.sc16.2741235.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291831.sc16.2741235.1 new file mode 100644 index 0000000000000000000000000000000000000000..114eba52f50b46236f260aa10a6c282f7593ed23 GIT binary patch literal 195 zcmeZZfPjCKJmzvXmLJPybH3>)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZF?r0%q6#(l)gH6YoP5WQgUuC~%%mmGCCbGY z0y17dJvA>rC%?E@l;OZ7yZ26uiq)W+iv1rma5f`plHd~K;)iHT$t+HW>T;NC!zd-U F1OSo_KKTFu literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291834.sc16.2741235.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291834.sc16.2741235.2 new file mode 100644 index 0000000000000000000000000000000000000000..ca6638eb1115e5a658105b47e75010158692d3f5 GIT binary patch literal 195 zcmeZZfPjCKJmzu=IM2&tbH3>)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZNo@{#uL?Ey)gH6YoR3aI%oUizq$SQJ%EcD~ zGG0GDH7`CVzqnYG;lKsEX>;6OszEgs`#)yjR6B*xB*7)d#ShVxl3AP#)s=AFM)Su1 FB><=1L7V^p literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291837.sc16.2741235.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291837.sc16.2741235.3 new file mode 100644 index 0000000000000000000000000000000000000000..baa25fb3148fa84c08a46a590bcabd38e89e0a9d GIT binary patch literal 135 zcmeZZfPjCKJmzwyX$0i6Ip1`Y;!P?_%*@ksElbTSu`&wM UVvc@JesQtr?3@PMs26NU0WKpdiU0rr literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291841.sc16.2741235.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291841.sc16.2741235.4 new file mode 100644 index 0000000000000000000000000000000000000000..ddb5ba4c0c3f8def7abd62e10bc1520fac021a2b GIT binary patch literal 195 zcmeZZfPjCKJmzx#+gwt>=6uspiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-lE~S+LKSN6t376)Ii-2Zz~%}}WzrJo66N9x z0U58Ko|+e*lV4mc%CO+P-R6SlN7bO3iv1rma85$fB*7)d#ShVxl3AP#)zz@gCh5h? Fg#fK+K%@Ww literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291845.sc16.2741235.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291845.sc16.2741235.5 new file mode 100644 index 0000000000000000000000000000000000000000..46cf37a55c5cb379508ee6924060c767eeb364d5 GIT binary patch literal 195 zcmeZZfPjCKJmzw4SQlH!=6uspiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-QfejMstPst)gH6YoV@$vz~%}}W6~1m66N9x z0U58Ko|+e*lV4mc%5dPaUH;T;N9^FFUP F9sp`AKbQaj literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291848.sc16.2741235.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291848.sc16.2741235.6 new file mode 100644 index 0000000000000000000000000000000000000000..87fbcc2987f9e432fa92b5abe9efc1529cb9d42e GIT binary patch literal 135 zcmeZZfPjCKJmzxdu!|J2Ip1`Y;!P?_%*@ksElbTSu`&wM UVvc@JesQsAY)PkWS?0ED04EG8_W%F@ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291852.sc16.2741235.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291852.sc16.2741235.7 new file mode 100644 index 0000000000000000000000000000000000000000..988cd371b49bea6131e1467e7053639b529dcdd3 GIT binary patch literal 195 zcmeZZfPjCKJmzxV+-F_P=6uspiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-5}>zaohsDaS9{DpbKcgr0h=o@ok>faOO%T* z1Z2E^dTL&LPJVH*D8qxRcJH}0XQ@Fo75hJC;B?VJXp-O()#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z5$AQ9rwTRq)gH6YoJ*bA!R89gVA2xj66N9x z0U58Ko|+e*lV4mc%JAU2-R#v9m#RTE75hJC;8b)$Xp-O()#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZxwLNnM0u#Wf!;0>oU`UY%oXTn)Dq<4=i&%U U%+b%uFD@1h4K20RlULjU0I)wRp#T5? literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291863.sc16.2741235.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291863.sc16.2741235.10 new file mode 100644 index 0000000000000000000000000000000000000000..96f94139f07451040a1f19a0b366c9f784ffbdbc GIT binary patch literal 195 zcmeZZfPjCKJmzvfQh!v+=6uspiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;*wyzUlnTZt376)Ib*g!%oUi)q$SQJ%EcD~ zGG0GDH7`CVzqnYG!Qrx9cmu~YHK?Xy|HllR!dnrVB)G)5_#v87GK-U;x(;;Ow8!bJ F0RXl@KdAr! literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291867.sc16.2741235.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291867.sc16.2741235.11 new file mode 100644 index 0000000000000000000000000000000000000000..3dfcac1d40ff38abe01f0f2c68c608578cdb1359 GIT binary patch literal 195 zcmeZZfPjCKJmzvfO*~b`=6uspiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^7bg-4OOVQulAUI=6v_^G}v5$Sxj2uT%ufj zAt2-R(^K=}bMlLeMHw6}*`0iHO;8Q0so4K9184FpgeD0tF)n_Hrj*R$WT>u&Zkt7{ G1;zl;`9fd- literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291870.sc16.2741235.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291870.sc16.2741235.12 new file mode 100644 index 0000000000000000000000000000000000000000..37082ef3ec4e0539082dcc1a843646a6554f3787 GIT binary patch literal 135 zcmeZZfPjCKJmzv9+SFLi=6uspiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a#m1hnLO0oKyMcbPB-Reu(<*~j9P+R{9GJi Vi8=Z?`NhSer_?fSn?9IU0syo3D`x-z literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291874.sc16.2741235.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291874.sc16.2741235.13 new file mode 100644 index 0000000000000000000000000000000000000000..023c074fcab1a2c7e3675567f4175600054212bb GIT binary patch literal 195 zcmeZZfPjCKJmzw~__wTr&H1LI6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#CNjhS5>IFulAUI=6oN%0&K3pY$h#nE>SMN z5Rmcu>8W|~Ir+uKq6`Zz*!3RXA+841RP6tlfipbn$EH^= GC=dXLM?j+h literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291878.sc16.2741235.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291878.sc16.2741235.14 new file mode 100644 index 0000000000000000000000000000000000000000..dd422e2d054007f0a371367b7b0d8ffa7199d453 GIT binary patch literal 195 zcmeZZfPjCKJmzwqJTak?&H1LI6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#94o0q${W{KJ)+p literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291881.sc16.2741235.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291881.sc16.2741235.15 new file mode 100644 index 0000000000000000000000000000000000000000..ac1efa200b32b7b8d7fe55e867e4401dc5eef5e0 GIT binary patch literal 135 zcmeZZfPjCKJmzw)DfX&jbH3>)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZDQ>%HD-Sg{(A!0V^Wi0Lu(<-gj9P+R{9GJi Ui8=Z?`NhSeJ$iMvO5$tk0H}*AjQ{`u literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291885.sc16.2741235.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291885.sc16.2741235.16 new file mode 100644 index 0000000000000000000000000000000000000000..76a3c6857467e11259ce37ae0de12cf3f0c7e884 GIT binary patch literal 195 zcmeZZfPjCKJmzwCAIhs{bH3>)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZseY-oQWa|Mt376)IrX&)z~&0fWzrJo66N9x z0U58Ko|+e*lV4mc%8+o`?$6@S|J0zGiv1rma9%>vB*7)d#ShVxl3AP#)#cD;^N`b; F4*==dKl=ax literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291889.sc16.2741235.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291889.sc16.2741235.17 new file mode 100644 index 0000000000000000000000000000000000000000..c2faeeedf02b899e85f65ff2eb26a9bc9d89f5f7 GIT binary patch literal 195 zcmeZZfPjCKJmzxFTJKTA=6uspiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-q8@SOw<^@!S9{Dpb4Idxfz1_|$D}3BCCbGY z0y17dJvA>rC%?E@l)>S$-Q2#`TWU~E#r}^OICmmxlHd~K;)iHT$t+HW>QZR1$v78~ F2LO57KHmTU literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291891.sc16.2741235.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291891.sc16.2741235.18 new file mode 100644 index 0000000000000000000000000000000000000000..b6c2bac84732f0e31c3f573ba5acdaab01922932 GIT binary patch literal 135 zcmeZZfPjCKJmzwG#{92gbH3>)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZkzC7aB@Z<>(A!0VbILb}xdMHRT7q2sTpVGE UIr=&I#l@n#UNqQ#^Hl2t0Id})&;S4c literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291896.sc16.2741235.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291896.sc16.2741235.19 new file mode 100644 index 0000000000000000000000000000000000000000..ce8cc39eced2835fe4771be1e58538d164712d59 GIT binary patch literal 195 zcmeZZfPjCKJmzvXYs%NLIp1`Y;!P?_%*@ksElbTSu`&wMNW0WcfRQ;#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zkz?7vS`}*Ut376)IaO3lz~&0fXVMbq66N9x z0U58Ko|+e*lV4mc%D`~NuFdWE12w3oV*kesoc&0eB)G)5_#v87GK-U;x(-y@_;It9 F000;wJu3hJ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292975.sc9.1353776.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292975.sc9.1353776.1 new file mode 100644 index 0000000000000000000000000000000000000000..d41e9c4f0cf76d992cb2a8f5a826e4afa3879bb6 GIT binary patch literal 195 zcmeZZfPjCKJmzv%SfA=NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zv7VpVqY5?m)gH6YoO{=w2AeCefJsZ7OO%T* z1Z2E^dTL&LPJVH*C_}?JyNgGCFRDQ`75hJC;7rEfOl#*GT4AWI(lUn4t F1OOCKL302A literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292977.sc9.1353776.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292977.sc9.1353776.2 new file mode 100644 index 0000000000000000000000000000000000000000..c8ff3ce347193011e9606811d113164155d9eef3 GIT binary patch literal 135 zcmeZZfPjCKJmzw4*;?Df?tIfxiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-5)^&+i#*iaKyMcb&L8acU~>if8MOqt__;X3 V5_9x(@{5Z_7kBsAK3Sgq6#)DZE;9fC literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292982.sc9.1353776.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292982.sc9.1353776.3 new file mode 100644 index 0000000000000000000000000000000000000000..bc60d89ab1323645ff40dc54b8608d53680ace89 GIT binary patch literal 195 zcmeZZfPjCKJmzxB3C-(ecfRQ;#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z`S)N}t}4{rS9{Dpb4par2b(LfkV#9NOO%T* z1Z2E^dTL&LPJVH*D8qwGb~lXA?Nx(nD)xWOz*&Z*NrFp^iyxvXC9^mgs*9n~=2nP` FD*(14Kr8?N literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292986.sc9.1353776.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292986.sc9.1353776.4 new file mode 100644 index 0000000000000000000000000000000000000000..1e2ab76130621d4ac6b1d1b91f9a261694e0d998 GIT binary patch literal 195 zcmeZZfPjCKJmzwKi*D&-cfRQ;#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZId1lEvMSWvS9{DpbJ{7ifz1_I#H1z8CCbGY z0y17dJvA>rC%?E@lwrXYy9-HT@7181iv1rmaPC3UB*7)d#ShVxl3AP#)x}U{^OU)| F6aWTYKm-5) literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292988.sc9.1353776.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292988.sc9.1353776.5 new file mode 100644 index 0000000000000000000000000000000000000000..dfb085c1f62d670d6a9648f06ceb08b0871385be GIT binary patch literal 135 zcmeZZfPjCKJmzw88=3dBJKuDa;!P?_%*@ksElbTSu`&wMNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZdCc?7O%-bHt376)IoZUbz~%}pX3`Sp66N9x z0U58Ko|+e*lV4mc%Aj!BZoO*yV>PIzV*kesoOMW=B)G)5_#v87GK-U;x*Do&0w*5j F0s!S^J)!^r literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292996.sc9.1353776.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292996.sc9.1353776.7 new file mode 100644 index 0000000000000000000000000000000000000000..19ec8f7f0b776571572fc35fcc89f91ed98aded8 GIT binary patch literal 195 zcmeZZfPjCKJmzvbUeTY(?tIfxiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@-FiEb5*FhulAUI<}9-^1e+_cgh@-BOO%T* z1Z2E^dTL&LPJVH*D8qpZc5$&k=c_?A75hJC;Jk*UNrFp^iyxvXC9^mgs_Q|MjnT~$ F^8o8aL7V^p literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292999.sc9.1353776.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292999.sc9.1353776.8 new file mode 100644 index 0000000000000000000000000000000000000000..6a87dc4577b0692f462ceefc0e213ee972ad2719 GIT binary patch literal 135 zcmeZZfPjCKJmzxV_q;ce-T9`Y6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq`p@wP#$V-ptp+zXWE(jU~>f~GHMBO@pEy6 VCFbbodVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vSMN z5Rmcu>8W|~Ir+uKq6`j~><+&dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v0JN(t8vpSMN z5Rmcu>8W|~Ir+uKq6`X`>|}35epZ8OD)xWO!0Ez*&?Lbn#>EfOl#*GT4Aqs;VsqBp G&mI8%bU%Xt literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293017.sc9.1353776.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293017.sc9.1353776.13 new file mode 100644 index 0000000000000000000000000000000000000000..80d27cadc74ccb73be46a7a9cd091c8d993d6bde GIT binary patch literal 195 zcmeZZfPjCKJmzxt?X8){?tIfxiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(*5|2hbq+ES9{DpbJmE|fz1_I!K5Y5CCbGY z0y17dJvA>rC%?E@l;OY~I}JmbS87mA#r}^OIG-bFlHd~K;)iHT$t+HW>QY!}!`2sA F1pxgaKaBtY literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293020.sc9.1353776.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293020.sc9.1353776.14 new file mode 100644 index 0000000000000000000000000000000000000000..f4a855278f10f70075530351662f9fe8c0d8f73f GIT binary patch literal 135 zcmeZZfPjCKJmzwKlGU5e?tIfxiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-qM>NRArCb-(A!0V^GdNk*j$0hj9P+R{9GJi Vi8=Z?`NhSeJ$ebY&lo-l0RTdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWX0sm^{P;FU+ppb%qbWj1U6S-C6ksomnaus z2*`N-^whlgoc!WqQHBRs>>7ol;?SEYvvtv2q F9su)+KfM3| literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293028.sc9.1353776.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293028.sc9.1353776.16 new file mode 100644 index 0000000000000000000000000000000000000000..d145a85c8bc3c1df7672b2018f2c958cd5155f12 GIT binary patch literal 195 zcmeZZfPjCKJmzw8tx%iE?tIfxiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-BK<-Bo+{MbS9{Dpb2eINfXx+H#iS+9CCbGY z0y17dJvA>rC%?E@ltJOPUH-%LEH$X6V*kesoXnO8O%hyUT>KDCDVfE|P+bj+ZP-Fq G+XDdM={+|9 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293031.sc9.1353776.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293031.sc9.1353776.17 new file mode 100644 index 0000000000000000000000000000000000000000..7f3c830f8ecd3a918c1dd5711d94dc0ffc0e3a29 GIT binary patch literal 135 zcmeZZfPjCKJmzw4dvtpyyYo#)Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Ti1eaa4)RcQ1HD}&IA4X|1)D1{g;7h8i=T@l WEHOtvC%?E@bk~a{+Z}zomI44Us4koU literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293035.sc9.1353776.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293035.sc9.1353776.18 new file mode 100644 index 0000000000000000000000000000000000000000..626b42460e453d28bd06df770b2ca02eedfe9ac9 GIT binary patch literal 195 zcmeZZfPjCKJmzxRuJ}2N-T9`Y6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v^IqKyMcb&Ut~HU~>hgGHMBO@pEy6 VCFbboNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZiJ8L`uL?Ey)gH6YoIAPKDCDVfE|P+bcq+q8LU G=K=r?I6bcb literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293876.sc14.2649097.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293876.sc14.2649097.3 new file mode 100644 index 0000000000000000000000000000000000000000..8b9aa1fb7693f0be76ac861767153a5e38875617 GIT binary patch literal 195 zcmeZZfPjCKJmzvTatP&eINx-X;!P?_%*@ksElbTSu`&wMi5F=>f&iE{CU zfQ;8qPtA+Z$uBMzWoWoy_cS2pp&C?EvHxQRPM&gvCJ8PvE`Ernl+5B}sICW-Y&=t+ G{09I>Q9elk literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293879.sc14.2649097.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293879.sc14.2649097.4 new file mode 100644 index 0000000000000000000000000000000000000000..9d2dc5839a08ed53dde321f4d87feefcbf5874c9 GIT binary patch literal 135 zcmeZZfPjCKJmzxxW}eLDaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZdHzA}raaW#KyMcb&P`V!<_b(>)Dq<4=i&%U U%+b%uFD@1>x!7&1q`z||0O6l4mH+?% literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293883.sc14.2649097.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293883.sc14.2649097.5 new file mode 100644 index 0000000000000000000000000000000000000000..7f82409cb263be0f3da1db99ea4ab04c330d1f45 GIT binary patch literal 195 zcmeZZfPjCKJmzx#x$!!W!}+G86mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v;m8n5B75hJC;QWH5NrFp^iyxvXC9^mgs>@-b&C5r1 F_W=7cL<0Z- literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293887.sc14.2649097.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293887.sc14.2649097.6 new file mode 100644 index 0000000000000000000000000000000000000000..4d214fe6b5be39450e600f97da6052c229865c70 GIT binary patch literal 195 zcmeZZfPjCKJmzvPV!o8m;e69kiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-Qg-0(1y!iIulAUI=KNK78Eme=1|}_WE>SMN z5Rmcu>8W|~Ir+uKq6`gJ?82Te%vXbID)xWOz&Wu9p-F;EjEf(lDJ8Qw8LEq+$A+&_ Ggb@H!zCd39 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293890.sc14.2649097.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293890.sc14.2649097.7 new file mode 100644 index 0000000000000000000000000000000000000000..0069594ea466982b0ba7beb3d97e2b9f8d1854eb GIT binary patch literal 135 zcmeZZfPjCKJmzxdzw9mGaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z34iCuBM&t<(A!0V^F?|e*j$0>j9P+R{9GJi Ui8=Z?`NhSelYN_POZ05o0m#TJrT_o{ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293894.sc14.2649097.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293894.sc14.2649097.8 new file mode 100644 index 0000000000000000000000000000000000000000..86b3627175486bf95c56fc736d21eec4b98cbdbd GIT binary patch literal 195 zcmeZZfPjCKJmzxVu3uZo;e69kiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;<9&hhbq+ES9{Dpb6P)J2R2t=Ba@ammnaus z2*`N-^whlgoc!WqQ3i+WcDm2DN2x(I75hJC;CzClNrFp^iyxvXC9^mgs;i;f=G&#S Ff&f@ILY@Es literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293897.sc14.2649097.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293897.sc14.2649097.9 new file mode 100644 index 0000000000000000000000000000000000000000..d36e1a17aadb92dcda079a0f9d8934d3bf44c2a3 GIT binary patch literal 195 zcmeZZfPjCKJmzxlU0GMe;e69kiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vV2E~zbe$+S9{Dpb9ys0fXx-y#H1z8CCbGY z0y17dJvA>rC%?E@lwrXYJB#DabJU=kiv1rma0)UaG)ZuYaq&YmrDPT-Lv=ND*f1ts GHwOTKFFrW{ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293900.sc14.2649097.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293900.sc14.2649097.10 new file mode 100644 index 0000000000000000000000000000000000000000..6495ba6e3800cc9d916543e763f082eabf75c6b8 GIT binary patch literal 135 zcmeZZfPjCKJmzxBzBDN2aK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZsXzCAy*$+1KyMcb&h1-`z~&0fVAK-i;^*QB VOU%*F$uBMz?a|A#oqH{!8vw9tEW!W) literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293905.sc14.2649097.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293905.sc14.2649097.11 new file mode 100644 index 0000000000000000000000000000000000000000..3940546c0458db72602003e7bd3692bb6b9a9819 GIT binary patch literal 195 zcmeZZfPjCKJmzwW_=T5nINx-X;!P?_%*@ksElbTSu`&wMEnk2ZyxcDKOQZkE^VY+&3cD{Jl F1OUv5KWYE~ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293908.sc14.2649097.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293908.sc14.2649097.12 new file mode 100644 index 0000000000000000000000000000000000000000..3c68c1853494ea73ce4d35cd6e7187b4bc13a3cc GIT binary patch literal 195 zcmeZZfPjCKJmzvfw6rPZaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z@%Z*EQx$6Nt376)IloP|1)D3dg-J`COO%T* z1Z2E^dTL&LPJVH*D1*WUJKd(=qH0i0#r}^OI4hNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZshTqJjy%-dKyMcb&dqZ^gUuC~$*3jB#m~hN VmYAcTlV4mcIyNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZxwZUHlPc8QS9{DpbH2YW3N}|@E0dNumnaus z2*`N-^whlgoc!WqQHBMV>>jOn`c@69so4K9182+)geD0tF)n_Hrj*R$WSFi=Hm8N& GO#%Q>HA43Q literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293919.sc14.2649097.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293919.sc14.2649097.15 new file mode 100644 index 0000000000000000000000000000000000000000..2209c712db15ee5f0e5a5bb2c4f3e595fb7e7a72 GIT binary patch literal 195 zcmeZZfPjCKJmzxNFZof<;e69kiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;=*>mRTXOPt376)IS;-52{udVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq;j{9g*?>UKyMcb&NZCtz~&0fV$>4k;^*QB VOU%*F$uBMzRiB+=J8A8W1OUd@EHwZC literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293927.sc14.2649097.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293927.sc14.2649097.17 new file mode 100644 index 0000000000000000000000000000000000000000..371eee16d317cfef37b10833cbc1217ceefc61e1 GIT binary patch literal 195 zcmeZZfPjCKJmzw4=s#7-;e69kiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^8fYO2vw-LulAUI=G0?34K`O`JCl|;mnaus z2*`N-^whlgoc!WqQHF%GcKpw|WYnOViv1rma4tmBB*7)d#ShVxl3AP#)upi4=HQ#X FJOC8BKy3g3 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293930.sc14.2649097.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293930.sc14.2649097.18 new file mode 100644 index 0000000000000000000000000000000000000000..55ded12df7ca590140cb283eae6a110fb0a350c1 GIT binary patch literal 195 zcmeZZfPjCKJmzxFs9Ijd;e69kiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-ayv@gS`}*Ut376)Im>^p0GlhYgGo!AOO%T* z1Z2E^dTL&LPJVH*C_};(yVc9|_p3oQ75hJC;JkyRNrFp^iyxvXC9^mgs_VgKoA+Oq Glmh?{h(js> literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294796.sc14.2671192.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294796.sc14.2671192.0 new file mode 100644 index 0000000000000000000000000000000000000000..60beba1a7b0ad839a2b26e30dc51a24c00c70958 GIT binary patch literal 135 zcmeZZfPjCKJmzv%m)Hk$I^T4Z;!P?_%*@ksElbTSu`&wMrC%?E@l!4)vomZMxs~S{OvHxQR&eU{-CJ8PvE`Ernl+5B}sIG=8n|m9! GPXqw16Fz$Y literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294807.sc14.2671192.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294807.sc14.2671192.3 new file mode 100644 index 0000000000000000000000000000000000000000..67b9a072cd9983132a727e56080bbd999ee5a08d GIT binary patch literal 135 zcmeZZfPjCKJmzw~fB7kt)A^>O6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vMCbGNpYl+11HD}&I4{ro3^rF_4x^SJ7e5zA VSYnQTPJVH*Xpdf#tvNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZDG^zJR~2gRt376)Iav=%g3T4!&7>vHCCbGY z0y17dJvA>rC%?E@l%e6Oou#)^z8X|hvHxQR&hKDCDVfE|P+bfSHgnqg FG6B;tJ(>Uj literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294818.sc14.2671192.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294818.sc14.2671192.6 new file mode 100644 index 0000000000000000000000000000000000000000..60fbabc9e8dc9135b7363fb588acb9582b802332 GIT binary patch literal 135 zcmeZZfPjCKJmzxdSni16biU~*#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z;d=3Ir##f$KyMcb&Zkp$g3T3}%cv#D#m~hN VmYAcTlV4mcT5>VhmeEagIRLdZEHeNA literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294823.sc14.2671192.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294823.sc14.2671192.7 new file mode 100644 index 0000000000000000000000000000000000000000..8ae2075e0190446f3e1fa1b1b0bcc60bb2ae4124 GIT binary patch literal 195 zcmeZZfPjCKJmzu=J3o%(biU~*#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z@k+Yes0uar)gH6YoGtsFfXx-y%cLdFCCbGY z0y17dJvA>rC%?E@lp*1g9V@R(z8X|hvHxQRPV@Z;O%hyUT>KDCDVfE|P+bg-Hf!aS GCj$UbdOkM* literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294827.sc14.2671192.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294827.sc14.2671192.8 new file mode 100644 index 0000000000000000000000000000000000000000..4696dfb8aacd003b64cff2208a5e4ffd23bdc066 GIT binary patch literal 195 zcmeZZfPjCKJmzvL-#8P+>3q{siZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-BGc^tSQTpSt376)IawLbfz1`z$D}3BCCbGY z0y17dJvA>rC%?E@l)>Se9ZP89LN%zSV*kesoTW&bB)G)5_#v87GK-U;x)_>l`ehoM E0U`Q5NB{r; literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294830.sc14.2671192.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294830.sc14.2671192.9 new file mode 100644 index 0000000000000000000000000000000000000000..6e687df00f72eb2b373699bf91255a0b7a700f17 GIT binary patch literal 135 zcmeZZfPjCKJmzwK`_dTA>3q{siZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-V*0gekv!DgKyMcbP9~RTu(<;B7_|hs__;X3 V5_9x(@{5Z_V@uL)Z>3q{siZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!ZxRJyDHS&S9{DpbN>0Y3~a8zekLt(E>SMN z5Rmcu>8W|~Ir+uKq6`nN*zMp*S5<>*D)xWO!0GxMp-F;EjEf(lDJ8Qw8LF$H)F$+j G&IABDWI;3l literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294838.sc14.2671192.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294838.sc14.2671192.11 new file mode 100644 index 0000000000000000000000000000000000000000..6c630cbef5ed4bceb99dfb16c1c6fd0c1e386a20 GIT binary patch literal 195 zcmeZZfPjCKJmzv{Uu%lxbiU~*#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIrh|4RuyXQt376)Il~iLz~%}ZVA2xj66N9x z0U58Ko|+e*lV4mc%D`~bPM{@unHp46vHxQR&SOZLB)G)5_#v87GK-U;x(+njB$)a; F004MBK8*kX literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294840.sc14.2671192.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294840.sc14.2671192.12 new file mode 100644 index 0000000000000000000000000000000000000000..2e74a67b779c7b234132cbf388af180cad36b5f5 GIT binary patch literal 135 zcmeZZfPjCKJmzvf6*rCJbiU~*#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z;qv^xQyyw=ptp+zXTSr9xdQVUwFJ5Nxj4cS UbM$lai;G29ud1_MGDW-`0CAoxmH+?% literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294845.sc14.2671192.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294845.sc14.2671192.13 new file mode 100644 index 0000000000000000000000000000000000000000..70eee611f0a384662cc5ddbf2c09916e65ceec06 GIT binary patch literal 195 zcmeZZfPjCKJmzv{UJi}tbiU~*#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z`Ep#VQWa|Mt376)IlD~4!R87aWYQAn66N9x z0U58Ko|+e*lV4mc%D`~ZZr2T`WHqR!V*kesoV=z8O%hyUT>KDCDVfE|P+bq2Y)rb2 GgaQC;XFl@) literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294848.sc14.2671192.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294848.sc14.2671192.14 new file mode 100644 index 0000000000000000000000000000000000000000..49ced80eb27b174543cde6f1ce8c588f92f5b4e4 GIT binary patch literal 195 zcmeZZfPjCKJmzw$-!V?$biU~*#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z*{>ddL=|f8t376)IbW%ng3T2;#H1z8CCbGY z0y17dJvA>rC%?E@l%e68onzlz2{ovuV*kesocZbqO%hyUT>KDCDVfE|P+bg7Hp@Am GF9QG+tUdw& literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294851.sc14.2671192.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294851.sc14.2671192.15 new file mode 100644 index 0000000000000000000000000000000000000000..4ea3341d2f9123a9bf372335cc093cb96ecaef2b GIT binary patch literal 135 zcmeZZfPjCKJmzv%c|J?vbiU~*#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z3Efk&LLO>vptp+zr|Rf)Flq^M@pEy6 VCFbbo3q{siZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-QrCIrw<^@!S9{DpbG}Vx2b(K!m`O{VOO%T* z1Z2E^dTL&LPJVH*CNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zv6K#uSB0AUYLD4xPR8$#!R87aVbT)k66N9x z0U58Ko|+e*lV4mc%HVL>E=Fa~IyI=KV*kesoI8*-NpOjA@k2DFWELkwbv5MK2<_k4 F1OQz~Kmq^& literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294862.sc14.2671192.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294862.sc14.2671192.18 new file mode 100644 index 0000000000000000000000000000000000000000..ff747579f29594b79bc5e8016783f7d339133b0d GIT binary patch literal 135 zcmeZZfPjCKJmzw)VV#o9>3q{siZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!o$1$t~}J-KyMcb&c0TNxdICrwFJ5Nxj4cS UbM$lai;G2bH#ORF9W2}h0CJftp8x;= literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294866.sc14.2671192.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294866.sc14.2671192.19 new file mode 100644 index 0000000000000000000000000000000000000000..efb95a3d32a39fd833ff9a81708d5dd92b078a0c GIT binary patch literal 195 zcmeZZfPjCKJmzwytlyBr>3q{siZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(h{kgp$awk)gH6YoUGqBg3T2;%A_UECCbGY z0y17dJvA>rC%?E@l;OY)yEJRVYBi{)V*kesoS8_PB)G)5_#v87GK-U;x)}0o;@1eW F0sxjaK9c|d literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295534.sc14.2696291.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295534.sc14.2696291.0 new file mode 100644 index 0000000000000000000000000000000000000000..57df03f7e946aa8a65ad6e2864535ce4dce5c535 GIT binary patch literal 195 zcmeZZfPjCKJmzw~Q0U<1a=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zv32<{OBHJFt376)IqgqF%oRAsq$SQJ%EcD~ zGG0GDH7`CVzqnYG;lT|%3z^Gj)S#M*{U0-MENT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZDPxe!m4})e=rC%?E@l;Od3yNNpj>eQf`iv1rmaO$r>Xp-O(gdFlmW%iE{CU zfQ;8qPtA+Z$uBMzWl*?iw`+@UgBny*vHxQR&Pzy|B)G)5_#v87GK-U;x*Do%1UG6P F1pswIJzoF- literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295547.sc14.2696291.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295547.sc14.2696291.4 new file mode 100644 index 0000000000000000000000000000000000000000..ea6bd2535c757e4f343c21ec2179cc58252a72fc GIT binary patch literal 135 zcmeZZfPjCKJmzvXZoJON<$TjoiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-BKOVfhdk8WKyMcb&UC?>U~>f)GinKP@pEy6 VCFbboYB$Jjnmnaus z2*`N-^whlgoc!WqQ3i%z#UDcqPiv1rmaPl-GG)ZuYaq&YmrDPT-Lv=kUw&9dH G6Ab{+0zWeV literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295555.sc14.2696291.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295555.sc14.2696291.6 new file mode 100644 index 0000000000000000000000000000000000000000..42964cb3ab0793c97e268af0e0cfffcd90f1cff6 GIT binary patch literal 195 zcmeZZfPjCKJmzxFWxFE4<$TjoiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!q+BtQWa|Mt376)IYUyeg3T2;#iS+9CCbGY z0y17dJvA>rC%?E@lwrXoyXRWxPpCmP75hJC;9QENNrFp^iyxvXC9^mgrYqm(lzE&s E05j-6MF0Q* literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295558.sc14.2696291.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295558.sc14.2696291.7 new file mode 100644 index 0000000000000000000000000000000000000000..fcd2523ae8ac7ae3da709c89b872d539756017ff GIT binary patch literal 135 zcmeZZfPjCKJmzxR+qMgGIp1`Y;!P?_%*@ksElbTSu`&wM(A!0VbNwNRxdKZVwFJ5Nxj4cS UbM$lai;G2nJgBt2+wHUj06q~cGynhq literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295562.sc14.2696291.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295562.sc14.2696291.8 new file mode 100644 index 0000000000000000000000000000000000000000..ecded2ae747e60d44ac599a5a6b22984df0ae74a GIT binary patch literal 195 zcmeZZfPjCKJmzv9|F=?z%lW3G6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq`ft3y(-k)S9{DpbBgw_0-Gyvnn_EXOO%T* z1Z2E^dTL&LPJVH*D1*XvyF?E21U0CpV*kesoM(_UNpOjA@k2DFWELkwbvcyT{9@35 F1OTNeKH~rY literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295565.sc14.2696291.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295565.sc14.2696291.9 new file mode 100644 index 0000000000000000000000000000000000000000..e16d2df4816fee0f6160d396600523d2c98f55fe GIT binary patch literal 195 zcmeZZfPjCKJmzv5Yu5;KIp1`Y;!P?_%*@ksElbTSu`&wMh|FlmW%iE{CU zfQ;8qPtA+Z$uBMzWpKD|_j{L@pc+(DvHxQR&iP21B)G)5_#v87GK-U;x)`c$B2LzC F0{}#RJh1=( literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295568.sc14.2696291.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295568.sc14.2696291.10 new file mode 100644 index 0000000000000000000000000000000000000000..74ec7b990641a0b42d124698f3dbaaec1bc9c870 GIT binary patch literal 135 zcmeZZfPjCKJmzvXi|UJTIp1`Y;!P?_%*@ksElbTSu`&wMh|GHHo(iE{CU zfQ;8qPtA+Z$uBMzWoS5W*O1HhSPiPF*#9vD=PM*l5?o?j{18nknZ?OaT@EcaN|Blt E0C?3rivR!s literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295576.sc14.2696291.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295576.sc14.2696291.12 new file mode 100644 index 0000000000000000000000000000000000000000..2a142b54484f85829ffef32becda3a93f4062b12 GIT binary patch literal 195 zcmeZZfPjCKJmzvf`C%-^<$TjoiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-B3c%cs0uar)gH6YoTh|F=>f&iE{CU zfQ;8qPtA+Z$uBMzWpKD;7e7CvM-8f}*#9vD=Po2o5?o?j{18nknZ?OaT?d+MKA*U* F0RRZRKG6UG literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295579.sc14.2696291.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295579.sc14.2696291.13 new file mode 100644 index 0000000000000000000000000000000000000000..78096f77f5816060991e5b1e5de4b5ce7bbbe908 GIT binary patch literal 135 zcmeZZfPjCKJmzvvEqWow<$TjoiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(zvYcqCC{xKyMcb&g~B%<_auh)Dq<4=i&%U U%+b%uFD@1}yBTLYaXYgV0NTMUn*aa+ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295584.sc14.2696291.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295584.sc14.2696291.14 new file mode 100644 index 0000000000000000000000000000000000000000..f3b092e7a47acfbedaf63258c05b1b14cb61d174 GIT binary patch literal 195 zcmeZZfPjCKJmzvT|a=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z`Msp+wkp)zS9{Dpb3T0tF<0O`la@G_C>LJ{ z$awwq)V%nd{NiF!h6k7IEWb5>RD)_N_J7R4nfM5yNrFp^iyxvXC9^mgs!O5C=B4ew F3;@*>K|cTh literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295587.sc14.2696291.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295587.sc14.2696291.15 new file mode 100644 index 0000000000000000000000000000000000000000..f7cfb0f4f0989f797875792bcde013f94b249e2c GIT binary patch literal 195 zcmeZZfPjCKJmzw$ay*mZa=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZG3fZBt_n5x)gH6YoWYZygUuDVz@#P4CCbGY z0y17dJvA>rC%?E@lp*1gT|uSs95tw>V*kesoX3$gNpOjA@k2DFWELmGbXD8<$8A^y E00VwM82|tP literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295590.sc14.2696291.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295590.sc14.2696291.16 new file mode 100644 index 0000000000000000000000000000000000000000..4cc1f39fbb4793bbef5db9c3d717f2dbc5799dce GIT binary patch literal 135 zcmeZZfPjCKJmzxVpFUHP%lW3G6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWQOaN1bL{rf!;0>oD&#kgUuCK&Zs5G#m~hN VmYAcTlV4mcIy)!B_91_>J^-xFD?k7M literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295594.sc14.2696291.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295594.sc14.2696291.17 new file mode 100644 index 0000000000000000000000000000000000000000..2b57a5a569cd5b27fb9eade43ad899d5c1fe94cb GIT binary patch literal 195 zcmeZZfPjCKJmzv*sqd8Ha=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZkqhH&R)w1TYLD4x&i`w6fz1`T$fPCCCCbGY z0y17dJvA>rC%?E@lp*1go%jdQg=$bu#r}^OI47({Xp-O(EfOl#*GT4AsR@WwVvN GzzP7*fj^=E literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295601.sc14.2696291.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295601.sc14.2696291.19 new file mode 100644 index 0000000000000000000000000000000000000000..2ee15b81428fe65c4d5b53de880759261fa99ff6 GIT binary patch literal 135 zcmeZZfPjCKJmzxtaC*ydIp1`Y;!P?_%*@ksElbTSu`&wMrC%?E@l;OcSJGL!lacWRa#r}^OIPW28lHd~K;)iHT$t+HW>T;;Fne@KP F0RX;(KzjfH literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296482.sc14.2714192.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296482.sc14.2714192.1 new file mode 100644 index 0000000000000000000000000000000000000000..a98a26608951ae08b95d0846a4c3cca4a43ae40e GIT binary patch literal 195 zcmeZZfPjCKJmzwC*iQMu<$TjoiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GP_OVqbk(gS9{Dpb2c5E3N}~Z3X_&Nmnaus z2*`N-^whlgoc!WqQHBHO?4(ajDN=)KD)xWOz$tJDp-F;EjEf(lDJ8Qw8LBIx!KU_4 G?L+`|v_iQ6 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296485.sc14.2714192.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296485.sc14.2714192.2 new file mode 100644 index 0000000000000000000000000000000000000000..2cd2cfc66097545e3862ef1c8d0fdbcf1611be3d GIT binary patch literal 135 zcmeZZfPjCKJmzu=FY^4!<$TjoiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-BG#P6Cl56@(A!0V^DwtJ*j#~?j9P+R{9GJi Vi8=Z?`NhSe9P(+lFE;O)4ghzXD@gzV literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296489.sc14.2714192.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296489.sc14.2714192.3 new file mode 100644 index 0000000000000000000000000000000000000000..d410692bf96a1a5b3f65eaaa61f4aed2a51ae27f GIT binary patch literal 195 zcmeZZfPjCKJmzw0br$^Ma=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z37yz~P!($Kt376)IVW%zg3T4U%A_UECCbGY z0y17dJvA>rC%?E@lp*1?-F4j>4mGHzV*kesod1zDNpOjA@k2DFWELkwbuDPJnfZQs FIsoAqL0kX; literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296493.sc14.2714192.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296493.sc14.2714192.4 new file mode 100644 index 0000000000000000000000000000000000000000..50e7ca521ebf29b6e3b260cd32069630e464aff4 GIT binary patch literal 195 zcmeZZfPjCKJmzxVQ;Po0<$TjoiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GIjk-K2@l>ulAUI=6o?N25hdtH6|@_E>SMN z5Rmcu>8W|~Ir+uKq6`UV?R3`N^HzguD)xWOz!@+dp-F;EjEf(lDJ8Qw8LDeRiw)18 GMn3>XUO}V) literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296496.sc14.2714192.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296496.sc14.2714192.5 new file mode 100644 index 0000000000000000000000000000000000000000..691c2d5d0f48b759f8ca57b20533cb935ad44c5e GIT binary patch literal 135 zcmeZZfPjCKJmzxBZsGsK<$TjoiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@^Pt^t~}J-KyMcb&dJ<@U~>glF=`2N@pEy6 VCFbbodVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWU`Wvsw&jnS9{Dpb2_rC%?E@l;OcSyVKk4PpCmP75hJC;9P;ENrFp^iyxvXC9^mgs;i;hrt;(F F^8gbZLLC4A literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296504.sc14.2714192.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296504.sc14.2714192.7 new file mode 100644 index 0000000000000000000000000000000000000000..36bb8537083d431db625982b90e56b829712becc GIT binary patch literal 195 zcmeZZfPjCKJmzvb%P9Tha=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZiTLn-l`7QSS9{Dpb3Q0g2AeBzgGo!AOO%T* z1Z2E^dTL&LPJVH*D8qp>b`sU8wrWsK#r}^OI4cVgnk2ZyxcDKOQZkE^p}H74ZEANh G_yGXV^FP!8 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296507.sc14.2714192.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296507.sc14.2714192.8 new file mode 100644 index 0000000000000000000000000000000000000000..b790e251d720d0b081b58d55a32da34ed9b6ca95 GIT binary patch literal 135 zcmeZZfPjCKJmzwmuDtw@%lW3G6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL`c@xS`}*Ut376)IX|5G0XA3QCX<#pmnaus z2*`N-^whlgoc!WqQHF$bb`yRt+o1;4RP6tlfivNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zx$e62u{_k=KyMcb&a{^ha|PBgY6)`jb8&dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v)WvHxQRPF53yCJ8PvE`Ernl+5B}sICR&HietR Fc>#ozJe2?d literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296528.sc14.2714192.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296528.sc14.2714192.14 new file mode 100644 index 0000000000000000000000000000000000000000..f24e19c207573ae60f1e39207bb8b935f810c537 GIT binary patch literal 135 zcmeZZfPjCKJmzw`*Qv2^JKuDa;!P?_%*@ksElbTSu`&wMNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZiMVBXMipxAt376)IX}(t1)D2ymq|;UOO%T* z1Z2E^dTL&LPJVH*CNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZQ9u5PM-^)Bt376)Ijvqn%oVuDq$SQJ%EcD~ zGG0GDH7`CVzqnYGA>oppPFT%PHK?Xy|HllRGm$h&aEWp8Lo}si7AHe>F=W~->N~$4 E0Q@CB)&Kwi literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296539.sc14.2714192.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296539.sc14.2714192.17 new file mode 100644 index 0000000000000000000000000000000000000000..f29744a2bac8196a8c2499e79aa9b23b038ebfe6 GIT binary patch literal 135 zcmeZZfPjCKJmzxlJ@A-~+xe!W6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?voUOc1!R89AW7HDl;^*QB VOU%*F$uBMzUA?NxHpko54gk&3E0_QP literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296544.sc14.2714192.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296544.sc14.2714192.18 new file mode 100644 index 0000000000000000000000000000000000000000..92be09368e75b38c1993390da1dfd49394c1cbd1 GIT binary patch literal 195 zcmeZZfPjCKJmzwSbF*-8JKuDa;!P?_%*@ksElbTSu`&wMdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWcw_?8LCioU+ppb%qg%7Vy?ggCM|I;Q7*m^ zkn#HIsd@1^`NhSe3=b~b#b1#vQG;qK_J7R4S%RcVf=i5xAEGHGvp5;5DdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vB$mVZk}A~PS9{DpbH0o|3N}~ZA(NIkmnaus z2*`N-^whlgoc!WqQHBGT?J5`Q>`{YiD)xWOz!@Hc&?Lbn#>EfOl#*GT4Ate3X!9ei GfD-^(C_!8R literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297326.sc14.2733087.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297326.sc14.2733087.3 new file mode 100644 index 0000000000000000000000000000000000000000..69bf7635fb0c2b5ad4acbd0d84d107f2249bf7a7 GIT binary patch literal 195 zcmeZZfPjCKJmzxFsat)T+xe!W6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWXmn{{i;xNU+ppb%*i0R25hdtBPK0zE>SMN z5Rmcu>8W|~Ir+uKq6`j~>{M&srKmwQ75hJC;PgY%B*7)d#ShVxl3AP#)um8s!`F7f F9snWhKmPy# literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297329.sc14.2733087.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297329.sc14.2733087.4 new file mode 100644 index 0000000000000000000000000000000000000000..6f8366a1400201d267618ae169b5ef2bd3ae4b83 GIT binary patch literal 135 zcmeZZfPjCKJmzwSypB7=?R?WwiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GHdPERq{}C1HD}&IL(|Az~%~UVAK-i;^*QB VOU%*F$uBMz<&aOXO_Opv1_1VVE5iT) literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297334.sc14.2733087.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297334.sc14.2733087.5 new file mode 100644 index 0000000000000000000000000000000000000000..15cf8b288185b1e42b253140a60b2b9b7b4f7f6e GIT binary patch literal 195 zcmeZZfPjCKJmzw;-)uh1?R?WwiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-B7F0aw<^@!S9{DpbKXg41)D4Im`O{VOO%T* z1Z2E^dTL&LPJVH*D8qwGcFwUAgVdm!iv1rmaJDBRG)ZuYaq&YmrDPT-LvvRd G-vR&})IcQw literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297340.sc14.2733087.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297340.sc14.2733087.7 new file mode 100644 index 0000000000000000000000000000000000000000..2271279208cecb823dfbdc7157f04d34de5060dc GIT binary patch literal 135 zcmeZZfPjCKJmzv<7gjsZ?R?WwiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-VkA{`Ode`(ptp+z=YwcQow30|1M)D^36a literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297345.sc14.2733087.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297345.sc14.2733087.8 new file mode 100644 index 0000000000000000000000000000000000000000..78874e99b8d43b833e2dc5e21b532ef680224af2 GIT binary patch literal 195 zcmeZZfPjCKJmzxl6Lq`5?R?WwiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vgyR-?W$07U+ppb%qddq4mMZdDU+5smnaus z2*`N-^whlgoc!WqQ3i%9cE(zIwrWsK#r}^OI9ri4NpOjA@k2DFWELkwbuGxXxt|f} F3;?VNKX?ED literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297348.sc14.2733087.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297348.sc14.2733087.9 new file mode 100644 index 0000000000000000000000000000000000000000..6beee5fff1bc4b36346f4be39a5195e98f3e16d9 GIT binary patch literal 195 zcmeZZfPjCKJmzvTi11(JcE0H-#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZkvB4BRE3)RYLD4x&W}j~U~>hYF=>f&iE{CU zfQ;8qPtA+Z$uBMzWl*?ar+kvlT@9+K*#9vDXJ#@&lLVI-7e7Q(N@j5~RM&%Ko5Lxq G{r~_ooJWp8%oW(gs3pk7&&3gz Un4_PQUtBCYJEy@`uVCMD00RRqQUCw| literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297355.sc14.2733087.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297355.sc14.2733087.11 new file mode 100644 index 0000000000000000000000000000000000000000..b40649ebd339f1260db1558661e69329a9c6125c GIT binary patch literal 195 zcmeZZfPjCKJmzv*&cAz!+xe!W6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#6D1rC%?E@l)>S;U0FPfsTx#MvHxQR&VD3K5?o?j{18nknZ?O4UG+9nV#@CU Dv|&FF literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297358.sc14.2733087.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297358.sc14.2733087.12 new file mode 100644 index 0000000000000000000000000000000000000000..b653a4299abf518298130aaee349f8c116613a91 GIT binary patch literal 195 zcmeZZfPjCKJmzwqcinuM+xe!W6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWY^(Cj;c^|U+ppb%z0+(7O=SjFPOB%xkS16 zLO{mrr>Ew{=j0a`i!wA^v@?0vE~Ez4RP6tlfzx;!LX!lS7#BZ8Q%YuWGE7&aO@{NU FL;%tOLg4@a literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297361.sc14.2733087.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297361.sc14.2733087.13 new file mode 100644 index 0000000000000000000000000000000000000000..c4962654df85cf1c34771e932016f87db5c10ea8 GIT binary patch literal 135 zcmeZZfPjCKJmzxx-b}v2?R?WwiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!u_ezNgir$ptp+zXJkSu*j$0lj9P+R{9GJi Vi8=Z?`NhSep`o$1K4;hT0|3VDEFu5^ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297366.sc14.2733087.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297366.sc14.2733087.14 new file mode 100644 index 0000000000000000000000000000000000000000..9fc9746d7222bd1be22d644e768e99aca98ea737 GIT binary patch literal 195 zcmeZZfPjCKJmzxpKW@6p?R?WwiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!tu)Qk1EvMS9{DpbG{912AeDJl1WRPOO%T* z1Z2E^dTL&LPJVH*CNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZF{(~eRE3)RYLD4xPWJ_VU~>gtF=>f&iE{CU zfQ;8qPtA+Z$uBMzWjJubPJhCEH8rTFV*kesoU4&ENpOjA@k2DFWELmGbXC}F?BAXT E06SMecmMzZ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297372.sc14.2733087.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297372.sc14.2733087.16 new file mode 100644 index 0000000000000000000000000000000000000000..930287729966809c8aa455beae2f280099429107 GIT binary patch literal 135 zcmeZZfPjCKJmzxB$S_~$cE0H-#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z;X2(GCl56@(A!0V^XOEFxdK}lwFJ5Nxj4cS UbM$lai;G1C>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vgvo2Nm@3rVS9{DpbNZNRg3T3p&7>vHCCbGY z0y17dJvA>rC%?E@lwrXIyQ?+FeAJ+tiv1rma6U)UB*7)d#ShVxl3AP#)zwgDlNFPv F3ILL_KWP8} literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297380.sc14.2733087.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297380.sc14.2733087.18 new file mode 100644 index 0000000000000000000000000000000000000000..75e8b879cb72ce29983d603071b1a3296b460a2b GIT binary patch literal 195 zcmeZZfPjCKJmzu+y%)O4?R?WwiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-Qm1-GMipxAt376)Ip1W9fXx+n!=xq7CCbGY z0y17dJvA>rC%?E@ltJOLolf~O9W|(?V*kesoZdMIO%hyUT>KDCDVfE|P+br5Y;4lI G{{sM$lRwq~ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297383.sc14.2733087.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297383.sc14.2733087.19 new file mode 100644 index 0000000000000000000000000000000000000000..dcd2258e5dbf35f063eaa0129e8cc1a5364ddcfa GIT binary patch literal 135 zcmeZZfPjCKJmzw;UORb{+xe!W6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vgm+C3k37`eKyMcb&ht}Ffz1`z%BUsC#m~hN VmYAcTlV4mcdP=R?c3-s-I{@eSEPDU| literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298153.sc9.1458633.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298153.sc9.1458633.0 new file mode 100644 index 0000000000000000000000000000000000000000..af676e43a007b7ea365eb85b7c327c156ce0e39a GIT binary patch literal 135 zcmeZZfPjCKJmzwqvJ0HW<9yRmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-()!nUvpm$?KyMcbPT8vva|O0CY6)`jb8&Nd>galYv&#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZVHOowt_n5x)gH6YoX@5}%oTXcq$SQJ%EcD~ zGG0GDH7`CVzqnYGf#JHHB5%btHK?Xy|HllRK2s5zB)G)5_#v87GK-U;x*UpaE-ttF F4FJlWKD__{ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298161.sc9.1458633.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298161.sc9.1458633.2 new file mode 100644 index 0000000000000000000000000000000000000000..7a051472f85276a64acf108cb4443b01615d8ebe GIT binary patch literal 195 zcmeZZfPjCKJmzxNvHQ>AalYv&#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZS)??5l`7QSS9{Dpa~=;10GliDj!8?LOO%T* z1Z2E^dTL&LPJVH*C_}>)JLgcx`D##2#r}^OI1R%Qnk2ZyxcDKOQZkE^VY;$xe)MYG F0suZfKZ*bV literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298164.sc9.1458633.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298164.sc9.1458633.3 new file mode 100644 index 0000000000000000000000000000000000000000..cc3d84a7c11d2269525c25d01dcc4c90b8e76ec6 GIT binary patch literal 135 zcmeZZfPjCKJmzv*onV;D<9yRmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vgM=US$U|rf!;0>oQJ{~!R88VW7HDl;^*QB VOU%*F$uBMzot+b7`|c^rY5O!esQrVgTe(nzmF}~)S#M*{U0-MirqnIlHd~K;)iHT$t+HW>UxlAlXBs| FAOOiwK{@~c literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298172.sc9.1458633.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298172.sc9.1458633.5 new file mode 100644 index 0000000000000000000000000000000000000000..38c7b38e08d3140df774fdd671e7c6b317a561e1 GIT binary patch literal 195 zcmeZZfPjCKJmzxVkz}9G<9yRmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^7i28X{u0jU+ppb%&FVX0XA3Q1Cy3Gmnaus z2*`N-^whlgoc!WqQHBE-?G#F`&r^eHD)xWOz&Q~~lLVI-7e7Q(N@j5~RM&$Pn~Ucz GYy<${c0x%2 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298174.sc9.1458633.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298174.sc9.1458633.6 new file mode 100644 index 0000000000000000000000000000000000000000..2d8f5956ebb0d9f9b1f745e2b2f5c35d6b8d4d9f GIT binary patch literal 135 zcmeZZfPjCKJmzxFR^L6J$N8qC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vMDS23w>;F`KyMcbPS318U~>hwGinKP@pEy6 VCFbbodVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWM1``C8|(!U+ppb%qh6^A=q4jk4#$PT%ufj zAt2-R(^K=}bMlLeMHvoUvvU#2=~shlD)xWOz}btWNrFp^iyxvXC9^mgs>>nW=8b;F F9RRNdLYM#m literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298183.sc9.1458633.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298183.sc9.1458633.8 new file mode 100644 index 0000000000000000000000000000000000000000..463eb640fd0919b12ea13020f5a66c4180cbd89a GIT binary patch literal 195 zcmeZZfPjCKJmzwG3msa><9yRmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-BA*jqqY5?m)gH6YoHjEc<_dgb(h}zq<>Ctg z8Lyw7nirpwUtBE8&~Vw#v0?rzHK?Xy|HllRlaMq?aEWp8Lo}si7AHe>IV9RdzHr+Q E0GWM2(EtDd literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298185.sc9.1458633.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298185.sc9.1458633.9 new file mode 100644 index 0000000000000000000000000000000000000000..349db945d421866636f972602e11c1c20e69cd16 GIT binary patch literal 135 zcmeZZfPjCKJmzv*-6>hb<9yRmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-QZ0V7S{`a{ptp+zr+Zx)*j#}fj9P+R{9GJi Ui8=Z?`NhSehuA}H>utE10JWPcoB#j- literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298190.sc9.1458633.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298190.sc9.1458633.10 new file mode 100644 index 0000000000000000000000000000000000000000..7ee8630c53f2b7164e5c71df5535f1fafaf4126a GIT binary patch literal 195 zcmeZZfPjCKJmzwq)t|eV$N8qC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?va)U+ppb%*niN9@t!g&rDk4T%ufj zAt2-R(^K=}bMlLeMHvoUu(Me5!&D8bso4K91LtNWO%hyUT>KDCDVfE|P+baXHjk54 GegFVs>q1ii literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298193.sc9.1458633.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298193.sc9.1458633.11 new file mode 100644 index 0000000000000000000000000000000000000000..b40790fc1be8e6429132e1c5a468f764afba5ad1 GIT binary patch literal 195 zcmeZZfPjCKJmzv9e_g(W$N8qC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vMDarCaaE|fulAUI<~-_D2{u>Y3zL>Omnaus z2*`N-^whlgoc!WqQHBGT?V9bqs@0&Hiv1rmaPm1LG)ZuYaq&YmrDPT-LvNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZF>&5{NqlD)xWOz40Bto13 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298205.sc9.1458633.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298205.sc9.1458633.14 new file mode 100644 index 0000000000000000000000000000000000000000..ae366902375e96d957f108a5c7388c5de4f466c7 GIT binary patch literal 195 zcmeZZfPjCKJmzxhXuB-ualYv&#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIhs*4OBHJFt376)IXTz5g3T59#-t_ACCbGY z0y17dJvA>rC%?E@l)>SW-HmC&)77Awiv1rma7H3&lHd~K;)iHT$t+HW>RQlh^JBY+ FDgZk=Kd}G+ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298207.sc9.1458633.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298207.sc9.1458633.15 new file mode 100644 index 0000000000000000000000000000000000000000..050ac2ff6fa26464048cc3044e8934ad6790f511 GIT binary patch literal 135 zcmeZZfPjCKJmzxxmi=AM<9yRmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-B3)O-E)O*~(A!0V^U}nBU~>g_F=`2N@pEy6 VCFbbofaOO%T* z1Z2E^dTL&LPJVH*CR6 F6#$@aKx6;_ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298218.sc9.1458633.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298218.sc9.1458633.18 new file mode 100644 index 0000000000000000000000000000000000000000..c9516e3f2ff78c38e7ce6970b7f0ec7658976ef4 GIT binary patch literal 135 zcmeZZfPjCKJmzxho!`ES$N8qC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWa+urqViC41HD}&IE{mMg3T4!&8Q{F#m~hN VmYAcTlV4mc8e39rn`+1$0s!kQE4=^! literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298994.sc14.2774085.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298994.sc14.2774085.0 new file mode 100644 index 0000000000000000000000000000000000000000..cb6dd8356a47f032bae8dab2d31fc9fd63927fe0 GIT binary patch literal 195 zcmeZZfPjCKJmzxxl+MiIb-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zv7Y$#mnziUS9{DpbKY4x3v8~yPbMvKE>SMN z5Rmcu>8W|~Ir+uKq6`U_?H)0Gw^oB{D)xWOz?r%Xp-F;EjEf(lDJ8Qw8Kx`O=I9lv Fs{k+NLMs3O literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298996.sc14.2774085.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298996.sc14.2774085.1 new file mode 100644 index 0000000000000000000000000000000000000000..d733b1643666a8b914a5bbc8a75827c9d56a76bd GIT binary patch literal 195 zcmeZZfPjCKJmzu+ZcxnSb-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZS@F*Ck1EvMS9{DpbG~v_0-G!Fi%CnIOO%T* z1Z2E^dTL&LPJVH*Co<@6@20iv1rmaK^hKG)ZuYaq&YmrDPT-Lv=ai*!;R= Gegy!v*Fe<( literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298999.sc14.2774085.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298999.sc14.2774085.2 new file mode 100644 index 0000000000000000000000000000000000000000..73a7b86729f63e33e85bcb526b8a13e16b36c745 GIT binary patch literal 135 zcmeZZfPjCKJmzwyXNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZF|B1gA`dk;(A!0Vb5{2?u(<+z7_|hs__;X3 V5_9x(@{5Z_SFg&mz1MVA5&)8tER_HN literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299003.sc14.2774085.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299003.sc14.2774085.3 new file mode 100644 index 0000000000000000000000000000000000000000..298059ecf2c8e29ee1a97e8f70e4dad070e1d9b7 GIT binary patch literal 195 zcmeZZfPjCKJmzxVwMEuiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!t?&p3RS4NulAUI=471;F<0O>la@G_C>LJ{ z$awwq)V%nd{NiF!28By@dwMEuiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^6cofTve#KulAUI=3F0t4Q#H!A0{nvE>SMN z5Rmcu>8W|~Ir+uKq6`g}?KHNDO;m$wD)xWOz^R^q&?Lbn#>EfOl#*GT4ApfY(MBZE G@CX2q9zkXR literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299010.sc14.2774085.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299010.sc14.2774085.5 new file mode 100644 index 0000000000000000000000000000000000000000..8d2b7fcc8a505c9eceb3d564cc4b4b2b57d96b0b GIT binary patch literal 135 zcmeZZfPjCKJmzwqeb`;V>wMEuiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a{g{}xIEO{KyMcb&VH9(u(<+z8MOqt__;X3 V5_9x(@{5Z_7k78r&gZ}03;-9tEsX#G literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299014.sc14.2774085.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299014.sc14.2774085.6 new file mode 100644 index 0000000000000000000000000000000000000000..ba5920d36f09ec92f53ff8c66e4c56216e9682c4 GIT binary patch literal 195 zcmeZZfPjCKJmzwKjb2yC>wMEuiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-q8_AvQx$6Nt376)IXCTJ4>nieFO!xymnaus z2*`N-^whlgoc!WqQHF%8cF&~O+o(Y`75hJC;0!u|&?Lbn#>EfOl#*GT4Ar$D)`oM{ Gp>F^mdqJ=O literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299018.sc14.2774085.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299018.sc14.2774085.7 new file mode 100644 index 0000000000000000000000000000000000000000..59aded474cc54941cd69eb72e7e72e7ec5af2aea GIT binary patch literal 195 zcmeZZfPjCKJmzwK`P5#->wMEuiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vSa6JepRTsulAUI<}6F^1e+`Hk4a0MOO%T* z1Z2E^dTL&LPJVH*C_}?#JCT0hy=qWR#r}^OI4>Y+lHd~K;)iHT$t+HW>N*f+6EszG F9sqK7K~?|& literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299020.sc14.2774085.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299020.sc14.2774085.8 new file mode 100644 index 0000000000000000000000000000000000000000..2b992c93b3cc6bed5734981f91d5de58e66b4539 GIT binary patch literal 135 zcmeZZfPjCKJmzwKa<(nzb-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIl?12Q66e;ptp+zr?Idd*j#~qj9P+R{9GJi Vi8=Z?`NhSe`n!s3PtKV=1psJ~D+~Yt literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299025.sc14.2774085.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299025.sc14.2774085.9 new file mode 100644 index 0000000000000000000000000000000000000000..cb5f669235406ebd729ae074775e29e2bbbc7d71 GIT binary patch literal 195 zcmeZZfPjCKJmzv*Oo}Vvb-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zxw2jNfGX77S9{DpbAI?94>nieKa-X?mnaus z2*`N-^whlgoc!WqQ3i)gb~$2K&ZEfOl#*GT4AsRDX>+?V G%Mbts#z6M~ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299029.sc14.2774085.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299029.sc14.2774085.10 new file mode 100644 index 0000000000000000000000000000000000000000..c11c6f2520b287c7c78835a07e84decf5a053082 GIT binary patch literal 195 zcmeZZfPjCKJmzv<>+~q)b-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZiMw89qzX0n)gH6YoJNdZU~>f;n6<>YM7j7v zK*sB*r{=}yNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zk@uQ#UKMKYt376)IYrC0z~%}xGHZ!*iE{CU zfQ;8qPtA+Z$uBMzWms^@&fG5kzZz6ivHxQR&K4w15?o?j{18nknZ?OaT@UHz~%}xF>8r)iE{CU zfQ;8qPtA+Z$uBMzWk|SUH!J63kQ!7|vHxQR&hYyPO%hyUT>KDCDVfE|P+bS&Y*w2m GrUL*2F+Rco literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299042.sc14.2774085.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299042.sc14.2774085.14 new file mode 100644 index 0000000000000000000000000000000000000000..12a60b5b2f39d5fe95c774ee22b1a03247631950 GIT binary patch literal 135 zcmeZZfPjCKJmzv<+_dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vfyFlq^M@pEy6 VCFbbowMEuiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;<#kRXH}@VulAUI=G^=IA=q4jW@asME>SMN z5Rmcu>8W|~Ir+uKq6`Nv*~#6g7g2+1D)xWOz?tv@p-F;EjEf(lDJ8Qw8LI0+rp=sa Gt$qO2Cqjz= literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299050.sc14.2774085.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299050.sc14.2774085.16 new file mode 100644 index 0000000000000000000000000000000000000000..3f0da80d262a19cef98dec5d1433f1c4dc0e923c GIT binary patch literal 195 zcmeZZfPjCKJmzwqwMEuiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@@D7ddR3^oulAUI<`hfW4>nhzg;`6SOO%T* z1Z2E^dTL&LPJVH*D8qrvc1!Kq6V#xZiv1rmaIQkqB*7)d#ShVxl3AP#)uoVaWA^y# FbpY*5K-&NS literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299053.sc14.2774085.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299053.sc14.2774085.17 new file mode 100644 index 0000000000000000000000000000000000000000..1a34f0490fec5be60db0624d5cad8ffa0cfe455b GIT binary patch literal 135 zcmeZZfPjCKJmzxR`W9F7I^T4Z;!P?_%*@ksElbTSu`&wMNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZFwMEuiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(lu+2t}4{rS9{DpbN;q20-Gz)#;hgICCbGY z0y17dJvA>rC%?E@l;Oc;J8Q<&UusZI#r}^OI6G_*nk2ZyxcDKOQZkE^p}HKBY^)O% F@&VtCKGOgI literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299832.sc11.2623573.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299832.sc11.2623573.0 new file mode 100644 index 0000000000000000000000000000000000000000..2ff3c5eea602cde1122787a97f722ffc9e8ea79e GIT binary patch literal 195 zcmeZZfPjCKJmzvvT4Q6!=X}#qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-azL$fo+{MbS9{DpbMDi%1)D3-#;hgICCbGY z0y17dJvA>rC%?E@l;Oc;yIBj|y49eXiv1rmaE9n1G)ZuYaq&YmrDPT-!*pfXG+cNU F2mm)kKTH4s literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299835.sc11.2623573.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299835.sc11.2623573.1 new file mode 100644 index 0000000000000000000000000000000000000000..6cceb1d2777ef7747335f13bac949a22547c0bcc GIT binary patch literal 135 zcmeZZfPjCKJmzwqnDW|=&-tdK6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq%_u(TOMj|ptp+zrv~pEu(<+<7_|hs__;X3 V5_9x(@{5Z_IpjNR8<$L00syTBDrf)z literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299840.sc11.2623573.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299840.sc11.2623573.2 new file mode 100644 index 0000000000000000000000000000000000000000..baece8e5db66f8cccf42f5a881de5e7070a3299e GIT binary patch literal 195 zcmeZZfPjCKJmzwST8TLDIp1`Y;!P?_%*@ksElbTSu`&wMrC%?E@lwrYjyC(+^pI3uwD)xWOz$v!^p-F;EjEf(lDJ8Qw8LCSm&8B0* GzES`Y9X}iZ literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299843.sc11.2623573.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299843.sc11.2623573.3 new file mode 100644 index 0000000000000000000000000000000000000000..9f92a2f210ac4d03030985746358649963f0f2d4 GIT binary patch literal 195 zcmeZZfPjCKJmzwK6Z_`C=X}#qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-BFXtUSruyTt376)Ia{;8gUuD_VAc}n66N9x z0U58Ko|+e*lV4mc%Ajz;&iLu9N;RmaV*kesocEA4NpOjA@k2DFWELkwbsb2znIPA5 F7XYY;KtTWi literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299846.sc11.2623573.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299846.sc11.2623573.4 new file mode 100644 index 0000000000000000000000000000000000000000..150931dae4084ecb21e6eb888f1fbacefb255ff4 GIT binary patch literal 135 zcmeZZfPjCKJmzwqv|Q=P=X}#qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-V#xEZN*-!%ptp+zr+&{Wu(<+<8MOqt__;X3 V5_9x(@{5Z_1?+Nc-8B*x0RV%mDmVZD literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299851.sc11.2623573.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299851.sc11.2623573.5 new file mode 100644 index 0000000000000000000000000000000000000000..22e8fe612d785d5439f3991fff699a691adcf4c7 GIT binary patch literal 195 zcmeZZfPjCKJmzx#o^rv7&-tdK6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq^C^XPZetJt376)Ih{9M1e+_+$*d*LCCbGY z0y17dJvA>rC%?E@ltJOL-R+%WY-&(V#r}^OIHw_LlHd~K;)iHT$t+HW>SD;UQDB*{ F5dhH4KE?n5 literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299854.sc11.2623573.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299854.sc11.2623573.6 new file mode 100644 index 0000000000000000000000000000000000000000..d757e9115afc66f465e8241d6b2ed6a61285ce4b GIT binary patch literal 195 zcmeZZfPjCKJmzwK_=X}#qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-V%EodK^1E5t376)ISrCFfXx-?V%8Gp66N9x z0U58Ko|+e*lV4mc%CO*q-3qN6lhvS_iv1rmaBf1Bp F1^`r0K0N>c literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299857.sc11.2623573.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299857.sc11.2623573.7 new file mode 100644 index 0000000000000000000000000000000000000000..3b4503a91326bbfaa35943cb740a0c04a8d592d6 GIT binary patch literal 135 zcmeZZfPjCKJmzvTv}U^SIp1`Y;!P?_%*@ksElbTSu`&wMROOyGksRu FX8=FyKjQ!Z literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299865.sc11.2623573.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299865.sc11.2623573.9 new file mode 100644 index 0000000000000000000000000000000000000000..71b176da9e0851eb3b8f26c7e64f9f6cb19dd763 GIT binary patch literal 195 zcmeZZfPjCKJmzxps^+=zIp1`Y;!P?_%*@ksElbTSu`&wMg}n6<>YM7j7v zK*sB*r{=}y)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z5uA41Ngir$ptp+zXXHyIu(<+98MOqt__;X3 V5_9x(@{5Z_d-TF=rz_s$2LNBfDs=z= literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299873.sc11.2623573.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299873.sc11.2623573.11 new file mode 100644 index 0000000000000000000000000000000000000000..2ff8173b90bdf456c878758da7d4d50097efc854 GIT binary patch literal 195 zcmeZZfPjCKJmzx#yyWV^=X}#qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vRJ*`RuyXQt376)IRzr!!R89|GHZ!*iE{CU zfQ;8qPtA+Z$uBMzWoWo;_rCcMyBbtevHxQR&UPeC5?o?j{18nknZ?O4U6D2`rW>9E E07(Eog8%>k literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299876.sc11.2623573.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299876.sc11.2623573.12 new file mode 100644 index 0000000000000000000000000000000000000000..60cfa1eccc104277a96b156bb39016d9e20d7aac GIT binary patch literal 195 zcmeZZfPjCKJmzvrW-5B}Ip1`Y;!P?_%*@ksElbTSu`&wMs6x$swa4r;r}2Fyu(<+#%v$1HqFj6- zAmjDZQ}g0;@{5Z_85Uf!_h|KRy5e literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299887.sc11.2623573.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299887.sc11.2623573.15 new file mode 100644 index 0000000000000000000000000000000000000000..f67eb392cbd7da285e75104c2d0fab40b6870506 GIT binary patch literal 195 zcmeZZfPjCKJmzwyv%dA_bH3>)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z;g;8MP=%WNYLD4x&iwRuU~>f~Fl&i(iE{CU zfQ;8qPtA+Z$uBMzWnj2ucPCcSTMeqI*#9vDCrbuGlLVI-7e7Q(N@j5~RM&$@n?&LJ FeE^YKJt6=A literal 0 HcmV?d00001 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299890.sc11.2623573.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299890.sc11.2623573.16 new file mode 100644 index 0000000000000000000000000000000000000000..0ab62d033fa12415963fbf29363581c3a9a79c17 GIT binary patch literal 135 zcmeZZfPjCKJmzvbnlJI;bH3>)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZnOFAWf;`mRKyMcb&Us6hg3T2;&Zs5G#m~hN VmYAcTlV4mcT5>Vj)?xM~O8}Y5EYJV| literal 0 HcmV?d00001 diff --git a/tv-detection/maskrcnn_resnet101_fpn.isc b/tv-detection/maskrcnn_resnet101_fpn.isc index ad665820..c22ba318 100644 --- a/tv-detection/maskrcnn_resnet101_fpn.isc +++ b/tv-detection/maskrcnn_resnet101_fpn.isc @@ -3,4 +3,4 @@ gpu_type = "24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_maskrcnn_resnet101_fpn" -command = "train_cycling.py --dataset coco --model maskrcnn_resnet101_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_maskrcnn_resnet101_fpn/tb" \ No newline at end of file +command = "train_cycling.py --dataset coco --model maskrcnn_resnet101_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path $OUTPUT_PATH/tb" \ No newline at end of file diff --git a/tv-detection/retinanet_resnet101_fpn.isc b/tv-detection/retinanet_resnet101_fpn.isc index a1d3f65d..e6606711 100644 --- a/tv-detection/retinanet_resnet101_fpn.isc +++ b/tv-detection/retinanet_resnet101_fpn.isc @@ -3,5 +3,5 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_retinanet_resnet101_fpn" -command="train_cycling.py --dataset coco --model retinanet_resnet101_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --lr 0.01 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_retinanet_resnet101_fpn/tb" +command="train_cycling.py --dataset coco --model retinanet_resnet101_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --lr 0.01 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path $OUTPUT_PATH/tb" diff --git a/tv-segmentation/deeplabv3_mobilenet_v3_large.isc b/tv-segmentation/deeplabv3_mobilenet_v3_large.isc index e02a8c63..2a7c8e34 100644 --- a/tv-segmentation/deeplabv3_mobilenet_v3_large.isc +++ b/tv-segmentation/deeplabv3_mobilenet_v3_large.isc @@ -4,4 +4,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_deeplabv3_mobilenet_v3_large" -command="train_cycling.py --dataset coco -b 2 --model deeplabv3_mobilenet_v3_large --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_deeplabv3_mobilenet_v3_large/tb" +command="train_cycling.py --dataset coco -b 2 --model deeplabv3_mobilenet_v3_large --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path $OUTPUT_PATH/tb" diff --git a/tv-segmentation/fcn_resnet101.isc b/tv-segmentation/fcn_resnet101.isc index 8b148024..aef54fbb 100644 --- a/tv-segmentation/fcn_resnet101.isc +++ b/tv-segmentation/fcn_resnet101.isc @@ -4,4 +4,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_fcn_resnet101" -command="train_cycling.py --lr 0.02 --dataset coco -b 4 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path ~/output_fcn_resnet101/tb" \ No newline at end of file +command="train_cycling.py --lr 0.02 --dataset coco -b 4 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path $OUTPUT_PATH/tb --prev-resume /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/output_fcn_resnet101/exp_1847/checkpoint.isc" \ No newline at end of file From 37d878bfabe7eff4037c9315b263220c344f4e39 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Fri, 22 Sep 2023 12:59:46 +1000 Subject: [PATCH 38/44] tidy up, linting, deleting log files --- .../exp_1855 (autoencoder)/rank_0.txt | 12925 ---------------- .../exp_1855 (autoencoder)/rank_1.txt | 1310 -- .../exp_1855 (autoencoder)/rank_10.txt | 1310 -- .../exp_1855 (autoencoder)/rank_2.txt | 1310 -- .../exp_1855 (autoencoder)/rank_3.txt | 1310 -- .../exp_1855 (autoencoder)/rank_4.txt | 1310 -- .../exp_1855 (autoencoder)/rank_5.txt | 1310 -- .../exp_1855 (autoencoder)/rank_6.txt | 1310 -- .../exp_1855 (autoencoder)/rank_7.txt | 1310 -- .../exp_1855 (autoencoder)/rank_8.txt | 1310 -- .../exp_1855 (autoencoder)/rank_9.txt | 1310 -- ...nts.out.tfevents.1695275223.sc14.2163737.0 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695275225.sc14.2163737.1 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695275228.sc14.2163737.2 | Bin 132 -> 0 bytes ...nts.out.tfevents.1695275232.sc14.2163737.3 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695275235.sc14.2163737.4 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695275238.sc14.2163737.5 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695275242.sc14.2163737.6 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695275246.sc14.2163737.7 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695275249.sc14.2163737.8 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695275253.sc14.2163737.9 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695275257.sc14.2163737.10 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695275259.sc14.2163737.11 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695275264.sc14.2163737.12 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695275267.sc14.2163737.13 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695275270.sc14.2163737.14 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695275274.sc14.2163737.15 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695275278.sc14.2163737.16 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695275280.sc14.2163737.17 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695275285.sc14.2163737.18 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695275288.sc14.2163737.19 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695275975.sc12.3473404.0 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695275980.sc12.3473404.1 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695275983.sc12.3473404.2 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695275986.sc12.3473404.3 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695275991.sc12.3473404.4 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695275995.sc12.3473404.5 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695275997.sc12.3473404.6 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695276002.sc12.3473404.7 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695276006.sc12.3473404.8 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695276009.sc12.3473404.9 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695276013.sc12.3473404.10 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695276016.sc12.3473404.11 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695276019.sc12.3473404.12 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695276024.sc12.3473404.13 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695276027.sc12.3473404.14 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695276030.sc12.3473404.15 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695276035.sc12.3473404.16 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695276038.sc12.3473404.17 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695276041.sc12.3473404.18 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695276756.sc13.1777137.0 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695276758.sc13.1777137.1 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695276761.sc13.1777137.2 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695276765.sc13.1777137.3 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695276769.sc13.1777137.4 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695276772.sc13.1777137.5 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695276776.sc13.1777137.6 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695276780.sc13.1777137.7 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695276783.sc13.1777137.8 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695276787.sc13.1777137.9 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695276791.sc13.1777137.10 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695276793.sc13.1777137.11 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695276798.sc13.1777137.12 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695276801.sc13.1777137.13 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695276804.sc13.1777137.14 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695276809.sc13.1777137.15 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695276812.sc13.1777137.16 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695276815.sc13.1777137.17 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695276819.sc13.1777137.18 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695276822.sc13.1777137.19 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695277599.sc12.4183255.0 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695277604.sc12.4183255.1 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695277608.sc12.4183255.2 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695277610.sc12.4183255.3 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695277615.sc12.4183255.4 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695277619.sc12.4183255.5 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695277622.sc12.4183255.6 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695277626.sc12.4183255.7 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695277630.sc12.4183255.8 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695277632.sc12.4183255.9 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695277637.sc12.4183255.10 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695277641.sc12.4183255.11 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695277644.sc12.4183255.12 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695277648.sc12.4183255.13 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695277652.sc12.4183255.14 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695277655.sc12.4183255.15 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695277659.sc12.4183255.16 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695277663.sc12.4183255.17 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695277666.sc12.4183255.18 | Bin 134 -> 0 bytes ...ents.out.tfevents.1695278445.sc20.758883.0 | Bin 193 -> 0 bytes ...ents.out.tfevents.1695278447.sc20.758883.1 | Bin 193 -> 0 bytes ...ents.out.tfevents.1695278450.sc20.758883.2 | Bin 134 -> 0 bytes ...ents.out.tfevents.1695278455.sc20.758883.3 | Bin 193 -> 0 bytes ...ents.out.tfevents.1695278458.sc20.758883.4 | Bin 193 -> 0 bytes ...ents.out.tfevents.1695278461.sc20.758883.5 | Bin 134 -> 0 bytes ...ents.out.tfevents.1695278466.sc20.758883.6 | Bin 193 -> 0 bytes ...ents.out.tfevents.1695278469.sc20.758883.7 | Bin 193 -> 0 bytes ...ents.out.tfevents.1695278472.sc20.758883.8 | Bin 134 -> 0 bytes ...ents.out.tfevents.1695278476.sc20.758883.9 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695278480.sc20.758883.10 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695278483.sc20.758883.11 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695278487.sc20.758883.12 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695278491.sc20.758883.13 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695278493.sc20.758883.14 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695278498.sc20.758883.15 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695278501.sc20.758883.16 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695278504.sc20.758883.17 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695278508.sc20.758883.18 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695278512.sc20.758883.19 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695279205.sc10.1505340.0 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695279210.sc10.1505340.1 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695279213.sc10.1505340.2 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695279216.sc10.1505340.3 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695279220.sc10.1505340.4 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695279224.sc10.1505340.5 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695279227.sc10.1505340.6 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695279231.sc10.1505340.7 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695279234.sc10.1505340.8 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695279237.sc10.1505340.9 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695279242.sc10.1505340.10 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695279245.sc10.1505340.11 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695279248.sc10.1505340.12 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695279252.sc10.1505340.13 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695279255.sc10.1505340.14 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695279258.sc10.1505340.15 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695279263.sc10.1505340.16 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695279266.sc10.1505340.17 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695279269.sc10.1505340.18 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695279273.sc10.1505340.19 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695279943.sc18.3937612.0 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695279946.sc18.3937612.1 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695279950.sc18.3937612.2 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695279954.sc18.3937612.3 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695279956.sc18.3937612.4 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695279961.sc18.3937612.5 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695279964.sc18.3937612.6 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695279967.sc18.3937612.7 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695279971.sc18.3937612.8 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695279975.sc18.3937612.9 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695279978.sc18.3937612.10 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695279982.sc18.3937612.11 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695279985.sc18.3937612.12 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695279988.sc18.3937612.13 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695279993.sc18.3937612.14 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695279996.sc18.3937612.15 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695279999.sc18.3937612.16 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695280003.sc18.3937612.17 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695280006.sc18.3937612.18 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695280009.sc18.3937612.19 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695280680.sc10.1562555.0 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695280682.sc10.1562555.1 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695280685.sc10.1562555.2 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695280689.sc10.1562555.3 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695280693.sc10.1562555.4 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695280696.sc10.1562555.5 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695280700.sc10.1562555.6 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695280704.sc10.1562555.7 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695280707.sc10.1562555.8 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695280711.sc10.1562555.9 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695280714.sc10.1562555.10 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695280717.sc10.1562555.11 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695280722.sc10.1562555.12 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695280725.sc10.1562555.13 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695280728.sc10.1562555.14 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695280733.sc10.1562555.15 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695280736.sc10.1562555.16 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695280739.sc10.1562555.17 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695280743.sc10.1562555.18 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695281501.sc13.2274570.0 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695281504.sc13.2274570.1 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695281509.sc13.2274570.2 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695281512.sc13.2274570.3 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695281515.sc13.2274570.4 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695281520.sc13.2274570.5 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695281523.sc13.2274570.6 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695281526.sc13.2274570.7 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695281530.sc13.2274570.8 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695281534.sc13.2274570.9 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695281537.sc13.2274570.10 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695281541.sc13.2274570.11 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695281545.sc13.2274570.12 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695281548.sc13.2274570.13 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695281552.sc13.2274570.14 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695281555.sc13.2274570.15 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695281558.sc13.2274570.16 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695281563.sc13.2274570.17 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695281566.sc13.2274570.18 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695282323.sc13.2296022.0 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695282328.sc13.2296022.1 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695282331.sc13.2296022.2 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695282334.sc13.2296022.3 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695282339.sc13.2296022.4 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695282342.sc13.2296022.5 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695282345.sc13.2296022.6 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695282350.sc13.2296022.7 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695282353.sc13.2296022.8 | Bin 193 -> 0 bytes ...nts.out.tfevents.1695282356.sc13.2296022.9 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695282361.sc13.2296022.10 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695282364.sc13.2296022.11 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695282369.sc13.2296022.12 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695282374.sc13.2296022.13 | Bin 193 -> 0 bytes ...ts.out.tfevents.1695282377.sc13.2296022.14 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695282381.sc13.2296022.15 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695282385.sc13.2296022.16 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695282389.sc13.2296022.17 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695282391.sc13.2296022.18 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695283062.sc16.2520987.0 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695283064.sc16.2520987.1 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695283067.sc16.2520987.2 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695283072.sc16.2520987.3 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695283075.sc16.2520987.4 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695283078.sc16.2520987.5 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695283083.sc16.2520987.6 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695283086.sc16.2520987.7 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695283089.sc16.2520987.8 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695283094.sc16.2520987.9 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695283097.sc16.2520987.10 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695283100.sc16.2520987.11 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695283104.sc16.2520987.12 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695283108.sc16.2520987.13 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695283110.sc16.2520987.14 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695283115.sc16.2520987.15 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695283118.sc16.2520987.16 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695283121.sc16.2520987.17 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695283125.sc16.2520987.18 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695283129.sc16.2520987.19 | Bin 141 -> 0 bytes ...nts.out.tfevents.1695283892.sc11.2190319.0 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695283895.sc11.2190319.1 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695283900.sc11.2190319.2 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695283903.sc11.2190319.3 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695283906.sc11.2190319.4 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695283910.sc11.2190319.5 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695283914.sc11.2190319.6 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695283917.sc11.2190319.7 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695283921.sc11.2190319.8 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695283925.sc11.2190319.9 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695283927.sc11.2190319.10 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695283932.sc11.2190319.11 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695283936.sc11.2190319.12 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695283938.sc11.2190319.13 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695283943.sc11.2190319.14 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695283946.sc11.2190319.15 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695283949.sc11.2190319.16 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695283953.sc11.2190319.17 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695283957.sc11.2190319.18 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695283960.sc11.2190319.19 | Bin 134 -> 0 bytes ...ents.out.tfevents.1695284628.sc12.184661.0 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695284630.sc12.184661.1 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695284633.sc12.184661.2 | Bin 134 -> 0 bytes ...ents.out.tfevents.1695284638.sc12.184661.3 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695284641.sc12.184661.4 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695284645.sc12.184661.5 | Bin 134 -> 0 bytes ...ents.out.tfevents.1695284650.sc12.184661.6 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695284653.sc12.184661.7 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695284656.sc12.184661.8 | Bin 134 -> 0 bytes ...ents.out.tfevents.1695284660.sc12.184661.9 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695284663.sc12.184661.10 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695284666.sc12.184661.11 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695284671.sc12.184661.12 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695284674.sc12.184661.13 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695284677.sc12.184661.14 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695284681.sc12.184661.15 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695284684.sc12.184661.16 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695284687.sc12.184661.17 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695284692.sc12.184661.18 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695285450.sc9.1183018.0 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695285453.sc9.1183018.1 | Bin 134 -> 0 bytes ...ents.out.tfevents.1695285457.sc9.1183018.2 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695285461.sc9.1183018.3 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695285463.sc9.1183018.4 | Bin 134 -> 0 bytes ...ents.out.tfevents.1695285468.sc9.1183018.5 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695285471.sc9.1183018.6 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695285474.sc9.1183018.7 | Bin 134 -> 0 bytes ...ents.out.tfevents.1695285478.sc9.1183018.8 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695285482.sc9.1183018.9 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695285484.sc9.1183018.10 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695285489.sc9.1183018.11 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695285492.sc9.1183018.12 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695285495.sc9.1183018.13 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695285499.sc9.1183018.14 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695285502.sc9.1183018.15 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695285505.sc9.1183018.16 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695285510.sc9.1183018.17 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695285513.sc9.1183018.18 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695285516.sc9.1183018.19 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695286186.sc10.1717960.0 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695286188.sc10.1717960.1 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695286190.sc10.1717960.2 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695286195.sc10.1717960.3 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695286198.sc10.1717960.4 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695286201.sc10.1717960.5 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695286205.sc10.1717960.6 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695286209.sc10.1717960.7 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695286211.sc10.1717960.8 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695286216.sc10.1717960.9 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695286219.sc10.1717960.10 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695286222.sc10.1717960.11 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695286226.sc10.1717960.12 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695286229.sc10.1717960.13 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695286232.sc10.1717960.14 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695286236.sc10.1717960.15 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695286240.sc10.1717960.16 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695286242.sc10.1717960.17 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695286247.sc10.1717960.18 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695286250.sc10.1717960.19 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695286253.sc10.1717960.20 | Bin 134 -> 0 bytes ...ents.out.tfevents.1695287096.sc9.1214828.0 | Bin 134 -> 0 bytes ...ents.out.tfevents.1695287101.sc9.1214828.1 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695287104.sc9.1214828.2 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695287107.sc9.1214828.3 | Bin 134 -> 0 bytes ...ents.out.tfevents.1695287111.sc9.1214828.4 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695287114.sc9.1214828.5 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695287117.sc9.1214828.6 | Bin 134 -> 0 bytes ...ents.out.tfevents.1695287122.sc9.1214828.7 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695287125.sc9.1214828.8 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695287128.sc9.1214828.9 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695287132.sc9.1214828.10 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695287136.sc9.1214828.11 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695287139.sc9.1214828.12 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695287143.sc9.1214828.13 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695287147.sc9.1214828.14 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695287150.sc9.1214828.15 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695287154.sc9.1214828.16 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695287158.sc9.1214828.17 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695287161.sc9.1214828.18 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695287165.sc9.1214828.19 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695287833.sc20.1362611.0 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695287835.sc20.1362611.1 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695287838.sc20.1362611.2 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695287842.sc20.1362611.3 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695287846.sc20.1362611.4 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695287849.sc20.1362611.5 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695287853.sc20.1362611.6 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695287857.sc20.1362611.7 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695287860.sc20.1362611.8 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695287864.sc20.1362611.9 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695287867.sc20.1362611.10 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695287870.sc20.1362611.11 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695287875.sc20.1362611.12 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695287878.sc20.1362611.13 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695287881.sc20.1362611.14 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695287886.sc20.1362611.15 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695287889.sc20.1362611.16 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695287892.sc20.1362611.17 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695287896.sc20.1362611.18 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695287900.sc20.1362611.19 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695288568.sc10.1768728.0 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695288573.sc10.1768728.1 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695288576.sc10.1768728.2 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695288579.sc10.1768728.3 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695288584.sc10.1768728.4 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695288587.sc10.1768728.5 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695288590.sc10.1768728.6 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695288594.sc10.1768728.7 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695288598.sc10.1768728.8 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695288601.sc10.1768728.9 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695288605.sc10.1768728.10 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695288609.sc10.1768728.11 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695288611.sc10.1768728.12 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695288616.sc10.1768728.13 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695288620.sc10.1768728.14 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695288622.sc10.1768728.15 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695288627.sc10.1768728.16 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695288631.sc10.1768728.17 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695288634.sc10.1768728.18 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695289303.sc10.1793187.0 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695289305.sc10.1793187.1 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695289307.sc10.1793187.2 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695289311.sc10.1793187.3 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695289315.sc10.1793187.4 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695289317.sc10.1793187.5 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695289322.sc10.1793187.6 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695289325.sc10.1793187.7 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695289328.sc10.1793187.8 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695289332.sc10.1793187.9 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695289335.sc10.1793187.10 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695289338.sc10.1793187.11 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695289342.sc10.1793187.12 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695289346.sc10.1793187.13 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695289348.sc10.1793187.14 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695289353.sc10.1793187.15 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695289356.sc10.1793187.16 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695289359.sc10.1793187.17 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695289363.sc10.1793187.18 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695289367.sc10.1793187.19 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695289369.sc10.1793187.20 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695290122.sc11.2375140.0 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695290126.sc11.2375140.1 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695290130.sc11.2375140.2 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695290133.sc11.2375140.3 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695290137.sc11.2375140.4 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695290140.sc11.2375140.5 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695290143.sc11.2375140.6 | Bin 134 -> 0 bytes ...nts.out.tfevents.1695290148.sc11.2375140.7 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695290151.sc11.2375140.8 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695290154.sc11.2375140.9 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695290158.sc11.2375140.10 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695290162.sc11.2375140.11 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695290164.sc11.2375140.12 | Bin 134 -> 0 bytes ...ts.out.tfevents.1695290169.sc11.2375140.13 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695290172.sc11.2375140.14 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695290175.sc11.2375140.15 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695290179.sc11.2375140.16 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695290183.sc11.2375140.17 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695290185.sc11.2375140.18 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695290190.sc11.2375140.19 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695290935.sc12.371669.0 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695290938.sc12.371669.1 | Bin 135 -> 0 bytes ...ents.out.tfevents.1695290942.sc12.371669.2 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695290946.sc12.371669.3 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695290948.sc12.371669.4 | Bin 135 -> 0 bytes ...ents.out.tfevents.1695290953.sc12.371669.5 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695290956.sc12.371669.6 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695290959.sc12.371669.7 | Bin 135 -> 0 bytes ...ents.out.tfevents.1695290964.sc12.371669.8 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695290967.sc12.371669.9 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695290970.sc12.371669.10 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695290974.sc12.371669.11 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695290978.sc12.371669.12 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695290980.sc12.371669.13 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695290985.sc12.371669.14 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695290988.sc12.371669.15 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695290991.sc12.371669.16 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695290995.sc12.371669.17 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695290998.sc12.371669.18 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695291826.sc16.2741235.0 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695291831.sc16.2741235.1 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695291834.sc16.2741235.2 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695291837.sc16.2741235.3 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695291841.sc16.2741235.4 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695291845.sc16.2741235.5 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695291848.sc16.2741235.6 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695291852.sc16.2741235.7 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695291856.sc16.2741235.8 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695291858.sc16.2741235.9 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695291863.sc16.2741235.10 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695291867.sc16.2741235.11 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695291870.sc16.2741235.12 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695291874.sc16.2741235.13 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695291878.sc16.2741235.14 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695291881.sc16.2741235.15 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695291885.sc16.2741235.16 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695291889.sc16.2741235.17 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695291891.sc16.2741235.18 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695291896.sc16.2741235.19 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695292972.sc9.1353776.0 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695292975.sc9.1353776.1 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695292977.sc9.1353776.2 | Bin 135 -> 0 bytes ...ents.out.tfevents.1695292982.sc9.1353776.3 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695292986.sc9.1353776.4 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695292988.sc9.1353776.5 | Bin 135 -> 0 bytes ...ents.out.tfevents.1695292993.sc9.1353776.6 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695292996.sc9.1353776.7 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695292999.sc9.1353776.8 | Bin 135 -> 0 bytes ...ents.out.tfevents.1695293004.sc9.1353776.9 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695293007.sc9.1353776.10 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695293010.sc9.1353776.11 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695293014.sc9.1353776.12 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695293017.sc9.1353776.13 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695293020.sc9.1353776.14 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695293025.sc9.1353776.15 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695293028.sc9.1353776.16 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695293031.sc9.1353776.17 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695293035.sc9.1353776.18 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695293865.sc14.2649097.0 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695293868.sc14.2649097.1 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695293872.sc14.2649097.2 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695293876.sc14.2649097.3 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695293879.sc14.2649097.4 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695293883.sc14.2649097.5 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695293887.sc14.2649097.6 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695293890.sc14.2649097.7 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695293894.sc14.2649097.8 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695293897.sc14.2649097.9 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695293900.sc14.2649097.10 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695293905.sc14.2649097.11 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695293908.sc14.2649097.12 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695293911.sc14.2649097.13 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695293916.sc14.2649097.14 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695293919.sc14.2649097.15 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695293922.sc14.2649097.16 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695293927.sc14.2649097.17 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695293930.sc14.2649097.18 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695294796.sc14.2671192.0 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695294801.sc14.2671192.1 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695294804.sc14.2671192.2 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695294807.sc14.2671192.3 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695294812.sc14.2671192.4 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695294816.sc14.2671192.5 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695294818.sc14.2671192.6 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695294823.sc14.2671192.7 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695294827.sc14.2671192.8 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695294830.sc14.2671192.9 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695294834.sc14.2671192.10 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695294838.sc14.2671192.11 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695294840.sc14.2671192.12 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695294845.sc14.2671192.13 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695294848.sc14.2671192.14 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695294851.sc14.2671192.15 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695294856.sc14.2671192.16 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695294859.sc14.2671192.17 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695294862.sc14.2671192.18 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695294866.sc14.2671192.19 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695295534.sc14.2696291.0 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695295536.sc14.2696291.1 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695295541.sc14.2696291.2 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695295544.sc14.2696291.3 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695295547.sc14.2696291.4 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695295551.sc14.2696291.5 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695295555.sc14.2696291.6 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695295558.sc14.2696291.7 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695295562.sc14.2696291.8 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695295565.sc14.2696291.9 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695295568.sc14.2696291.10 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695295573.sc14.2696291.11 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695295576.sc14.2696291.12 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695295579.sc14.2696291.13 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695295584.sc14.2696291.14 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695295587.sc14.2696291.15 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695295590.sc14.2696291.16 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695295594.sc14.2696291.17 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695295598.sc14.2696291.18 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695295601.sc14.2696291.19 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695296480.sc14.2714192.0 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695296482.sc14.2714192.1 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695296485.sc14.2714192.2 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695296489.sc14.2714192.3 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695296493.sc14.2714192.4 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695296496.sc14.2714192.5 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695296500.sc14.2714192.6 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695296504.sc14.2714192.7 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695296507.sc14.2714192.8 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695296511.sc14.2714192.9 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695296515.sc14.2714192.10 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695296518.sc14.2714192.11 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695296522.sc14.2714192.12 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695296525.sc14.2714192.13 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695296528.sc14.2714192.14 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695296533.sc14.2714192.15 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695296536.sc14.2714192.16 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695296539.sc14.2714192.17 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695296544.sc14.2714192.18 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695297315.sc14.2733087.0 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695297318.sc14.2733087.1 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695297323.sc14.2733087.2 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695297326.sc14.2733087.3 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695297329.sc14.2733087.4 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695297334.sc14.2733087.5 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695297337.sc14.2733087.6 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695297340.sc14.2733087.7 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695297345.sc14.2733087.8 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695297348.sc14.2733087.9 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695297350.sc14.2733087.10 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695297355.sc14.2733087.11 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695297358.sc14.2733087.12 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695297361.sc14.2733087.13 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695297366.sc14.2733087.14 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695297369.sc14.2733087.15 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695297372.sc14.2733087.16 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695297376.sc14.2733087.17 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695297380.sc14.2733087.18 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695297383.sc14.2733087.19 | Bin 135 -> 0 bytes ...ents.out.tfevents.1695298153.sc9.1458633.0 | Bin 135 -> 0 bytes ...ents.out.tfevents.1695298158.sc9.1458633.1 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695298161.sc9.1458633.2 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695298164.sc9.1458633.3 | Bin 135 -> 0 bytes ...ents.out.tfevents.1695298168.sc9.1458633.4 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695298172.sc9.1458633.5 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695298174.sc9.1458633.6 | Bin 135 -> 0 bytes ...ents.out.tfevents.1695298179.sc9.1458633.7 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695298183.sc9.1458633.8 | Bin 195 -> 0 bytes ...ents.out.tfevents.1695298185.sc9.1458633.9 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695298190.sc9.1458633.10 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695298193.sc9.1458633.11 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695298196.sc9.1458633.12 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695298201.sc9.1458633.13 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695298205.sc9.1458633.14 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695298207.sc9.1458633.15 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695298212.sc9.1458633.16 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695298216.sc9.1458633.17 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695298218.sc9.1458633.18 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695298994.sc14.2774085.0 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695298996.sc14.2774085.1 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695298999.sc14.2774085.2 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695299003.sc14.2774085.3 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695299007.sc14.2774085.4 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695299010.sc14.2774085.5 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695299014.sc14.2774085.6 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695299018.sc14.2774085.7 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695299020.sc14.2774085.8 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695299025.sc14.2774085.9 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695299029.sc14.2774085.10 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695299032.sc14.2774085.11 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695299036.sc14.2774085.12 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695299040.sc14.2774085.13 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695299042.sc14.2774085.14 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695299047.sc14.2774085.15 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695299050.sc14.2774085.16 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695299053.sc14.2774085.17 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695299058.sc14.2774085.18 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695299061.sc14.2774085.19 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695299832.sc11.2623573.0 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695299835.sc11.2623573.1 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695299840.sc11.2623573.2 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695299843.sc11.2623573.3 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695299846.sc11.2623573.4 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695299851.sc11.2623573.5 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695299854.sc11.2623573.6 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695299857.sc11.2623573.7 | Bin 135 -> 0 bytes ...nts.out.tfevents.1695299862.sc11.2623573.8 | Bin 195 -> 0 bytes ...nts.out.tfevents.1695299865.sc11.2623573.9 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695299868.sc11.2623573.10 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695299873.sc11.2623573.11 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695299876.sc11.2623573.12 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695299879.sc11.2623573.13 | Bin 135 -> 0 bytes ...ts.out.tfevents.1695299883.sc11.2623573.14 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695299887.sc11.2623573.15 | Bin 195 -> 0 bytes ...ts.out.tfevents.1695299890.sc11.2623573.16 | Bin 135 -> 0 bytes monai_brats_mri_2d/loops.py | 9 + monai_brats_mri_2d/train_cycling_gen.py | 2 +- tv-detection/coco_eval.py | 4 +- tv-detection/engine.py | 6 +- tv-detection/prep.py | 9 +- tv-detection/pyproject.toml | 3 + tv-detection/train_cycling.py | 91 +- tv-segmentation/pyproject.toml | 3 + tv-segmentation/train_cycling.py | 68 +- tv-segmentation/utils.py | 2 +- 628 files changed, 71 insertions(+), 26151 deletions(-) delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_0.txt delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_1.txt delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_10.txt delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_2.txt delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_3.txt delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_4.txt delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_5.txt delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_6.txt delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_7.txt delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_8.txt delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/rank_9.txt delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275223.sc14.2163737.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275225.sc14.2163737.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275228.sc14.2163737.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275232.sc14.2163737.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275235.sc14.2163737.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275238.sc14.2163737.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275242.sc14.2163737.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275246.sc14.2163737.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275249.sc14.2163737.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275253.sc14.2163737.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275257.sc14.2163737.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275259.sc14.2163737.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275264.sc14.2163737.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275267.sc14.2163737.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275270.sc14.2163737.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275274.sc14.2163737.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275278.sc14.2163737.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275280.sc14.2163737.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275285.sc14.2163737.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275288.sc14.2163737.19 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275975.sc12.3473404.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275980.sc12.3473404.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275983.sc12.3473404.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275986.sc12.3473404.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275991.sc12.3473404.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275995.sc12.3473404.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275997.sc12.3473404.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276002.sc12.3473404.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276006.sc12.3473404.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276009.sc12.3473404.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276013.sc12.3473404.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276016.sc12.3473404.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276019.sc12.3473404.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276024.sc12.3473404.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276027.sc12.3473404.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276030.sc12.3473404.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276035.sc12.3473404.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276038.sc12.3473404.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276041.sc12.3473404.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276756.sc13.1777137.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276758.sc13.1777137.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276761.sc13.1777137.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276765.sc13.1777137.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276769.sc13.1777137.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276772.sc13.1777137.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276776.sc13.1777137.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276780.sc13.1777137.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276783.sc13.1777137.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276787.sc13.1777137.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276791.sc13.1777137.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276793.sc13.1777137.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276798.sc13.1777137.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276801.sc13.1777137.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276804.sc13.1777137.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276809.sc13.1777137.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276812.sc13.1777137.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276815.sc13.1777137.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276819.sc13.1777137.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276822.sc13.1777137.19 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277599.sc12.4183255.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277604.sc12.4183255.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277608.sc12.4183255.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277610.sc12.4183255.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277615.sc12.4183255.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277619.sc12.4183255.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277622.sc12.4183255.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277626.sc12.4183255.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277630.sc12.4183255.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277632.sc12.4183255.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277637.sc12.4183255.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277641.sc12.4183255.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277644.sc12.4183255.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277648.sc12.4183255.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277652.sc12.4183255.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277655.sc12.4183255.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277659.sc12.4183255.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277663.sc12.4183255.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277666.sc12.4183255.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278445.sc20.758883.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278447.sc20.758883.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278450.sc20.758883.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278455.sc20.758883.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278458.sc20.758883.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278461.sc20.758883.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278466.sc20.758883.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278469.sc20.758883.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278472.sc20.758883.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278476.sc20.758883.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278480.sc20.758883.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278483.sc20.758883.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278487.sc20.758883.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278491.sc20.758883.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278493.sc20.758883.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278498.sc20.758883.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278501.sc20.758883.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278504.sc20.758883.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278508.sc20.758883.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278512.sc20.758883.19 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279205.sc10.1505340.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279210.sc10.1505340.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279213.sc10.1505340.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279216.sc10.1505340.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279220.sc10.1505340.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279224.sc10.1505340.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279227.sc10.1505340.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279231.sc10.1505340.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279234.sc10.1505340.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279237.sc10.1505340.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279242.sc10.1505340.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279245.sc10.1505340.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279248.sc10.1505340.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279252.sc10.1505340.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279255.sc10.1505340.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279258.sc10.1505340.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279263.sc10.1505340.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279266.sc10.1505340.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279269.sc10.1505340.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279273.sc10.1505340.19 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279943.sc18.3937612.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279946.sc18.3937612.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279950.sc18.3937612.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279954.sc18.3937612.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279956.sc18.3937612.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279961.sc18.3937612.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279964.sc18.3937612.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279967.sc18.3937612.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279971.sc18.3937612.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279975.sc18.3937612.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279978.sc18.3937612.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279982.sc18.3937612.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279985.sc18.3937612.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279988.sc18.3937612.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279993.sc18.3937612.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279996.sc18.3937612.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279999.sc18.3937612.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280003.sc18.3937612.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280006.sc18.3937612.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280009.sc18.3937612.19 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280680.sc10.1562555.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280682.sc10.1562555.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280685.sc10.1562555.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280689.sc10.1562555.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280693.sc10.1562555.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280696.sc10.1562555.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280700.sc10.1562555.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280704.sc10.1562555.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280707.sc10.1562555.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280711.sc10.1562555.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280714.sc10.1562555.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280717.sc10.1562555.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280722.sc10.1562555.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280725.sc10.1562555.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280728.sc10.1562555.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280733.sc10.1562555.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280736.sc10.1562555.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280739.sc10.1562555.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280743.sc10.1562555.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281501.sc13.2274570.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281504.sc13.2274570.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281509.sc13.2274570.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281512.sc13.2274570.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281515.sc13.2274570.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281520.sc13.2274570.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281523.sc13.2274570.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281526.sc13.2274570.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281530.sc13.2274570.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281534.sc13.2274570.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281537.sc13.2274570.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281541.sc13.2274570.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281545.sc13.2274570.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281548.sc13.2274570.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281552.sc13.2274570.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281555.sc13.2274570.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281558.sc13.2274570.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281563.sc13.2274570.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281566.sc13.2274570.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282323.sc13.2296022.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282328.sc13.2296022.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282331.sc13.2296022.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282334.sc13.2296022.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282339.sc13.2296022.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282342.sc13.2296022.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282345.sc13.2296022.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282350.sc13.2296022.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282353.sc13.2296022.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282356.sc13.2296022.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282361.sc13.2296022.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282364.sc13.2296022.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282369.sc13.2296022.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282374.sc13.2296022.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282377.sc13.2296022.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282381.sc13.2296022.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282385.sc13.2296022.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282389.sc13.2296022.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282391.sc13.2296022.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283062.sc16.2520987.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283064.sc16.2520987.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283067.sc16.2520987.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283072.sc16.2520987.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283075.sc16.2520987.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283078.sc16.2520987.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283083.sc16.2520987.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283086.sc16.2520987.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283089.sc16.2520987.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283094.sc16.2520987.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283097.sc16.2520987.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283100.sc16.2520987.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283104.sc16.2520987.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283108.sc16.2520987.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283110.sc16.2520987.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283115.sc16.2520987.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283118.sc16.2520987.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283121.sc16.2520987.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283125.sc16.2520987.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283129.sc16.2520987.19 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283892.sc11.2190319.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283895.sc11.2190319.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283900.sc11.2190319.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283903.sc11.2190319.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283906.sc11.2190319.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283910.sc11.2190319.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283914.sc11.2190319.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283917.sc11.2190319.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283921.sc11.2190319.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283925.sc11.2190319.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283927.sc11.2190319.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283932.sc11.2190319.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283936.sc11.2190319.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283938.sc11.2190319.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283943.sc11.2190319.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283946.sc11.2190319.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283949.sc11.2190319.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283953.sc11.2190319.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283957.sc11.2190319.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283960.sc11.2190319.19 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284628.sc12.184661.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284630.sc12.184661.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284633.sc12.184661.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284638.sc12.184661.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284641.sc12.184661.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284645.sc12.184661.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284650.sc12.184661.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284653.sc12.184661.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284656.sc12.184661.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284660.sc12.184661.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284663.sc12.184661.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284666.sc12.184661.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284671.sc12.184661.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284674.sc12.184661.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284677.sc12.184661.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284681.sc12.184661.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284684.sc12.184661.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284687.sc12.184661.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284692.sc12.184661.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285450.sc9.1183018.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285453.sc9.1183018.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285457.sc9.1183018.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285461.sc9.1183018.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285463.sc9.1183018.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285468.sc9.1183018.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285471.sc9.1183018.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285474.sc9.1183018.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285478.sc9.1183018.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285482.sc9.1183018.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285484.sc9.1183018.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285489.sc9.1183018.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285492.sc9.1183018.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285495.sc9.1183018.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285499.sc9.1183018.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285502.sc9.1183018.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285505.sc9.1183018.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285510.sc9.1183018.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285513.sc9.1183018.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285516.sc9.1183018.19 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286186.sc10.1717960.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286188.sc10.1717960.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286190.sc10.1717960.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286195.sc10.1717960.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286198.sc10.1717960.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286201.sc10.1717960.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286205.sc10.1717960.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286209.sc10.1717960.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286211.sc10.1717960.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286216.sc10.1717960.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286219.sc10.1717960.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286222.sc10.1717960.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286226.sc10.1717960.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286229.sc10.1717960.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286232.sc10.1717960.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286236.sc10.1717960.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286240.sc10.1717960.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286242.sc10.1717960.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286247.sc10.1717960.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286250.sc10.1717960.19 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286253.sc10.1717960.20 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287096.sc9.1214828.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287101.sc9.1214828.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287104.sc9.1214828.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287107.sc9.1214828.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287111.sc9.1214828.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287114.sc9.1214828.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287117.sc9.1214828.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287122.sc9.1214828.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287125.sc9.1214828.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287128.sc9.1214828.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287132.sc9.1214828.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287136.sc9.1214828.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287139.sc9.1214828.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287143.sc9.1214828.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287147.sc9.1214828.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287150.sc9.1214828.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287154.sc9.1214828.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287158.sc9.1214828.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287161.sc9.1214828.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287165.sc9.1214828.19 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287833.sc20.1362611.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287835.sc20.1362611.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287838.sc20.1362611.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287842.sc20.1362611.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287846.sc20.1362611.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287849.sc20.1362611.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287853.sc20.1362611.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287857.sc20.1362611.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287860.sc20.1362611.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287864.sc20.1362611.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287867.sc20.1362611.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287870.sc20.1362611.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287875.sc20.1362611.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287878.sc20.1362611.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287881.sc20.1362611.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287886.sc20.1362611.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287889.sc20.1362611.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287892.sc20.1362611.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287896.sc20.1362611.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287900.sc20.1362611.19 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288568.sc10.1768728.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288573.sc10.1768728.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288576.sc10.1768728.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288579.sc10.1768728.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288584.sc10.1768728.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288587.sc10.1768728.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288590.sc10.1768728.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288594.sc10.1768728.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288598.sc10.1768728.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288601.sc10.1768728.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288605.sc10.1768728.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288609.sc10.1768728.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288611.sc10.1768728.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288616.sc10.1768728.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288620.sc10.1768728.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288622.sc10.1768728.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288627.sc10.1768728.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288631.sc10.1768728.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288634.sc10.1768728.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289303.sc10.1793187.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289305.sc10.1793187.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289307.sc10.1793187.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289311.sc10.1793187.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289315.sc10.1793187.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289317.sc10.1793187.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289322.sc10.1793187.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289325.sc10.1793187.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289328.sc10.1793187.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289332.sc10.1793187.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289335.sc10.1793187.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289338.sc10.1793187.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289342.sc10.1793187.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289346.sc10.1793187.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289348.sc10.1793187.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289353.sc10.1793187.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289356.sc10.1793187.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289359.sc10.1793187.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289363.sc10.1793187.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289367.sc10.1793187.19 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289369.sc10.1793187.20 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290122.sc11.2375140.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290126.sc11.2375140.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290130.sc11.2375140.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290133.sc11.2375140.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290137.sc11.2375140.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290140.sc11.2375140.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290143.sc11.2375140.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290148.sc11.2375140.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290151.sc11.2375140.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290154.sc11.2375140.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290158.sc11.2375140.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290162.sc11.2375140.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290164.sc11.2375140.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290169.sc11.2375140.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290172.sc11.2375140.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290175.sc11.2375140.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290179.sc11.2375140.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290183.sc11.2375140.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290185.sc11.2375140.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290190.sc11.2375140.19 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290935.sc12.371669.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290938.sc12.371669.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290942.sc12.371669.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290946.sc12.371669.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290948.sc12.371669.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290953.sc12.371669.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290956.sc12.371669.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290959.sc12.371669.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290964.sc12.371669.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290967.sc12.371669.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290970.sc12.371669.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290974.sc12.371669.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290978.sc12.371669.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290980.sc12.371669.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290985.sc12.371669.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290988.sc12.371669.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290991.sc12.371669.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290995.sc12.371669.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290998.sc12.371669.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291826.sc16.2741235.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291831.sc16.2741235.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291834.sc16.2741235.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291837.sc16.2741235.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291841.sc16.2741235.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291845.sc16.2741235.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291848.sc16.2741235.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291852.sc16.2741235.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291856.sc16.2741235.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291858.sc16.2741235.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291863.sc16.2741235.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291867.sc16.2741235.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291870.sc16.2741235.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291874.sc16.2741235.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291878.sc16.2741235.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291881.sc16.2741235.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291885.sc16.2741235.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291889.sc16.2741235.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291891.sc16.2741235.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291896.sc16.2741235.19 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292972.sc9.1353776.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292975.sc9.1353776.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292977.sc9.1353776.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292982.sc9.1353776.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292986.sc9.1353776.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292988.sc9.1353776.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292993.sc9.1353776.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292996.sc9.1353776.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292999.sc9.1353776.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293004.sc9.1353776.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293007.sc9.1353776.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293010.sc9.1353776.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293014.sc9.1353776.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293017.sc9.1353776.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293020.sc9.1353776.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293025.sc9.1353776.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293028.sc9.1353776.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293031.sc9.1353776.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293035.sc9.1353776.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293865.sc14.2649097.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293868.sc14.2649097.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293872.sc14.2649097.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293876.sc14.2649097.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293879.sc14.2649097.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293883.sc14.2649097.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293887.sc14.2649097.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293890.sc14.2649097.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293894.sc14.2649097.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293897.sc14.2649097.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293900.sc14.2649097.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293905.sc14.2649097.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293908.sc14.2649097.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293911.sc14.2649097.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293916.sc14.2649097.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293919.sc14.2649097.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293922.sc14.2649097.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293927.sc14.2649097.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293930.sc14.2649097.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294796.sc14.2671192.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294801.sc14.2671192.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294804.sc14.2671192.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294807.sc14.2671192.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294812.sc14.2671192.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294816.sc14.2671192.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294818.sc14.2671192.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294823.sc14.2671192.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294827.sc14.2671192.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294830.sc14.2671192.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294834.sc14.2671192.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294838.sc14.2671192.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294840.sc14.2671192.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294845.sc14.2671192.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294848.sc14.2671192.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294851.sc14.2671192.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294856.sc14.2671192.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294859.sc14.2671192.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294862.sc14.2671192.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294866.sc14.2671192.19 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295534.sc14.2696291.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295536.sc14.2696291.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295541.sc14.2696291.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295544.sc14.2696291.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295547.sc14.2696291.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295551.sc14.2696291.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295555.sc14.2696291.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295558.sc14.2696291.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295562.sc14.2696291.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295565.sc14.2696291.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295568.sc14.2696291.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295573.sc14.2696291.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295576.sc14.2696291.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295579.sc14.2696291.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295584.sc14.2696291.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295587.sc14.2696291.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295590.sc14.2696291.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295594.sc14.2696291.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295598.sc14.2696291.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295601.sc14.2696291.19 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296480.sc14.2714192.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296482.sc14.2714192.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296485.sc14.2714192.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296489.sc14.2714192.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296493.sc14.2714192.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296496.sc14.2714192.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296500.sc14.2714192.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296504.sc14.2714192.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296507.sc14.2714192.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296511.sc14.2714192.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296515.sc14.2714192.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296518.sc14.2714192.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296522.sc14.2714192.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296525.sc14.2714192.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296528.sc14.2714192.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296533.sc14.2714192.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296536.sc14.2714192.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296539.sc14.2714192.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296544.sc14.2714192.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297315.sc14.2733087.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297318.sc14.2733087.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297323.sc14.2733087.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297326.sc14.2733087.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297329.sc14.2733087.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297334.sc14.2733087.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297337.sc14.2733087.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297340.sc14.2733087.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297345.sc14.2733087.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297348.sc14.2733087.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297350.sc14.2733087.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297355.sc14.2733087.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297358.sc14.2733087.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297361.sc14.2733087.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297366.sc14.2733087.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297369.sc14.2733087.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297372.sc14.2733087.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297376.sc14.2733087.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297380.sc14.2733087.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297383.sc14.2733087.19 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298153.sc9.1458633.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298158.sc9.1458633.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298161.sc9.1458633.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298164.sc9.1458633.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298168.sc9.1458633.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298172.sc9.1458633.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298174.sc9.1458633.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298179.sc9.1458633.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298183.sc9.1458633.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298185.sc9.1458633.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298190.sc9.1458633.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298193.sc9.1458633.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298196.sc9.1458633.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298201.sc9.1458633.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298205.sc9.1458633.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298207.sc9.1458633.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298212.sc9.1458633.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298216.sc9.1458633.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298218.sc9.1458633.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298994.sc14.2774085.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298996.sc14.2774085.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298999.sc14.2774085.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299003.sc14.2774085.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299007.sc14.2774085.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299010.sc14.2774085.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299014.sc14.2774085.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299018.sc14.2774085.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299020.sc14.2774085.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299025.sc14.2774085.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299029.sc14.2774085.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299032.sc14.2774085.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299036.sc14.2774085.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299040.sc14.2774085.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299042.sc14.2774085.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299047.sc14.2774085.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299050.sc14.2774085.16 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299053.sc14.2774085.17 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299058.sc14.2774085.18 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299061.sc14.2774085.19 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299832.sc11.2623573.0 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299835.sc11.2623573.1 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299840.sc11.2623573.2 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299843.sc11.2623573.3 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299846.sc11.2623573.4 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299851.sc11.2623573.5 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299854.sc11.2623573.6 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299857.sc11.2623573.7 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299862.sc11.2623573.8 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299865.sc11.2623573.9 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299868.sc11.2623573.10 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299873.sc11.2623573.11 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299876.sc11.2623573.12 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299879.sc11.2623573.13 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299883.sc11.2623573.14 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299887.sc11.2623573.15 delete mode 100644 monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299890.sc11.2623573.16 create mode 100644 tv-detection/pyproject.toml create mode 100644 tv-segmentation/pyproject.toml diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_0.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_0.txt deleted file mode 100644 index 414e5290..00000000 --- a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_0.txt +++ /dev/null @@ -1,12925 +0,0 @@ -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 05:46:44 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 05:46:44 ] Completed importing Timer 0.024 ms, 0.00 s total -[ 2023-09-21 05:46:49 ] Completed importing everything else 4,759.109 ms, 4.76 s total -| distributed init (rank 5): env:// -| distributed init (rank 3): env:// -| distributed init (rank 1): env:// -| distributed init (rank 2): env:// -| distributed init (rank 4): env:// -| distributed init (rank 0): env:// -[ 2023-09-21 05:46:56 ] Completed preliminaries 7,462.666 ms, 12.22 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 05:46:56 ] Completed build datasets 11.769 ms, 12.23 s total -[ 2023-09-21 05:46:56 ] Completed build samplers 0.047 ms, 12.23 s total -[ 2023-09-21 05:46:56 ] Completed build dataloaders 0.134 ms, 12.23 s total -[ 2023-09-21 05:46:56 ] Completed generator to device 79.883 ms, 12.31 s total -[ 2023-09-21 05:46:56 ] Completed discriminator to device 7.031 ms, 12.32 s total -[ 2023-09-21 05:46:57 ] Completed loss functions 503.801 ms, 12.82 s total -[ 2023-09-21 05:46:57 ] Completed models prepped for distribution 38.852 ms, 12.86 s total -[ 2023-09-21 05:46:57 ] Completed optimizers 0.669 ms, 12.86 s total -[ 2023-09-21 05:46:57 ] Completed grad scalers 0.025 ms, 12.86 s total -[ 2023-09-21 05:46:57 ] Completed checkpoint retrieval 0.275 ms, 12.86 s total - - -EPOCH :: 0 - - -[ 2023-09-21 05:46:57 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 0 from training step 0 - -[ 2023-09-21 05:47:00 ] Completed train batch 0 to device 3,602.622 ms, 3.60 s total -[ 2023-09-21 05:47:02 ] Completed train batch 0 generator forward 1,421.216 ms, 5.02 s total -[ 2023-09-21 05:47:02 ] Completed train batch 0 generator loss: 2.480 87.080 ms, 5.11 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 05:47:03 ] Completed train batch 0 generator backward 1,006.777 ms, 6.12 s total -[ 2023-09-21 05:47:03 ] Completed train batch 0 discriminator loss 0.342 56.426 ms, 6.17 s total -[ 2023-09-21 05:47:03 ] Completed train batch 0 discriminator backward 101.381 ms, 6.28 s total -Epoch [0] Step [0/2], gen_loss: 0.827, disc_loss: 0.114 -[ 2023-09-21 05:47:03 ] Completed train batch 0 metrics update 6.901 ms, 6.28 s total -Saving checkpoint at epoch 0 train batch 0 -[ 2023-09-21 05:47:04 ] Completed saving temp checkpoint 433.835 ms, 6.72 s total -[ 2023-09-21 05:47:04 ] Completed replacing temp checkpoint with checkpoint 6.076 ms, 6.72 s total -[ 2023-09-21 05:47:04 ] Completed train batch 1 to device 125.666 ms, 6.85 s total -[ 2023-09-21 05:47:04 ] Completed train batch 1 generator forward 27.611 ms, 6.88 s total -[ 2023-09-21 05:47:04 ] Completed train batch 1 generator loss: 2.426 63.994 ms, 6.94 s total -[ 2023-09-21 05:47:05 ] Completed train batch 1 generator backward 1,094.595 ms, 8.03 s total -[ 2023-09-21 05:47:05 ] Completed train batch 1 discriminator loss 0.195 62.031 ms, 8.10 s total -[ 2023-09-21 05:47:05 ] Completed train batch 1 discriminator backward 58.162 ms, 8.15 s total -Epoch [0] Step [1/2], gen_loss: 0.807, disc_loss: 0.064 -[ 2023-09-21 05:47:05 ] Completed train batch 1 metrics update 5.524 ms, 8.16 s total -Saving checkpoint at epoch 0 train batch 1 -[ 2023-09-21 05:47:05 ] Completed saving temp checkpoint 476.097 ms, 8.64 s total -[ 2023-09-21 05:47:05 ] Completed replacing temp checkpoint with checkpoint 24.204 ms, 8.66 s total -Epoch [0] :: gen_loss: 0.817, disc_loss: 0.089 -[ 2023-09-21 05:47:06 ] Completed training generator for epoch 0 64.098 ms, 8.72 s total -[ 2023-09-21 05:47:06 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 0 from eval step 0 - -[ 2023-09-21 05:47:07 ] Completed eval batch 0 to device 1,133.913 ms, 1.13 s total -[ 2023-09-21 05:47:07 ] Completed eval batch 0 forward 44.514 ms, 1.18 s total -[ 2023-09-21 05:47:07 ] Completed eval batch 0 recons_loss 0.319 ms, 1.18 s total -[ 2023-09-21 05:47:07 ] Completed eval batch 0 metrics update 0.576 ms, 1.18 s total -Saving checkpoint at epoch 0 val batch 0 -[ 2023-09-21 05:47:07 ] Completed saving temp checkpoint 411.608 ms, 1.59 s total -[ 2023-09-21 05:47:07 ] Completed replacing temp checkpoint with checkpoint 25.754 ms, 1.62 s total -[ 2023-09-21 05:47:08 ] Completed eval batch 1 to device 364.721 ms, 1.98 s total -[ 2023-09-21 05:47:08 ] Completed eval batch 1 forward 39.759 ms, 2.02 s total -[ 2023-09-21 05:47:08 ] Completed eval batch 1 recons_loss 0.551 ms, 2.02 s total -[ 2023-09-21 05:47:08 ] Completed eval batch 1 metrics update 50.717 ms, 2.07 s total -Saving checkpoint at epoch 0 val batch 1 -Epoch 0 val loss: 0.4807 -[ 2023-09-21 05:47:08 ] Completed saving temp checkpoint 467.206 ms, 2.54 s total -[ 2023-09-21 05:47:08 ] Completed replacing temp checkpoint with checkpoint 21.521 ms, 2.56 s total -[ 2023-09-21 05:47:08 ] Completed evaluating generator for epoch 0 47.881 ms, 2.61 s total - - -EPOCH :: 1 - - -[ 2023-09-21 05:47:08 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 1 from training step 0 - -[ 2023-09-21 05:47:11 ] Completed train batch 0 to device 2,972.115 ms, 2.97 s total -[ 2023-09-21 05:47:11 ] Completed train batch 0 generator forward 33.176 ms, 3.01 s total -[ 2023-09-21 05:47:11 ] Completed train batch 0 generator loss: 2.501 66.624 ms, 3.07 s total -[ 2023-09-21 05:47:12 ] Completed train batch 0 generator backward 563.043 ms, 3.63 s total -[ 2023-09-21 05:47:12 ] Completed train batch 0 discriminator loss 0.142 74.985 ms, 3.71 s total -[ 2023-09-21 05:47:12 ] Completed train batch 0 discriminator backward 77.776 ms, 3.79 s total -Epoch [1] Step [0/2], gen_loss: 0.837, disc_loss: 0.047 -[ 2023-09-21 05:47:12 ] Completed train batch 0 metrics update 5.447 ms, 3.79 s total -Saving checkpoint at epoch 1 train batch 0 -[ 2023-09-21 05:47:12 ] Completed saving temp checkpoint 396.407 ms, 4.19 s total -[ 2023-09-21 05:47:12 ] Completed replacing temp checkpoint with checkpoint 30.443 ms, 4.22 s total -[ 2023-09-21 05:47:14 ] Completed train batch 1 to device 1,651.137 ms, 5.87 s total -[ 2023-09-21 05:47:14 ] Completed train batch 1 generator forward 41.709 ms, 5.91 s total -[ 2023-09-21 05:47:14 ] Completed train batch 1 generator loss: 2.302 69.503 ms, 5.98 s total -[ 2023-09-21 05:47:15 ] Completed train batch 1 generator backward 1,028.466 ms, 7.01 s total -[ 2023-09-21 05:47:15 ] Completed train batch 1 discriminator loss 0.128 56.712 ms, 7.07 s total -[ 2023-09-21 05:47:15 ] Completed train batch 1 discriminator backward 71.761 ms, 7.14 s total -Epoch [1] Step [1/2], gen_loss: 0.766, disc_loss: 0.042 -[ 2023-09-21 05:47:15 ] Completed train batch 1 metrics update 5.404 ms, 7.14 s total -Saving checkpoint at epoch 1 train batch 1 -[ 2023-09-21 05:47:16 ] Completed saving temp checkpoint 449.338 ms, 7.59 s total -[ 2023-09-21 05:47:16 ] Completed replacing temp checkpoint with checkpoint 32.826 ms, 7.63 s total -Epoch [1] :: gen_loss: 0.801, disc_loss: 0.045 -[ 2023-09-21 05:47:16 ] Completed training generator for epoch 1 52.284 ms, 7.68 s total -[ 2023-09-21 05:47:16 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 1 from eval step 0 - -[ 2023-09-21 05:47:17 ] Completed eval batch 0 to device 1,123.771 ms, 1.12 s total -[ 2023-09-21 05:47:17 ] Completed eval batch 0 forward 43.463 ms, 1.17 s total -[ 2023-09-21 05:47:17 ] Completed eval batch 0 recons_loss 0.361 ms, 1.17 s total -[ 2023-09-21 05:47:17 ] Completed eval batch 0 metrics update 0.577 ms, 1.17 s total -Saving checkpoint at epoch 1 val batch 0 -[ 2023-09-21 05:47:17 ] Completed saving temp checkpoint 393.795 ms, 1.56 s total -[ 2023-09-21 05:47:17 ] Completed replacing temp checkpoint with checkpoint 29.655 ms, 1.59 s total -[ 2023-09-21 05:47:18 ] Completed eval batch 1 to device 413.625 ms, 2.01 s total -[ 2023-09-21 05:47:18 ] Completed eval batch 1 forward 39.775 ms, 2.05 s total -[ 2023-09-21 05:47:18 ] Completed eval batch 1 recons_loss 0.605 ms, 2.05 s total -[ 2023-09-21 05:47:18 ] Completed eval batch 1 metrics update 134.619 ms, 2.18 s total -Saving checkpoint at epoch 1 val batch 1 -Epoch 1 val loss: 0.3936 -[ 2023-09-21 05:47:18 ] Completed saving temp checkpoint 470.263 ms, 2.65 s total -[ 2023-09-21 05:47:19 ] Completed replacing temp checkpoint with checkpoint 20.152 ms, 2.67 s total -[ 2023-09-21 05:47:19 ] Completed evaluating generator for epoch 1 45.420 ms, 2.72 s total - - -EPOCH :: 2 - - -[ 2023-09-21 05:47:19 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 2 from training step 0 - -[ 2023-09-21 05:47:22 ] Completed train batch 0 to device 2,986.176 ms, 2.99 s total -[ 2023-09-21 05:47:22 ] Completed train batch 0 generator forward 45.010 ms, 3.03 s total -[ 2023-09-21 05:47:22 ] Completed train batch 0 generator loss: nan 66.154 ms, 3.10 s total -[ 2023-09-21 05:47:22 ] Completed train batch 0 generator backward 546.607 ms, 3.64 s total -[ 2023-09-21 05:47:22 ] Completed train batch 0 discriminator loss 0.122 56.990 ms, 3.70 s total -[ 2023-09-21 05:47:22 ] Completed train batch 0 discriminator backward 93.706 ms, 3.79 s total -Epoch [2] Step [0/2], gen_loss: nan, disc_loss: 0.039 -[ 2023-09-21 05:47:22 ] Completed train batch 0 metrics update 12.546 ms, 3.81 s total -Saving checkpoint at epoch 2 train batch 0 -[ 2023-09-21 05:47:23 ] Completed saving temp checkpoint 483.371 ms, 4.29 s total -[ 2023-09-21 05:47:23 ] Completed replacing temp checkpoint with checkpoint 20.450 ms, 4.31 s total -[ 2023-09-21 05:47:24 ] Completed train batch 1 to device 1,409.171 ms, 5.72 s total -[ 2023-09-21 05:47:24 ] Completed train batch 1 generator forward 40.428 ms, 5.76 s total -[ 2023-09-21 05:47:24 ] Completed train batch 1 generator loss: nan 63.653 ms, 5.82 s total -[ 2023-09-21 05:47:26 ] Completed train batch 1 generator backward 1,392.355 ms, 7.22 s total -[ 2023-09-21 05:47:26 ] Completed train batch 1 discriminator loss 0.102 56.399 ms, 7.27 s total -[ 2023-09-21 05:47:26 ] Completed train batch 1 discriminator backward 66.305 ms, 7.34 s total -Epoch [2] Step [1/2], gen_loss: nan, disc_loss: 0.034 -[ 2023-09-21 05:47:26 ] Completed train batch 1 metrics update 5.850 ms, 7.35 s total -Saving checkpoint at epoch 2 train batch 1 -[ 2023-09-21 05:47:26 ] Completed saving temp checkpoint 482.316 ms, 7.83 s total -[ 2023-09-21 05:47:26 ] Completed replacing temp checkpoint with checkpoint 35.606 ms, 7.86 s total -Epoch [2] :: gen_loss: nan, disc_loss: 0.036 -[ 2023-09-21 05:47:26 ] Completed training generator for epoch 2 66.120 ms, 7.93 s total -[ 2023-09-21 05:47:26 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 2 from eval step 0 - -[ 2023-09-21 05:47:28 ] Completed eval batch 0 to device 1,094.025 ms, 1.09 s total -[ 2023-09-21 05:47:28 ] Completed eval batch 0 forward 28.024 ms, 1.12 s total -[ 2023-09-21 05:47:28 ] Completed eval batch 0 recons_loss 0.311 ms, 1.12 s total -[ 2023-09-21 05:47:28 ] Completed eval batch 0 metrics update 0.551 ms, 1.12 s total -Saving checkpoint at epoch 2 val batch 0 -[ 2023-09-21 05:47:28 ] Completed saving temp checkpoint 494.025 ms, 1.62 s total -[ 2023-09-21 05:47:28 ] Completed replacing temp checkpoint with checkpoint 33.083 ms, 1.65 s total -[ 2023-09-21 05:47:28 ] Completed eval batch 1 to device 308.368 ms, 1.96 s total -[ 2023-09-21 05:47:28 ] Completed eval batch 1 forward 39.712 ms, 2.00 s total -[ 2023-09-21 05:47:28 ] Completed eval batch 1 recons_loss 0.557 ms, 2.00 s total -[ 2023-09-21 05:47:29 ] Completed eval batch 1 metrics update 236.971 ms, 2.24 s total -Saving checkpoint at epoch 2 val batch 1 -Epoch 2 val loss: 0.3945 -[ 2023-09-21 05:47:29 ] Completed saving temp checkpoint 683.919 ms, 2.92 s total -[ 2023-09-21 05:47:29 ] Completed replacing temp checkpoint with checkpoint 41.794 ms, 2.96 s total -[ 2023-09-21 05:47:30 ] Completed evaluating generator for epoch 2 64.710 ms, 3.03 s total - - -EPOCH :: 3 - - -[ 2023-09-21 05:47:30 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 3 from training step 0 - -[ 2023-09-21 05:47:32 ] Completed train batch 0 to device 2,987.141 ms, 2.99 s total -[ 2023-09-21 05:47:33 ] Completed train batch 0 generator forward 29.920 ms, 3.02 s total -[ 2023-09-21 05:47:33 ] Completed train batch 0 generator loss: nan 66.475 ms, 3.08 s total -[ 2023-09-21 05:47:33 ] Completed train batch 0 generator backward 483.431 ms, 3.57 s total -[ 2023-09-21 05:47:33 ] Completed train batch 0 discriminator loss 0.097 64.411 ms, 3.63 s total -[ 2023-09-21 05:47:33 ] Completed train batch 0 discriminator backward 84.491 ms, 3.72 s total -Epoch [3] Step [0/2], gen_loss: nan, disc_loss: 0.031 -[ 2023-09-21 05:47:33 ] Completed train batch 0 metrics update 4.911 ms, 3.72 s total -Saving checkpoint at epoch 3 train batch 0 -[ 2023-09-21 05:47:34 ] Completed saving temp checkpoint 441.548 ms, 4.16 s total -[ 2023-09-21 05:47:34 ] Completed replacing temp checkpoint with checkpoint 17.724 ms, 4.18 s total -[ 2023-09-21 05:47:35 ] Completed train batch 1 to device 1,596.127 ms, 5.78 s total -[ 2023-09-21 05:47:35 ] Completed train batch 1 generator forward 24.086 ms, 5.80 s total -[ 2023-09-21 05:47:35 ] Completed train batch 1 generator loss: nan 64.102 ms, 5.86 s total -[ 2023-09-21 05:47:37 ] Completed train batch 1 generator backward 1,181.406 ms, 7.05 s total -[ 2023-09-21 05:47:37 ] Completed train batch 1 discriminator loss 0.074 56.884 ms, 7.10 s total -[ 2023-09-21 05:47:37 ] Completed train batch 1 discriminator backward 77.488 ms, 7.18 s total -Epoch [3] Step [1/2], gen_loss: nan, disc_loss: 0.026 -[ 2023-09-21 05:47:37 ] Completed train batch 1 metrics update 5.155 ms, 7.19 s total -Saving checkpoint at epoch 3 train batch 1 -[ 2023-09-21 05:47:37 ] Completed saving temp checkpoint 473.955 ms, 7.66 s total -[ 2023-09-21 05:47:37 ] Completed replacing temp checkpoint with checkpoint 18.245 ms, 7.68 s total -Epoch [3] :: gen_loss: nan, disc_loss: 0.028 -[ 2023-09-21 05:47:37 ] Completed training generator for epoch 3 52.778 ms, 7.73 s total -[ 2023-09-21 05:47:37 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 3 from eval step 0 - -[ 2023-09-21 05:47:38 ] Completed eval batch 0 to device 1,113.652 ms, 1.11 s total -[ 2023-09-21 05:47:38 ] Completed eval batch 0 forward 28.587 ms, 1.14 s total -[ 2023-09-21 05:47:38 ] Completed eval batch 0 recons_loss 0.305 ms, 1.14 s total -[ 2023-09-21 05:47:38 ] Completed eval batch 0 metrics update 0.564 ms, 1.14 s total -Saving checkpoint at epoch 3 val batch 0 -[ 2023-09-21 05:47:39 ] Completed saving temp checkpoint 435.592 ms, 1.58 s total -[ 2023-09-21 05:47:39 ] Completed replacing temp checkpoint with checkpoint 18.484 ms, 1.60 s total -[ 2023-09-21 05:47:39 ] Completed eval batch 1 to device 452.839 ms, 2.05 s total -[ 2023-09-21 05:47:39 ] Completed eval batch 1 forward 21.929 ms, 2.07 s total -[ 2023-09-21 05:47:39 ] Completed eval batch 1 recons_loss 0.401 ms, 2.07 s total -[ 2023-09-21 05:47:39 ] Completed eval batch 1 metrics update 154.161 ms, 2.23 s total -Saving checkpoint at epoch 3 val batch 1 -Epoch 3 val loss: 0.3858 -[ 2023-09-21 05:47:40 ] Completed saving temp checkpoint 452.450 ms, 2.68 s total -[ 2023-09-21 05:47:40 ] Completed replacing temp checkpoint with checkpoint 25.877 ms, 2.70 s total -[ 2023-09-21 05:47:40 ] Completed evaluating generator for epoch 3 44.876 ms, 2.75 s total - - -EPOCH :: 4 - - -[ 2023-09-21 05:47:40 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 4 from training step 0 - -[ 2023-09-21 05:47:43 ] Completed train batch 0 to device 3,034.499 ms, 3.03 s total -[ 2023-09-21 05:47:43 ] Completed train batch 0 generator forward 34.966 ms, 3.07 s total -[ 2023-09-21 05:47:43 ] Completed train batch 0 generator loss: nan 77.904 ms, 3.15 s total -[ 2023-09-21 05:47:44 ] Completed train batch 0 generator backward 625.718 ms, 3.77 s total -[ 2023-09-21 05:47:44 ] Completed train batch 0 discriminator loss 0.063 63.370 ms, 3.84 s total -[ 2023-09-21 05:47:44 ] Completed train batch 0 discriminator backward 87.465 ms, 3.92 s total -Epoch [4] Step [0/2], gen_loss: nan, disc_loss: 0.021 -[ 2023-09-21 05:47:44 ] Completed train batch 0 metrics update 5.202 ms, 3.93 s total -Saving checkpoint at epoch 4 train batch 0 -[ 2023-09-21 05:47:44 ] Completed saving temp checkpoint 461.724 ms, 4.39 s total -[ 2023-09-21 05:47:44 ] Completed replacing temp checkpoint with checkpoint 33.478 ms, 4.42 s total -[ 2023-09-21 05:47:46 ] Completed train batch 1 to device 1,426.859 ms, 5.85 s total -[ 2023-09-21 05:47:46 ] Completed train batch 1 generator forward 41.331 ms, 5.89 s total -[ 2023-09-21 05:47:46 ] Completed train batch 1 generator loss: nan 64.118 ms, 5.96 s total -[ 2023-09-21 05:47:47 ] Completed train batch 1 generator backward 833.377 ms, 6.79 s total -[ 2023-09-21 05:47:47 ] Completed train batch 1 discriminator loss 0.047 56.337 ms, 6.85 s total -[ 2023-09-21 05:47:47 ] Completed train batch 1 discriminator backward 76.802 ms, 6.92 s total -Epoch [4] Step [1/2], gen_loss: nan, disc_loss: 0.016 -[ 2023-09-21 05:47:47 ] Completed train batch 1 metrics update 5.243 ms, 6.93 s total -Saving checkpoint at epoch 4 train batch 1 -[ 2023-09-21 05:47:47 ] Completed saving temp checkpoint 485.188 ms, 7.41 s total -[ 2023-09-21 05:47:47 ] Completed replacing temp checkpoint with checkpoint 32.691 ms, 7.45 s total -Epoch [4] :: gen_loss: nan, disc_loss: 0.018 -[ 2023-09-21 05:47:47 ] Completed training generator for epoch 4 54.131 ms, 7.50 s total -[ 2023-09-21 05:47:47 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 4 from eval step 0 - -[ 2023-09-21 05:47:49 ] Completed eval batch 0 to device 1,057.164 ms, 1.06 s total -[ 2023-09-21 05:47:49 ] Completed eval batch 0 forward 24.136 ms, 1.08 s total -[ 2023-09-21 05:47:49 ] Completed eval batch 0 recons_loss 0.227 ms, 1.08 s total -[ 2023-09-21 05:47:49 ] Completed eval batch 0 metrics update 0.487 ms, 1.08 s total -Saving checkpoint at epoch 4 val batch 0 -[ 2023-09-21 05:47:49 ] Completed saving temp checkpoint 459.566 ms, 1.54 s total -[ 2023-09-21 05:47:49 ] Completed replacing temp checkpoint with checkpoint 27.068 ms, 1.57 s total -[ 2023-09-21 05:47:49 ] Completed eval batch 1 to device 371.450 ms, 1.94 s total -[ 2023-09-21 05:47:49 ] Completed eval batch 1 forward 37.941 ms, 1.98 s total -[ 2023-09-21 05:47:49 ] Completed eval batch 1 recons_loss 0.344 ms, 1.98 s total -[ 2023-09-21 05:47:50 ] Completed eval batch 1 metrics update 266.220 ms, 2.24 s total -Saving checkpoint at epoch 4 val batch 1 -Epoch 4 val loss: 0.3746 -[ 2023-09-21 05:47:50 ] Completed saving temp checkpoint 455.324 ms, 2.70 s total -[ 2023-09-21 05:47:50 ] Completed replacing temp checkpoint with checkpoint 25.711 ms, 2.73 s total -[ 2023-09-21 05:47:50 ] Completed evaluating generator for epoch 4 51.014 ms, 2.78 s total - - -EPOCH :: 5 - - -[ 2023-09-21 05:47:50 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 5 from training step 0 - -[ 2023-09-21 05:47:53 ] Completed train batch 0 to device 2,978.069 ms, 2.98 s total -[ 2023-09-21 05:47:53 ] Completed train batch 0 generator forward 44.741 ms, 3.02 s total -[ 2023-09-21 05:47:53 ] Completed train batch 0 generator loss: nan 66.353 ms, 3.09 s total -[ 2023-09-21 05:47:54 ] Completed train batch 0 generator backward 607.318 ms, 3.70 s total -[ 2023-09-21 05:47:54 ] Completed train batch 0 discriminator loss 0.040 61.459 ms, 3.76 s total -[ 2023-09-21 05:47:54 ] Completed train batch 0 discriminator backward 84.707 ms, 3.84 s total -Epoch [5] Step [0/2], gen_loss: nan, disc_loss: 0.013 -[ 2023-09-21 05:47:54 ] Completed train batch 0 metrics update 6.009 ms, 3.85 s total -Saving checkpoint at epoch 5 train batch 0 -[ 2023-09-21 05:47:55 ] Completed saving temp checkpoint 486.331 ms, 4.33 s total -[ 2023-09-21 05:47:55 ] Completed replacing temp checkpoint with checkpoint 33.656 ms, 4.37 s total -[ 2023-09-21 05:47:56 ] Completed train batch 1 to device 1,476.606 ms, 5.85 s total -[ 2023-09-21 05:47:56 ] Completed train batch 1 generator forward 42.024 ms, 5.89 s total -[ 2023-09-21 05:47:56 ] Completed train batch 1 generator loss: nan 63.460 ms, 5.95 s total -[ 2023-09-21 05:47:57 ] Completed train batch 1 generator backward 1,190.381 ms, 7.14 s total -[ 2023-09-21 05:47:57 ] Completed train batch 1 discriminator loss 0.036 56.555 ms, 7.20 s total -[ 2023-09-21 05:47:58 ] Completed train batch 1 discriminator backward 63.022 ms, 7.26 s total -Epoch [5] Step [1/2], gen_loss: nan, disc_loss: 0.012 -[ 2023-09-21 05:47:58 ] Completed train batch 1 metrics update 1.791 ms, 7.26 s total -Saving checkpoint at epoch 5 train batch 1 -[ 2023-09-21 05:47:58 ] Completed saving temp checkpoint 769.802 ms, 8.03 s total -[ 2023-09-21 05:47:58 ] Completed replacing temp checkpoint with checkpoint 31.174 ms, 8.06 s total -Epoch [5] :: gen_loss: nan, disc_loss: 0.013 -[ 2023-09-21 05:47:58 ] Completed training generator for epoch 5 54.946 ms, 8.12 s total -[ 2023-09-21 05:47:58 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 5 from eval step 0 - -[ 2023-09-21 05:48:00 ] Completed eval batch 0 to device 1,164.036 ms, 1.16 s total -[ 2023-09-21 05:48:00 ] Completed eval batch 0 forward 41.934 ms, 1.21 s total -[ 2023-09-21 05:48:00 ] Completed eval batch 0 recons_loss 0.216 ms, 1.21 s total -[ 2023-09-21 05:48:00 ] Completed eval batch 0 metrics update 0.484 ms, 1.21 s total -Saving checkpoint at epoch 5 val batch 0 -[ 2023-09-21 05:48:00 ] Completed saving temp checkpoint 405.120 ms, 1.61 s total -[ 2023-09-21 05:48:00 ] Completed replacing temp checkpoint with checkpoint 27.864 ms, 1.64 s total -[ 2023-09-21 05:48:00 ] Completed eval batch 1 to device 341.629 ms, 1.98 s total -[ 2023-09-21 05:48:00 ] Completed eval batch 1 forward 27.358 ms, 2.01 s total -[ 2023-09-21 05:48:00 ] Completed eval batch 1 recons_loss 0.365 ms, 2.01 s total -[ 2023-09-21 05:48:00 ] Completed eval batch 1 metrics update 0.319 ms, 2.01 s total -Saving checkpoint at epoch 5 val batch 1 -Epoch 5 val loss: 0.3652 -[ 2023-09-21 05:48:01 ] Completed saving temp checkpoint 459.710 ms, 2.47 s total -[ 2023-09-21 05:48:01 ] Completed replacing temp checkpoint with checkpoint 19.135 ms, 2.49 s total -[ 2023-09-21 05:48:01 ] Completed evaluating generator for epoch 5 51.450 ms, 2.54 s total - - -EPOCH :: 6 - - -[ 2023-09-21 05:48:01 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 6 from training step 0 - -[ 2023-09-21 05:48:04 ] Completed train batch 0 to device 3,051.714 ms, 3.05 s total -[ 2023-09-21 05:48:04 ] Completed train batch 0 generator forward 27.535 ms, 3.08 s total -[ 2023-09-21 05:48:04 ] Completed train batch 0 generator loss: 2.529 67.139 ms, 3.15 s total -[ 2023-09-21 05:48:05 ] Completed train batch 0 generator backward 565.106 ms, 3.71 s total -[ 2023-09-21 05:48:05 ] Completed train batch 0 discriminator loss 0.033 79.844 ms, 3.79 s total -[ 2023-09-21 05:48:05 ] Completed train batch 0 discriminator backward 69.594 ms, 3.86 s total -Epoch [6] Step [0/2], gen_loss: nan, disc_loss: 0.011 -[ 2023-09-21 05:48:05 ] Completed train batch 0 metrics update 0.817 ms, 3.86 s total -Saving checkpoint at epoch 6 train batch 0 -[ 2023-09-21 05:48:05 ] Completed saving temp checkpoint 455.497 ms, 4.32 s total -[ 2023-09-21 05:48:05 ] Completed replacing temp checkpoint with checkpoint 17.827 ms, 4.34 s total -[ 2023-09-21 05:48:07 ] Completed train batch 1 to device 1,413.333 ms, 5.75 s total -[ 2023-09-21 05:48:07 ] Completed train batch 1 generator forward 22.135 ms, 5.77 s total -[ 2023-09-21 05:48:07 ] Completed train batch 1 generator loss: 2.525 63.923 ms, 5.83 s total -[ 2023-09-21 05:48:08 ] Completed train batch 1 generator backward 1,410.301 ms, 7.24 s total -[ 2023-09-21 05:48:08 ] Completed train batch 1 discriminator loss 0.036 57.864 ms, 7.30 s total -[ 2023-09-21 05:48:08 ] Completed train batch 1 discriminator backward 56.641 ms, 7.36 s total -Epoch [6] Step [1/2], gen_loss: 0.840, disc_loss: 0.012 -[ 2023-09-21 05:48:08 ] Completed train batch 1 metrics update 0.922 ms, 7.36 s total -Saving checkpoint at epoch 6 train batch 1 -[ 2023-09-21 05:48:09 ] Completed saving temp checkpoint 460.122 ms, 7.82 s total -[ 2023-09-21 05:48:09 ] Completed replacing temp checkpoint with checkpoint 27.414 ms, 7.85 s total -Epoch [6] :: gen_loss: nan, disc_loss: 0.012 -[ 2023-09-21 05:48:09 ] Completed training generator for epoch 6 67.314 ms, 7.92 s total -[ 2023-09-21 05:48:09 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 6 from eval step 0 - -[ 2023-09-21 05:48:10 ] Completed eval batch 0 to device 1,080.185 ms, 1.08 s total -[ 2023-09-21 05:48:10 ] Completed eval batch 0 forward 43.646 ms, 1.12 s total -[ 2023-09-21 05:48:10 ] Completed eval batch 0 recons_loss 0.311 ms, 1.12 s total -[ 2023-09-21 05:48:10 ] Completed eval batch 0 metrics update 0.599 ms, 1.12 s total -Saving checkpoint at epoch 6 val batch 0 -[ 2023-09-21 05:48:10 ] Completed saving temp checkpoint 428.917 ms, 1.55 s total -[ 2023-09-21 05:48:10 ] Completed replacing temp checkpoint with checkpoint 35.399 ms, 1.59 s total -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 05:59:18 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 05:59:18 ] Completed importing Timer 0.023 ms, 0.00 s total -[ 2023-09-21 05:59:23 ] Completed importing everything else 4,955.559 ms, 4.96 s total -| distributed init (rank 0): env:// -| distributed init (rank 3): env:// -| distributed init (rank 2): env:// -| distributed init (rank 1): env:// -| distributed init (rank 4): env:// -| distributed init (rank 5): env:// -[ 2023-09-21 05:59:31 ] Completed preliminaries 7,795.846 ms, 12.75 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 05:59:31 ] Completed build datasets 11.824 ms, 12.76 s total -[ 2023-09-21 05:59:31 ] Completed build samplers 0.046 ms, 12.76 s total -[ 2023-09-21 05:59:31 ] Completed build dataloaders 0.134 ms, 12.76 s total -[ 2023-09-21 05:59:31 ] Completed generator to device 74.621 ms, 12.84 s total -[ 2023-09-21 05:59:31 ] Completed discriminator to device 7.130 ms, 12.85 s total -[ 2023-09-21 05:59:32 ] Completed loss functions 412.476 ms, 13.26 s total -[ 2023-09-21 05:59:32 ] Completed models prepped for distribution 69.399 ms, 13.33 s total -[ 2023-09-21 05:59:32 ] Completed optimizers 0.682 ms, 13.33 s total -[ 2023-09-21 05:59:32 ] Completed grad scalers 0.037 ms, 13.33 s total -[ 2023-09-21 05:59:32 ] Completed checkpoint retrieval 438.620 ms, 13.77 s total - - -EPOCH :: 6 - - -[ 2023-09-21 05:59:32 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 6 from training step 2 - -Epoch [6] :: gen_loss: nan, disc_loss: 0.012 -[ 2023-09-21 05:59:32 ] Completed training generator for epoch 6 132.440 ms, 0.13 s total -[ 2023-09-21 05:59:32 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 6 from eval step 1 - -[ 2023-09-21 05:59:34 ] Completed eval batch 1 to device 1,321.682 ms, 1.32 s total -[ 2023-09-21 05:59:35 ] Completed eval batch 1 forward 1,167.208 ms, 2.49 s total -[ 2023-09-21 05:59:35 ] Completed eval batch 1 recons_loss 0.266 ms, 2.49 s total -[ 2023-09-21 05:59:35 ] Completed eval batch 1 metrics update 404.749 ms, 2.89 s total -Saving checkpoint at epoch 6 val batch 1 -Epoch 6 val loss: 0.2630 -[ 2023-09-21 05:59:36 ] Completed saving temp checkpoint 466.935 ms, 3.36 s total -[ 2023-09-21 05:59:36 ] Completed replacing temp checkpoint with checkpoint 134.088 ms, 3.49 s total -[ 2023-09-21 05:59:36 ] Completed evaluating generator for epoch 6 48.808 ms, 3.54 s total - - -EPOCH :: 7 - - -[ 2023-09-21 05:59:36 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 7 from training step 0 - -[ 2023-09-21 05:59:39 ] Completed train batch 0 to device 3,009.194 ms, 3.01 s total -[ 2023-09-21 05:59:39 ] Completed train batch 0 generator forward 70.285 ms, 3.08 s total -[ 2023-09-21 05:59:39 ] Completed train batch 0 generator loss: 2.295 81.576 ms, 3.16 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 05:59:40 ] Completed train batch 0 generator backward 685.762 ms, 3.85 s total -[ 2023-09-21 05:59:40 ] Completed train batch 0 discriminator loss 0.120 58.438 ms, 3.91 s total -[ 2023-09-21 05:59:40 ] Completed train batch 0 discriminator backward 135.834 ms, 4.04 s total -Epoch [7] Step [0/2], gen_loss: 0.769, disc_loss: 0.040 -[ 2023-09-21 05:59:40 ] Completed train batch 0 metrics update 0.744 ms, 4.04 s total -Saving checkpoint at epoch 7 train batch 0 -[ 2023-09-21 05:59:40 ] Completed saving temp checkpoint 399.034 ms, 4.44 s total -[ 2023-09-21 05:59:40 ] Completed replacing temp checkpoint with checkpoint 21.614 ms, 4.46 s total -[ 2023-09-21 05:59:42 ] Completed train batch 1 to device 1,383.704 ms, 5.85 s total -[ 2023-09-21 05:59:42 ] Completed train batch 1 generator forward 41.533 ms, 5.89 s total -[ 2023-09-21 05:59:42 ] Completed train batch 1 generator loss: 2.237 63.407 ms, 5.95 s total -[ 2023-09-21 05:59:43 ] Completed train batch 1 generator backward 1,477.580 ms, 7.43 s total -[ 2023-09-21 05:59:43 ] Completed train batch 1 discriminator loss 0.294 56.625 ms, 7.49 s total -[ 2023-09-21 05:59:43 ] Completed train batch 1 discriminator backward 77.874 ms, 7.56 s total -Epoch [7] Step [1/2], gen_loss: 0.750, disc_loss: 0.098 -[ 2023-09-21 05:59:43 ] Completed train batch 1 metrics update 2.224 ms, 7.57 s total -Saving checkpoint at epoch 7 train batch 1 -[ 2023-09-21 05:59:44 ] Completed saving temp checkpoint 421.899 ms, 7.99 s total -[ 2023-09-21 05:59:44 ] Completed replacing temp checkpoint with checkpoint 19.280 ms, 8.01 s total -Epoch [7] :: gen_loss: 0.759, disc_loss: 0.069 -[ 2023-09-21 05:59:44 ] Completed training generator for epoch 7 47.638 ms, 8.05 s total -[ 2023-09-21 05:59:44 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 7 from eval step 0 - -[ 2023-09-21 05:59:45 ] Completed eval batch 0 to device 1,177.301 ms, 1.18 s total -[ 2023-09-21 05:59:45 ] Completed eval batch 0 forward 30.266 ms, 1.21 s total -[ 2023-09-21 05:59:45 ] Completed eval batch 0 recons_loss 0.335 ms, 1.21 s total -[ 2023-09-21 05:59:45 ] Completed eval batch 0 metrics update 0.606 ms, 1.21 s total -Saving checkpoint at epoch 7 val batch 0 -[ 2023-09-21 05:59:46 ] Completed saving temp checkpoint 398.766 ms, 1.61 s total -[ 2023-09-21 05:59:46 ] Completed replacing temp checkpoint with checkpoint 21.775 ms, 1.63 s total -[ 2023-09-21 05:59:46 ] Completed eval batch 1 to device 447.385 ms, 2.08 s total -[ 2023-09-21 05:59:46 ] Completed eval batch 1 forward 40.043 ms, 2.12 s total -[ 2023-09-21 05:59:46 ] Completed eval batch 1 recons_loss 0.591 ms, 2.12 s total -[ 2023-09-21 05:59:46 ] Completed eval batch 1 metrics update 297.923 ms, 2.41 s total -Saving checkpoint at epoch 7 val batch 1 -Epoch 7 val loss: 0.2485 -[ 2023-09-21 05:59:47 ] Completed saving temp checkpoint 415.167 ms, 2.83 s total -[ 2023-09-21 05:59:47 ] Completed replacing temp checkpoint with checkpoint 31.136 ms, 2.86 s total -[ 2023-09-21 05:59:47 ] Completed evaluating generator for epoch 7 48.140 ms, 2.91 s total - - -EPOCH :: 8 - - -[ 2023-09-21 05:59:47 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 8 from training step 0 - -[ 2023-09-21 05:59:50 ] Completed train batch 0 to device 3,099.081 ms, 3.10 s total -[ 2023-09-21 05:59:50 ] Completed train batch 0 generator forward 44.707 ms, 3.14 s total -[ 2023-09-21 05:59:50 ] Completed train batch 0 generator loss: 2.214 66.533 ms, 3.21 s total -[ 2023-09-21 05:59:51 ] Completed train batch 0 generator backward 701.440 ms, 3.91 s total -[ 2023-09-21 05:59:51 ] Completed train batch 0 discriminator loss 0.248 58.150 ms, 3.97 s total -[ 2023-09-21 05:59:51 ] Completed train batch 0 discriminator backward 106.197 ms, 4.08 s total -Epoch [8] Step [0/2], gen_loss: 0.730, disc_loss: 0.083 -[ 2023-09-21 05:59:51 ] Completed train batch 0 metrics update 3.011 ms, 4.08 s total -Saving checkpoint at epoch 8 train batch 0 -[ 2023-09-21 05:59:51 ] Completed saving temp checkpoint 405.351 ms, 4.48 s total -[ 2023-09-21 05:59:51 ] Completed replacing temp checkpoint with checkpoint 23.332 ms, 4.51 s total -[ 2023-09-21 05:59:53 ] Completed train batch 1 to device 1,541.309 ms, 6.05 s total -[ 2023-09-21 05:59:53 ] Completed train batch 1 generator forward 22.959 ms, 6.07 s total -[ 2023-09-21 05:59:53 ] Completed train batch 1 generator loss: 2.356 64.554 ms, 6.14 s total -[ 2023-09-21 05:59:54 ] Completed train batch 1 generator backward 1,474.490 ms, 7.61 s total -[ 2023-09-21 05:59:55 ] Completed train batch 1 discriminator loss 0.197 56.802 ms, 7.67 s total -[ 2023-09-21 05:59:55 ] Completed train batch 1 discriminator backward 82.041 ms, 7.75 s total -Epoch [8] Step [1/2], gen_loss: 0.782, disc_loss: 0.066 -[ 2023-09-21 05:59:55 ] Completed train batch 1 metrics update 5.189 ms, 7.76 s total -Saving checkpoint at epoch 8 train batch 1 -[ 2023-09-21 05:59:55 ] Completed saving temp checkpoint 422.054 ms, 8.18 s total -[ 2023-09-21 05:59:55 ] Completed replacing temp checkpoint with checkpoint 15.549 ms, 8.19 s total -Epoch [8] :: gen_loss: 0.756, disc_loss: 0.074 -[ 2023-09-21 05:59:55 ] Completed training generator for epoch 8 54.933 ms, 8.25 s total -[ 2023-09-21 05:59:55 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 8 from eval step 0 - -[ 2023-09-21 05:59:56 ] Completed eval batch 0 to device 1,257.212 ms, 1.26 s total -[ 2023-09-21 05:59:56 ] Completed eval batch 0 forward 24.573 ms, 1.28 s total -[ 2023-09-21 05:59:56 ] Completed eval batch 0 recons_loss 0.271 ms, 1.28 s total -[ 2023-09-21 05:59:56 ] Completed eval batch 0 metrics update 0.557 ms, 1.28 s total -Saving checkpoint at epoch 8 val batch 0 -[ 2023-09-21 05:59:57 ] Completed saving temp checkpoint 393.026 ms, 1.68 s total -[ 2023-09-21 05:59:57 ] Completed replacing temp checkpoint with checkpoint 24.425 ms, 1.70 s total -[ 2023-09-21 05:59:57 ] Completed eval batch 1 to device 572.233 ms, 2.27 s total -[ 2023-09-21 05:59:57 ] Completed eval batch 1 forward 22.035 ms, 2.29 s total -[ 2023-09-21 05:59:57 ] Completed eval batch 1 recons_loss 0.386 ms, 2.29 s total -[ 2023-09-21 05:59:57 ] Completed eval batch 1 metrics update 82.575 ms, 2.38 s total -Saving checkpoint at epoch 8 val batch 1 -Epoch 8 val loss: 0.4310 -[ 2023-09-21 05:59:58 ] Completed saving temp checkpoint 419.567 ms, 2.80 s total -[ 2023-09-21 05:59:58 ] Completed replacing temp checkpoint with checkpoint 25.248 ms, 2.82 s total -[ 2023-09-21 05:59:58 ] Completed evaluating generator for epoch 8 48.283 ms, 2.87 s total - - -EPOCH :: 9 - - -[ 2023-09-21 05:59:58 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 9 from training step 0 - -[ 2023-09-21 06:00:01 ] Completed train batch 0 to device 3,289.524 ms, 3.29 s total -[ 2023-09-21 06:00:01 ] Completed train batch 0 generator forward 27.593 ms, 3.32 s total -[ 2023-09-21 06:00:01 ] Completed train batch 0 generator loss: 2.384 67.243 ms, 3.38 s total -[ 2023-09-21 06:00:02 ] Completed train batch 0 generator backward 583.331 ms, 3.97 s total -[ 2023-09-21 06:00:02 ] Completed train batch 0 discriminator loss 0.172 57.930 ms, 4.03 s total -[ 2023-09-21 06:00:02 ] Completed train batch 0 discriminator backward 97.708 ms, 4.12 s total -Epoch [9] Step [0/2], gen_loss: 0.793, disc_loss: 0.057 -[ 2023-09-21 06:00:02 ] Completed train batch 0 metrics update 7.794 ms, 4.13 s total -Saving checkpoint at epoch 9 train batch 0 -[ 2023-09-21 06:00:03 ] Completed saving temp checkpoint 453.248 ms, 4.58 s total -[ 2023-09-21 06:00:03 ] Completed replacing temp checkpoint with checkpoint 19.509 ms, 4.60 s total -[ 2023-09-21 06:00:04 ] Completed train batch 1 to device 1,836.372 ms, 6.44 s total -[ 2023-09-21 06:00:04 ] Completed train batch 1 generator forward 22.679 ms, 6.46 s total -[ 2023-09-21 06:00:05 ] Completed train batch 1 generator loss: 2.349 64.442 ms, 6.53 s total -[ 2023-09-21 06:00:06 ] Completed train batch 1 generator backward 1,083.678 ms, 7.61 s total -[ 2023-09-21 06:00:06 ] Completed train batch 1 discriminator loss 0.156 57.158 ms, 7.67 s total -[ 2023-09-21 06:00:06 ] Completed train batch 1 discriminator backward 75.780 ms, 7.74 s total -Epoch [9] Step [1/2], gen_loss: 0.793, disc_loss: 0.051 -[ 2023-09-21 06:00:06 ] Completed train batch 1 metrics update 1.662 ms, 7.75 s total -Saving checkpoint at epoch 9 train batch 1 -[ 2023-09-21 06:00:06 ] Completed saving temp checkpoint 427.739 ms, 8.17 s total -[ 2023-09-21 06:00:06 ] Completed replacing temp checkpoint with checkpoint 18.060 ms, 8.19 s total -Epoch [9] :: gen_loss: 0.794, disc_loss: 0.054 -[ 2023-09-21 06:00:06 ] Completed training generator for epoch 9 50.404 ms, 8.24 s total -[ 2023-09-21 06:00:06 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 9 from eval step 0 - -[ 2023-09-21 06:00:07 ] Completed eval batch 0 to device 1,182.060 ms, 1.18 s total -[ 2023-09-21 06:00:07 ] Completed eval batch 0 forward 28.041 ms, 1.21 s total -[ 2023-09-21 06:00:07 ] Completed eval batch 0 recons_loss 0.260 ms, 1.21 s total -[ 2023-09-21 06:00:07 ] Completed eval batch 0 metrics update 0.622 ms, 1.21 s total -Saving checkpoint at epoch 9 val batch 0 -[ 2023-09-21 06:00:08 ] Completed saving temp checkpoint 397.001 ms, 1.61 s total -[ 2023-09-21 06:00:08 ] Completed replacing temp checkpoint with checkpoint 34.948 ms, 1.64 s total -[ 2023-09-21 06:00:08 ] Completed eval batch 1 to device 608.771 ms, 2.25 s total -[ 2023-09-21 06:00:09 ] Completed eval batch 1 forward 22.189 ms, 2.27 s total -[ 2023-09-21 06:00:09 ] Completed eval batch 1 recons_loss 0.378 ms, 2.27 s total -[ 2023-09-21 06:00:09 ] Completed eval batch 1 metrics update 10.141 ms, 2.28 s total -Saving checkpoint at epoch 9 val batch 1 -Epoch 9 val loss: 0.2821 -[ 2023-09-21 06:00:09 ] Completed saving temp checkpoint 431.807 ms, 2.72 s total -[ 2023-09-21 06:00:09 ] Completed replacing temp checkpoint with checkpoint 22.822 ms, 2.74 s total -[ 2023-09-21 06:00:09 ] Completed evaluating generator for epoch 9 48.145 ms, 2.79 s total - - -EPOCH :: 10 - - -[ 2023-09-21 06:00:09 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 10 from training step 0 - -[ 2023-09-21 06:00:12 ] Completed train batch 0 to device 3,370.513 ms, 3.37 s total -[ 2023-09-21 06:00:12 ] Completed train batch 0 generator forward 26.396 ms, 3.40 s total -[ 2023-09-21 06:00:12 ] Completed train batch 0 generator loss: 2.241 65.611 ms, 3.46 s total -[ 2023-09-21 06:00:13 ] Completed train batch 0 generator backward 432.115 ms, 3.89 s total -[ 2023-09-21 06:00:13 ] Completed train batch 0 discriminator loss 0.153 57.791 ms, 3.95 s total -[ 2023-09-21 06:00:13 ] Completed train batch 0 discriminator backward 88.302 ms, 4.04 s total -Epoch [10] Step [0/2], gen_loss: 0.747, disc_loss: 0.053 -[ 2023-09-21 06:00:13 ] Completed train batch 0 metrics update 2.264 ms, 4.04 s total -Saving checkpoint at epoch 10 train batch 0 -[ 2023-09-21 06:00:13 ] Completed saving temp checkpoint 407.946 ms, 4.45 s total -[ 2023-09-21 06:00:13 ] Completed replacing temp checkpoint with checkpoint 23.869 ms, 4.47 s total -[ 2023-09-21 06:00:16 ] Completed train batch 1 to device 2,290.220 ms, 6.77 s total -[ 2023-09-21 06:00:16 ] Completed train batch 1 generator forward 41.284 ms, 6.81 s total -[ 2023-09-21 06:00:16 ] Completed train batch 1 generator loss: 2.050 63.915 ms, 6.87 s total -[ 2023-09-21 06:00:16 ] Completed train batch 1 generator backward 311.262 ms, 7.18 s total -[ 2023-09-21 06:00:16 ] Completed train batch 1 discriminator loss 0.152 56.414 ms, 7.24 s total -[ 2023-09-21 06:00:16 ] Completed train batch 1 discriminator backward 94.204 ms, 7.33 s total -Epoch [10] Step [1/2], gen_loss: 0.697, disc_loss: 0.051 -[ 2023-09-21 06:00:16 ] Completed train batch 1 metrics update 0.516 ms, 7.33 s total -Saving checkpoint at epoch 10 train batch 1 -[ 2023-09-21 06:00:17 ] Completed saving temp checkpoint 455.703 ms, 7.79 s total -[ 2023-09-21 06:00:17 ] Completed replacing temp checkpoint with checkpoint 31.564 ms, 7.82 s total -Epoch [10] :: gen_loss: 0.722, disc_loss: 0.052 -[ 2023-09-21 06:00:17 ] Completed training generator for epoch 10 53.745 ms, 7.87 s total -[ 2023-09-21 06:00:17 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 10 from eval step 0 - -[ 2023-09-21 06:00:18 ] Completed eval batch 0 to device 1,054.950 ms, 1.05 s total -[ 2023-09-21 06:00:18 ] Completed eval batch 0 forward 24.740 ms, 1.08 s total -[ 2023-09-21 06:00:18 ] Completed eval batch 0 recons_loss 0.241 ms, 1.08 s total -[ 2023-09-21 06:00:18 ] Completed eval batch 0 metrics update 0.545 ms, 1.08 s total -Saving checkpoint at epoch 10 val batch 0 -[ 2023-09-21 06:00:18 ] Completed saving temp checkpoint 377.685 ms, 1.46 s total -[ 2023-09-21 06:00:18 ] Completed replacing temp checkpoint with checkpoint 29.832 ms, 1.49 s total -[ 2023-09-21 06:00:19 ] Completed eval batch 1 to device 439.378 ms, 1.93 s total -[ 2023-09-21 06:00:19 ] Completed eval batch 1 forward 40.508 ms, 1.97 s total -[ 2023-09-21 06:00:19 ] Completed eval batch 1 recons_loss 0.378 ms, 1.97 s total -[ 2023-09-21 06:00:19 ] Completed eval batch 1 metrics update 293.240 ms, 2.26 s total -Saving checkpoint at epoch 10 val batch 1 -Epoch 10 val loss: 0.2217 -[ 2023-09-21 06:00:20 ] Completed saving temp checkpoint 384.638 ms, 2.65 s total -[ 2023-09-21 06:00:20 ] Completed replacing temp checkpoint with checkpoint 15.443 ms, 2.66 s total -[ 2023-09-21 06:00:20 ] Completed evaluating generator for epoch 10 66.866 ms, 2.73 s total - - -EPOCH :: 11 - - -[ 2023-09-21 06:00:20 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 11 from training step 0 - -[ 2023-09-21 06:00:23 ] Completed train batch 0 to device 3,233.901 ms, 3.23 s total -[ 2023-09-21 06:00:23 ] Completed train batch 0 generator forward 43.434 ms, 3.28 s total -[ 2023-09-21 06:00:23 ] Completed train batch 0 generator loss: 2.221 78.511 ms, 3.36 s total -[ 2023-09-21 06:00:24 ] Completed train batch 0 generator backward 542.500 ms, 3.90 s total -[ 2023-09-21 06:00:24 ] Completed train batch 0 discriminator loss 0.131 62.383 ms, 3.96 s total -[ 2023-09-21 06:00:24 ] Completed train batch 0 discriminator backward 101.983 ms, 4.06 s total -Epoch [11] Step [0/2], gen_loss: 0.747, disc_loss: 0.045 -[ 2023-09-21 06:00:24 ] Completed train batch 0 metrics update 7.883 ms, 4.07 s total -Saving checkpoint at epoch 11 train batch 0 -[ 2023-09-21 06:00:24 ] Completed saving temp checkpoint 457.511 ms, 4.53 s total -[ 2023-09-21 06:00:24 ] Completed replacing temp checkpoint with checkpoint 23.285 ms, 4.55 s total -[ 2023-09-21 06:00:26 ] Completed train batch 1 to device 2,242.198 ms, 6.79 s total -[ 2023-09-21 06:00:26 ] Completed train batch 1 generator forward 27.855 ms, 6.82 s total -[ 2023-09-21 06:00:27 ] Completed train batch 1 generator loss: 2.115 64.395 ms, 6.89 s total -[ 2023-09-21 06:00:27 ] Completed train batch 1 generator backward 565.879 ms, 7.45 s total -[ 2023-09-21 06:00:27 ] Completed train batch 1 discriminator loss 0.110 56.976 ms, 7.51 s total -[ 2023-09-21 06:00:27 ] Completed train batch 1 discriminator backward 73.999 ms, 7.58 s total -Epoch [11] Step [1/2], gen_loss: 0.690, disc_loss: 0.037 -[ 2023-09-21 06:00:27 ] Completed train batch 1 metrics update 3.654 ms, 7.59 s total -Saving checkpoint at epoch 11 train batch 1 -[ 2023-09-21 06:00:28 ] Completed saving temp checkpoint 467.824 ms, 8.05 s total -[ 2023-09-21 06:00:28 ] Completed replacing temp checkpoint with checkpoint 27.784 ms, 8.08 s total -Epoch [11] :: gen_loss: 0.718, disc_loss: 0.041 -[ 2023-09-21 06:00:28 ] Completed training generator for epoch 11 56.057 ms, 8.14 s total -[ 2023-09-21 06:00:28 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 11 from eval step 0 - -[ 2023-09-21 06:00:29 ] Completed eval batch 0 to device 1,188.366 ms, 1.19 s total -[ 2023-09-21 06:00:29 ] Completed eval batch 0 forward 43.755 ms, 1.23 s total -[ 2023-09-21 06:00:29 ] Completed eval batch 0 recons_loss 0.322 ms, 1.23 s total -[ 2023-09-21 06:00:29 ] Completed eval batch 0 metrics update 0.622 ms, 1.23 s total -Saving checkpoint at epoch 11 val batch 0 -[ 2023-09-21 06:00:29 ] Completed saving temp checkpoint 416.438 ms, 1.65 s total -[ 2023-09-21 06:00:29 ] Completed replacing temp checkpoint with checkpoint 31.206 ms, 1.68 s total -[ 2023-09-21 06:00:30 ] Completed eval batch 1 to device 403.993 ms, 2.08 s total -[ 2023-09-21 06:00:30 ] Completed eval batch 1 forward 38.467 ms, 2.12 s total -[ 2023-09-21 06:00:30 ] Completed eval batch 1 recons_loss 0.384 ms, 2.12 s total -[ 2023-09-21 06:00:30 ] Completed eval batch 1 metrics update 128.345 ms, 2.25 s total -Saving checkpoint at epoch 11 val batch 1 -Epoch 11 val loss: 0.2417 -[ 2023-09-21 06:00:30 ] Completed saving temp checkpoint 439.265 ms, 2.69 s total -[ 2023-09-21 06:00:30 ] Completed replacing temp checkpoint with checkpoint 41.616 ms, 2.73 s total -[ 2023-09-21 06:00:31 ] Completed evaluating generator for epoch 11 64.451 ms, 2.80 s total - - -EPOCH :: 12 - - -[ 2023-09-21 06:00:31 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 12 from training step 0 - -[ 2023-09-21 06:00:34 ] Completed train batch 0 to device 3,088.916 ms, 3.09 s total -[ 2023-09-21 06:00:34 ] Completed train batch 0 generator forward 43.973 ms, 3.13 s total -[ 2023-09-21 06:00:34 ] Completed train batch 0 generator loss: 2.049 64.833 ms, 3.20 s total -[ 2023-09-21 06:00:34 ] Completed train batch 0 generator backward 617.891 ms, 3.82 s total -[ 2023-09-21 06:00:34 ] Completed train batch 0 discriminator loss 0.122 57.490 ms, 3.87 s total -[ 2023-09-21 06:00:35 ] Completed train batch 0 discriminator backward 114.083 ms, 3.99 s total -Epoch [12] Step [0/2], gen_loss: 0.696, disc_loss: 0.041 -[ 2023-09-21 06:00:35 ] Completed train batch 0 metrics update 19.901 ms, 4.01 s total -Saving checkpoint at epoch 12 train batch 0 -[ 2023-09-21 06:00:35 ] Completed saving temp checkpoint 413.645 ms, 4.42 s total -[ 2023-09-21 06:00:35 ] Completed replacing temp checkpoint with checkpoint 24.526 ms, 4.45 s total -[ 2023-09-21 06:00:36 ] Completed train batch 1 to device 1,346.627 ms, 5.79 s total -[ 2023-09-21 06:00:36 ] Completed train batch 1 generator forward 41.295 ms, 5.83 s total -[ 2023-09-21 06:00:36 ] Completed train batch 1 generator loss: 2.142 64.071 ms, 5.90 s total -[ 2023-09-21 06:00:38 ] Completed train batch 1 generator backward 1,254.846 ms, 7.15 s total -[ 2023-09-21 06:00:38 ] Completed train batch 1 discriminator loss 0.140 56.630 ms, 7.21 s total -[ 2023-09-21 06:00:38 ] Completed train batch 1 discriminator backward 84.973 ms, 7.29 s total -Epoch [12] Step [1/2], gen_loss: 0.702, disc_loss: 0.045 -[ 2023-09-21 06:00:38 ] Completed train batch 1 metrics update 2.192 ms, 7.30 s total -Saving checkpoint at epoch 12 train batch 1 -[ 2023-09-21 06:00:38 ] Completed saving temp checkpoint 430.741 ms, 7.73 s total -[ 2023-09-21 06:00:38 ] Completed replacing temp checkpoint with checkpoint 24.532 ms, 7.75 s total -Epoch [12] :: gen_loss: 0.699, disc_loss: 0.043 -[ 2023-09-21 06:00:38 ] Completed training generator for epoch 12 61.222 ms, 7.81 s total -[ 2023-09-21 06:00:38 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 12 from eval step 0 - -[ 2023-09-21 06:00:40 ] Completed eval batch 0 to device 1,194.088 ms, 1.19 s total -[ 2023-09-21 06:00:40 ] Completed eval batch 0 forward 30.534 ms, 1.22 s total -[ 2023-09-21 06:00:40 ] Completed eval batch 0 recons_loss 0.290 ms, 1.22 s total -[ 2023-09-21 06:00:40 ] Completed eval batch 0 metrics update 0.693 ms, 1.23 s total -Saving checkpoint at epoch 12 val batch 0 -[ 2023-09-21 06:00:40 ] Completed saving temp checkpoint 521.137 ms, 1.75 s total -[ 2023-09-21 06:00:40 ] Completed replacing temp checkpoint with checkpoint 23.377 ms, 1.77 s total -[ 2023-09-21 06:00:41 ] Completed eval batch 1 to device 468.025 ms, 2.24 s total -[ 2023-09-21 06:00:41 ] Completed eval batch 1 forward 21.786 ms, 2.26 s total -[ 2023-09-21 06:00:41 ] Completed eval batch 1 recons_loss 0.373 ms, 2.26 s total -[ 2023-09-21 06:00:41 ] Completed eval batch 1 metrics update 65.120 ms, 2.33 s total -Saving checkpoint at epoch 12 val batch 1 -Epoch 12 val loss: 0.2570 -[ 2023-09-21 06:00:41 ] Completed saving temp checkpoint 735.990 ms, 3.06 s total -[ 2023-09-21 06:00:41 ] Completed replacing temp checkpoint with checkpoint 31.567 ms, 3.09 s total -[ 2023-09-21 06:00:42 ] Completed evaluating generator for epoch 12 66.252 ms, 3.16 s total - - -EPOCH :: 13 - - -[ 2023-09-21 06:00:42 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 13 from training step 0 - -[ 2023-09-21 06:00:45 ] Completed train batch 0 to device 3,123.151 ms, 3.12 s total -[ 2023-09-21 06:00:45 ] Completed train batch 0 generator forward 30.793 ms, 3.15 s total -[ 2023-09-21 06:00:45 ] Completed train batch 0 generator loss: 2.079 65.580 ms, 3.22 s total -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 06:12:16 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 06:12:16 ] Completed importing Timer 0.023 ms, 0.00 s total -[ 2023-09-21 06:12:21 ] Completed importing everything else 4,545.767 ms, 4.55 s total -| distributed init (rank 1): env:// -| distributed init (rank 3): env:// -| distributed init (rank 4): env:// -| distributed init (rank 0): env:// -| distributed init (rank 5): env:// -| distributed init (rank 2): env:// -[ 2023-09-21 06:12:28 ] Completed preliminaries 7,524.959 ms, 12.07 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 06:12:28 ] Completed build datasets 10.408 ms, 12.08 s total -[ 2023-09-21 06:12:28 ] Completed build samplers 0.042 ms, 12.08 s total -[ 2023-09-21 06:12:28 ] Completed build dataloaders 0.127 ms, 12.08 s total -[ 2023-09-21 06:12:28 ] Completed generator to device 73.793 ms, 12.16 s total -[ 2023-09-21 06:12:28 ] Completed discriminator to device 7.053 ms, 12.16 s total -[ 2023-09-21 06:12:29 ] Completed loss functions 425.567 ms, 12.59 s total -[ 2023-09-21 06:12:29 ] Completed models prepped for distribution 68.759 ms, 12.66 s total -[ 2023-09-21 06:12:29 ] Completed optimizers 0.679 ms, 12.66 s total -[ 2023-09-21 06:12:29 ] Completed grad scalers 0.027 ms, 12.66 s total -[ 2023-09-21 06:12:29 ] Completed checkpoint retrieval 469.977 ms, 13.13 s total - - -EPOCH :: 12 - - -[ 2023-09-21 06:12:29 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 12 from training step 2 - -Epoch [12] :: gen_loss: 0.699, disc_loss: 0.043 -[ 2023-09-21 06:12:29 ] Completed training generator for epoch 12 129.347 ms, 0.13 s total -[ 2023-09-21 06:12:29 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 12 from eval step 2 - -[ 2023-09-21 06:12:29 ] Completed evaluating generator for epoch 12 129.344 ms, 0.13 s total - - -EPOCH :: 13 - - -[ 2023-09-21 06:12:29 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 13 from training step 0 - -[ 2023-09-21 06:12:33 ] Completed train batch 0 to device 3,636.977 ms, 3.64 s total -[ 2023-09-21 06:12:34 ] Completed train batch 0 generator forward 1,311.772 ms, 4.95 s total -[ 2023-09-21 06:12:34 ] Completed train batch 0 generator loss: 2.058 75.786 ms, 5.02 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 06:12:36 ] Completed train batch 0 generator backward 1,183.120 ms, 6.21 s total -[ 2023-09-21 06:12:36 ] Completed train batch 0 discriminator loss 0.143 55.956 ms, 6.26 s total -[ 2023-09-21 06:12:36 ] Completed train batch 0 discriminator backward 98.642 ms, 6.36 s total -Epoch [13] Step [0/2], gen_loss: 0.685, disc_loss: 0.049 -[ 2023-09-21 06:12:36 ] Completed train batch 0 metrics update 7.430 ms, 6.37 s total -Saving checkpoint at epoch 13 train batch 0 -[ 2023-09-21 06:12:36 ] Completed saving temp checkpoint 415.744 ms, 6.79 s total -[ 2023-09-21 06:12:36 ] Completed replacing temp checkpoint with checkpoint 119.098 ms, 6.90 s total -[ 2023-09-21 06:12:37 ] Completed train batch 1 to device 344.769 ms, 7.25 s total -[ 2023-09-21 06:12:37 ] Completed train batch 1 generator forward 22.179 ms, 7.27 s total -[ 2023-09-21 06:12:37 ] Completed train batch 1 generator loss: 1.970 62.843 ms, 7.33 s total -[ 2023-09-21 06:12:38 ] Completed train batch 1 generator backward 1,143.028 ms, 8.48 s total -[ 2023-09-21 06:12:38 ] Completed train batch 1 discriminator loss 0.151 56.105 ms, 8.53 s total -[ 2023-09-21 06:12:38 ] Completed train batch 1 discriminator backward 71.245 ms, 8.60 s total -Epoch [13] Step [1/2], gen_loss: 0.665, disc_loss: 0.050 -[ 2023-09-21 06:12:38 ] Completed train batch 1 metrics update 3.547 ms, 8.61 s total -Saving checkpoint at epoch 13 train batch 1 -[ 2023-09-21 06:12:38 ] Completed saving temp checkpoint 438.367 ms, 9.05 s total -[ 2023-09-21 06:12:38 ] Completed replacing temp checkpoint with checkpoint 24.197 ms, 9.07 s total -Epoch [13] :: gen_loss: 0.675, disc_loss: 0.050 -[ 2023-09-21 06:12:38 ] Completed training generator for epoch 13 33.344 ms, 9.10 s total -[ 2023-09-21 06:12:38 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 13 from eval step 0 - -[ 2023-09-21 06:12:40 ] Completed eval batch 0 to device 1,076.729 ms, 1.08 s total -[ 2023-09-21 06:12:40 ] Completed eval batch 0 forward 25.197 ms, 1.10 s total -[ 2023-09-21 06:12:40 ] Completed eval batch 0 recons_loss 0.303 ms, 1.10 s total -[ 2023-09-21 06:12:40 ] Completed eval batch 0 metrics update 0.586 ms, 1.10 s total -Saving checkpoint at epoch 13 val batch 0 -[ 2023-09-21 06:12:40 ] Completed saving temp checkpoint 481.183 ms, 1.58 s total -[ 2023-09-21 06:12:40 ] Completed replacing temp checkpoint with checkpoint 25.016 ms, 1.61 s total -[ 2023-09-21 06:12:40 ] Completed eval batch 1 to device 329.505 ms, 1.94 s total -[ 2023-09-21 06:12:40 ] Completed eval batch 1 forward 40.561 ms, 1.98 s total -[ 2023-09-21 06:12:40 ] Completed eval batch 1 recons_loss 0.337 ms, 1.98 s total -[ 2023-09-21 06:12:41 ] Completed eval batch 1 metrics update 259.834 ms, 2.24 s total -Saving checkpoint at epoch 13 val batch 1 -Epoch 13 val loss: 0.1831 -[ 2023-09-21 06:12:41 ] Completed saving temp checkpoint 551.265 ms, 2.79 s total -[ 2023-09-21 06:12:41 ] Completed replacing temp checkpoint with checkpoint 30.475 ms, 2.82 s total -[ 2023-09-21 06:12:41 ] Completed evaluating generator for epoch 13 63.648 ms, 2.88 s total - - -EPOCH :: 14 - - -[ 2023-09-21 06:12:41 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 14 from training step 0 - -[ 2023-09-21 06:12:44 ] Completed train batch 0 to device 2,881.983 ms, 2.88 s total -[ 2023-09-21 06:12:44 ] Completed train batch 0 generator forward 26.457 ms, 2.91 s total -[ 2023-09-21 06:12:44 ] Completed train batch 0 generator loss: 1.837 66.258 ms, 2.97 s total -[ 2023-09-21 06:12:45 ] Completed train batch 0 generator backward 774.953 ms, 3.75 s total -[ 2023-09-21 06:12:45 ] Completed train batch 0 discriminator loss 0.158 59.696 ms, 3.81 s total -[ 2023-09-21 06:12:45 ] Completed train batch 0 discriminator backward 108.043 ms, 3.92 s total -Epoch [14] Step [0/2], gen_loss: 0.607, disc_loss: 0.054 -[ 2023-09-21 06:12:45 ] Completed train batch 0 metrics update 19.270 ms, 3.94 s total -Saving checkpoint at epoch 14 train batch 0 -[ 2023-09-21 06:12:46 ] Completed saving temp checkpoint 414.018 ms, 4.35 s total -[ 2023-09-21 06:12:46 ] Completed replacing temp checkpoint with checkpoint 26.998 ms, 4.38 s total -[ 2023-09-21 06:12:47 ] Completed train batch 1 to device 1,238.225 ms, 5.62 s total -[ 2023-09-21 06:12:47 ] Completed train batch 1 generator forward 40.425 ms, 5.66 s total -[ 2023-09-21 06:12:47 ] Completed train batch 1 generator loss: 1.742 63.544 ms, 5.72 s total -[ 2023-09-21 06:12:49 ] Completed train batch 1 generator backward 1,616.803 ms, 7.34 s total -[ 2023-09-21 06:12:49 ] Completed train batch 1 discriminator loss 0.163 55.897 ms, 7.39 s total -[ 2023-09-21 06:12:49 ] Completed train batch 1 discriminator backward 75.497 ms, 7.47 s total -Epoch [14] Step [1/2], gen_loss: 0.598, disc_loss: 0.054 -[ 2023-09-21 06:12:49 ] Completed train batch 1 metrics update 2.217 ms, 7.47 s total -Saving checkpoint at epoch 14 train batch 1 -[ 2023-09-21 06:12:49 ] Completed saving temp checkpoint 503.225 ms, 7.97 s total -[ 2023-09-21 06:12:49 ] Completed replacing temp checkpoint with checkpoint 24.537 ms, 8.00 s total -Epoch [14] :: gen_loss: 0.602, disc_loss: 0.054 -[ 2023-09-21 06:12:49 ] Completed training generator for epoch 14 49.569 ms, 8.05 s total -[ 2023-09-21 06:12:49 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 14 from eval step 0 - -[ 2023-09-21 06:12:51 ] Completed eval batch 0 to device 1,105.344 ms, 1.11 s total -[ 2023-09-21 06:12:51 ] Completed eval batch 0 forward 41.653 ms, 1.15 s total -[ 2023-09-21 06:12:51 ] Completed eval batch 0 recons_loss 0.293 ms, 1.15 s total -[ 2023-09-21 06:12:51 ] Completed eval batch 0 metrics update 0.564 ms, 1.15 s total -Saving checkpoint at epoch 14 val batch 0 -[ 2023-09-21 06:12:51 ] Completed saving temp checkpoint 425.916 ms, 1.57 s total -[ 2023-09-21 06:12:51 ] Completed replacing temp checkpoint with checkpoint 28.912 ms, 1.60 s total -[ 2023-09-21 06:12:51 ] Completed eval batch 1 to device 383.356 ms, 1.99 s total -[ 2023-09-21 06:12:51 ] Completed eval batch 1 forward 38.152 ms, 2.02 s total -[ 2023-09-21 06:12:51 ] Completed eval batch 1 recons_loss 0.332 ms, 2.02 s total -[ 2023-09-21 06:12:52 ] Completed eval batch 1 metrics update 163.543 ms, 2.19 s total -Saving checkpoint at epoch 14 val batch 1 -Epoch 14 val loss: 0.2282 -[ 2023-09-21 06:12:52 ] Completed saving temp checkpoint 433.918 ms, 2.62 s total -[ 2023-09-21 06:12:52 ] Completed replacing temp checkpoint with checkpoint 20.315 ms, 2.64 s total -[ 2023-09-21 06:12:52 ] Completed evaluating generator for epoch 14 55.441 ms, 2.70 s total - - -EPOCH :: 15 - - -[ 2023-09-21 06:12:52 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 15 from training step 0 - -[ 2023-09-21 06:12:55 ] Completed train batch 0 to device 2,934.298 ms, 2.93 s total -[ 2023-09-21 06:12:55 ] Completed train batch 0 generator forward 29.177 ms, 2.96 s total -[ 2023-09-21 06:12:55 ] Completed train batch 0 generator loss: 2.037 66.560 ms, 3.03 s total -[ 2023-09-21 06:12:56 ] Completed train batch 0 generator backward 798.036 ms, 3.83 s total -[ 2023-09-21 06:12:56 ] Completed train batch 0 discriminator loss 0.150 57.090 ms, 3.89 s total -[ 2023-09-21 06:12:56 ] Completed train batch 0 discriminator backward 98.159 ms, 3.98 s total -Epoch [15] Step [0/2], gen_loss: 0.676, disc_loss: 0.051 -[ 2023-09-21 06:12:56 ] Completed train batch 0 metrics update 3.074 ms, 3.99 s total -Saving checkpoint at epoch 15 train batch 0 -[ 2023-09-21 06:12:56 ] Completed saving temp checkpoint 395.024 ms, 4.38 s total -[ 2023-09-21 06:12:57 ] Completed replacing temp checkpoint with checkpoint 24.384 ms, 4.41 s total -[ 2023-09-21 06:12:58 ] Completed train batch 1 to device 1,150.190 ms, 5.56 s total -[ 2023-09-21 06:12:58 ] Completed train batch 1 generator forward 40.896 ms, 5.60 s total -[ 2023-09-21 06:12:58 ] Completed train batch 1 generator loss: 1.981 62.045 ms, 5.66 s total -[ 2023-09-21 06:13:00 ] Completed train batch 1 generator backward 1,869.957 ms, 7.53 s total -[ 2023-09-21 06:13:00 ] Completed train batch 1 discriminator loss 0.134 56.303 ms, 7.59 s total -[ 2023-09-21 06:13:00 ] Completed train batch 1 discriminator backward 71.053 ms, 7.66 s total -Epoch [15] Step [1/2], gen_loss: 0.676, disc_loss: 0.045 -[ 2023-09-21 06:13:00 ] Completed train batch 1 metrics update 5.735 ms, 7.66 s total -Saving checkpoint at epoch 15 train batch 1 -[ 2023-09-21 06:13:00 ] Completed saving temp checkpoint 455.769 ms, 8.12 s total -[ 2023-09-21 06:13:00 ] Completed replacing temp checkpoint with checkpoint 23.246 ms, 8.14 s total -Epoch [15] :: gen_loss: 0.676, disc_loss: 0.048 -[ 2023-09-21 06:13:00 ] Completed training generator for epoch 15 57.371 ms, 8.20 s total -[ 2023-09-21 06:13:00 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 15 from eval step 0 - -[ 2023-09-21 06:13:01 ] Completed eval batch 0 to device 1,133.617 ms, 1.13 s total -[ 2023-09-21 06:13:01 ] Completed eval batch 0 forward 43.453 ms, 1.18 s total -[ 2023-09-21 06:13:01 ] Completed eval batch 0 recons_loss 0.327 ms, 1.18 s total -[ 2023-09-21 06:13:01 ] Completed eval batch 0 metrics update 0.612 ms, 1.18 s total -Saving checkpoint at epoch 15 val batch 0 -[ 2023-09-21 06:13:02 ] Completed saving temp checkpoint 432.473 ms, 1.61 s total -[ 2023-09-21 06:13:02 ] Completed replacing temp checkpoint with checkpoint 35.602 ms, 1.65 s total -[ 2023-09-21 06:13:02 ] Completed eval batch 1 to device 397.637 ms, 2.04 s total -[ 2023-09-21 06:13:02 ] Completed eval batch 1 forward 39.920 ms, 2.08 s total -[ 2023-09-21 06:13:02 ] Completed eval batch 1 recons_loss 0.565 ms, 2.08 s total -[ 2023-09-21 06:13:03 ] Completed eval batch 1 metrics update 152.794 ms, 2.24 s total -Saving checkpoint at epoch 15 val batch 1 -Epoch 15 val loss: 0.2527 -[ 2023-09-21 06:13:03 ] Completed saving temp checkpoint 629.026 ms, 2.87 s total -[ 2023-09-21 06:13:03 ] Completed replacing temp checkpoint with checkpoint 39.706 ms, 2.91 s total -[ 2023-09-21 06:13:03 ] Completed evaluating generator for epoch 15 57.582 ms, 2.96 s total - - -EPOCH :: 16 - - -[ 2023-09-21 06:13:03 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 16 from training step 0 - -[ 2023-09-21 06:13:06 ] Completed train batch 0 to device 3,045.157 ms, 3.05 s total -[ 2023-09-21 06:13:06 ] Completed train batch 0 generator forward 25.900 ms, 3.07 s total -[ 2023-09-21 06:13:06 ] Completed train batch 0 generator loss: 2.156 66.812 ms, 3.14 s total -[ 2023-09-21 06:13:07 ] Completed train batch 0 generator backward 507.921 ms, 3.65 s total -[ 2023-09-21 06:13:07 ] Completed train batch 0 discriminator loss 0.123 57.292 ms, 3.70 s total -[ 2023-09-21 06:13:07 ] Completed train batch 0 discriminator backward 104.515 ms, 3.81 s total -Epoch [16] Step [0/2], gen_loss: 0.729, disc_loss: 0.043 -[ 2023-09-21 06:13:07 ] Completed train batch 0 metrics update 5.730 ms, 3.81 s total -Saving checkpoint at epoch 16 train batch 0 -[ 2023-09-21 06:13:08 ] Completed saving temp checkpoint 414.706 ms, 4.23 s total -[ 2023-09-21 06:13:08 ] Completed replacing temp checkpoint with checkpoint 22.189 ms, 4.25 s total -[ 2023-09-21 06:13:09 ] Completed train batch 1 to device 1,712.595 ms, 5.96 s total -[ 2023-09-21 06:13:09 ] Completed train batch 1 generator forward 22.413 ms, 5.99 s total -[ 2023-09-21 06:13:09 ] Completed train batch 1 generator loss: 2.354 63.482 ms, 6.05 s total -[ 2023-09-21 06:13:11 ] Completed train batch 1 generator backward 1,195.438 ms, 7.24 s total -[ 2023-09-21 06:13:11 ] Completed train batch 1 discriminator loss 0.112 57.134 ms, 7.30 s total -[ 2023-09-21 06:13:11 ] Completed train batch 1 discriminator backward 73.378 ms, 7.37 s total -Epoch [16] Step [1/2], gen_loss: 0.765, disc_loss: 0.038 -[ 2023-09-21 06:13:11 ] Completed train batch 1 metrics update 1.040 ms, 7.38 s total -Saving checkpoint at epoch 16 train batch 1 -[ 2023-09-21 06:13:11 ] Completed saving temp checkpoint 436.627 ms, 7.81 s total -[ 2023-09-21 06:13:11 ] Completed replacing temp checkpoint with checkpoint 18.214 ms, 7.83 s total -Epoch [16] :: gen_loss: 0.747, disc_loss: 0.040 -[ 2023-09-21 06:13:11 ] Completed training generator for epoch 16 54.639 ms, 7.89 s total -[ 2023-09-21 06:13:11 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 16 from eval step 0 - -[ 2023-09-21 06:13:12 ] Completed eval batch 0 to device 1,164.517 ms, 1.16 s total -[ 2023-09-21 06:13:12 ] Completed eval batch 0 forward 26.908 ms, 1.19 s total -[ 2023-09-21 06:13:12 ] Completed eval batch 0 recons_loss 0.378 ms, 1.19 s total -[ 2023-09-21 06:13:12 ] Completed eval batch 0 metrics update 0.625 ms, 1.19 s total -Saving checkpoint at epoch 16 val batch 0 -[ 2023-09-21 06:13:13 ] Completed saving temp checkpoint 391.904 ms, 1.58 s total -[ 2023-09-21 06:13:13 ] Completed replacing temp checkpoint with checkpoint 26.002 ms, 1.61 s total -[ 2023-09-21 06:13:13 ] Completed eval batch 1 to device 408.181 ms, 2.02 s total -[ 2023-09-21 06:13:13 ] Completed eval batch 1 forward 21.611 ms, 2.04 s total -[ 2023-09-21 06:13:13 ] Completed eval batch 1 recons_loss 0.369 ms, 2.04 s total -[ 2023-09-21 06:13:13 ] Completed eval batch 1 metrics update 171.785 ms, 2.21 s total -Saving checkpoint at epoch 16 val batch 1 -Epoch 16 val loss: 0.3175 -[ 2023-09-21 06:13:14 ] Completed saving temp checkpoint 446.802 ms, 2.66 s total -[ 2023-09-21 06:13:14 ] Completed replacing temp checkpoint with checkpoint 26.578 ms, 2.69 s total -[ 2023-09-21 06:13:14 ] Completed evaluating generator for epoch 16 55.591 ms, 2.74 s total - - -EPOCH :: 17 - - -[ 2023-09-21 06:13:14 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 17 from training step 0 - -[ 2023-09-21 06:13:17 ] Completed train batch 0 to device 3,089.605 ms, 3.09 s total -[ 2023-09-21 06:13:17 ] Completed train batch 0 generator forward 31.533 ms, 3.12 s total -[ 2023-09-21 06:13:17 ] Completed train batch 0 generator loss: 2.296 66.255 ms, 3.19 s total -[ 2023-09-21 06:13:18 ] Completed train batch 0 generator backward 627.823 ms, 3.82 s total -[ 2023-09-21 06:13:18 ] Completed train batch 0 discriminator loss 0.108 56.206 ms, 3.87 s total -[ 2023-09-21 06:13:18 ] Completed train batch 0 discriminator backward 115.541 ms, 3.99 s total -Epoch [17] Step [0/2], gen_loss: 0.776, disc_loss: 0.036 -[ 2023-09-21 06:13:18 ] Completed train batch 0 metrics update 6.672 ms, 3.99 s total -Saving checkpoint at epoch 17 train batch 0 -[ 2023-09-21 06:13:18 ] Completed saving temp checkpoint 394.360 ms, 4.39 s total -[ 2023-09-21 06:13:18 ] Completed replacing temp checkpoint with checkpoint 31.702 ms, 4.42 s total -[ 2023-09-21 06:13:20 ] Completed train batch 1 to device 1,497.831 ms, 5.92 s total -[ 2023-09-21 06:13:20 ] Completed train batch 1 generator forward 22.256 ms, 5.94 s total -[ 2023-09-21 06:13:20 ] Completed train batch 1 generator loss: 2.291 63.235 ms, 6.00 s total -[ 2023-09-21 06:13:21 ] Completed train batch 1 generator backward 1,321.630 ms, 7.32 s total -[ 2023-09-21 06:13:21 ] Completed train batch 1 discriminator loss 0.102 56.259 ms, 7.38 s total -[ 2023-09-21 06:13:21 ] Completed train batch 1 discriminator backward 74.014 ms, 7.45 s total -Epoch [17] Step [1/2], gen_loss: 0.768, disc_loss: 0.035 -[ 2023-09-21 06:13:21 ] Completed train batch 1 metrics update 5.670 ms, 7.46 s total -Saving checkpoint at epoch 17 train batch 1 -[ 2023-09-21 06:13:22 ] Completed saving temp checkpoint 464.422 ms, 7.93 s total -[ 2023-09-21 06:13:22 ] Completed replacing temp checkpoint with checkpoint 22.264 ms, 7.95 s total -Epoch [17] :: gen_loss: 0.772, disc_loss: 0.035 -[ 2023-09-21 06:13:22 ] Completed training generator for epoch 17 52.477 ms, 8.00 s total -[ 2023-09-21 06:13:22 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 17 from eval step 0 - -[ 2023-09-21 06:13:23 ] Completed eval batch 0 to device 1,060.799 ms, 1.06 s total -[ 2023-09-21 06:13:23 ] Completed eval batch 0 forward 25.276 ms, 1.09 s total -[ 2023-09-21 06:13:23 ] Completed eval batch 0 recons_loss 0.319 ms, 1.09 s total -[ 2023-09-21 06:13:23 ] Completed eval batch 0 metrics update 0.548 ms, 1.09 s total -Saving checkpoint at epoch 17 val batch 0 -[ 2023-09-21 06:13:23 ] Completed saving temp checkpoint 367.484 ms, 1.45 s total -[ 2023-09-21 06:13:23 ] Completed replacing temp checkpoint with checkpoint 29.368 ms, 1.48 s total -[ 2023-09-21 06:13:24 ] Completed eval batch 1 to device 439.374 ms, 1.92 s total -[ 2023-09-21 06:13:24 ] Completed eval batch 1 forward 38.815 ms, 1.96 s total -[ 2023-09-21 06:13:24 ] Completed eval batch 1 recons_loss 0.527 ms, 1.96 s total -[ 2023-09-21 06:13:24 ] Completed eval batch 1 metrics update 274.837 ms, 2.24 s total -Saving checkpoint at epoch 17 val batch 1 -Epoch 17 val loss: 0.2978 -[ 2023-09-21 06:13:25 ] Completed saving temp checkpoint 489.065 ms, 2.73 s total -[ 2023-09-21 06:13:25 ] Completed replacing temp checkpoint with checkpoint 27.289 ms, 2.75 s total -[ 2023-09-21 06:13:25 ] Completed evaluating generator for epoch 17 53.463 ms, 2.81 s total - - -EPOCH :: 18 - - -[ 2023-09-21 06:13:25 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 18 from training step 0 - -[ 2023-09-21 06:13:28 ] Completed train batch 0 to device 3,044.956 ms, 3.04 s total -[ 2023-09-21 06:13:28 ] Completed train batch 0 generator forward 30.691 ms, 3.08 s total -[ 2023-09-21 06:13:28 ] Completed train batch 0 generator loss: 2.224 66.440 ms, 3.14 s total -[ 2023-09-21 06:13:28 ] Completed train batch 0 generator backward 606.919 ms, 3.75 s total -[ 2023-09-21 06:13:29 ] Completed train batch 0 discriminator loss 0.112 57.587 ms, 3.81 s total -[ 2023-09-21 06:13:29 ] Completed train batch 0 discriminator backward 107.442 ms, 3.91 s total -Epoch [18] Step [0/2], gen_loss: 0.738, disc_loss: 0.038 -[ 2023-09-21 06:13:29 ] Completed train batch 0 metrics update 4.891 ms, 3.92 s total -Saving checkpoint at epoch 18 train batch 0 -[ 2023-09-21 06:13:29 ] Completed saving temp checkpoint 417.542 ms, 4.34 s total -[ 2023-09-21 06:13:29 ] Completed replacing temp checkpoint with checkpoint 23.134 ms, 4.36 s total -[ 2023-09-21 06:13:31 ] Completed train batch 1 to device 1,530.888 ms, 5.89 s total -[ 2023-09-21 06:13:31 ] Completed train batch 1 generator forward 42.228 ms, 5.93 s total -[ 2023-09-21 06:13:31 ] Completed train batch 1 generator loss: 2.190 63.661 ms, 6.00 s total -[ 2023-09-21 06:13:32 ] Completed train batch 1 generator backward 1,037.285 ms, 7.03 s total -[ 2023-09-21 06:13:32 ] Completed train batch 1 discriminator loss 0.131 61.955 ms, 7.10 s total -[ 2023-09-21 06:13:32 ] Completed train batch 1 discriminator backward 57.661 ms, 7.15 s total -Epoch [18] Step [1/2], gen_loss: 0.721, disc_loss: 0.042 -[ 2023-09-21 06:13:32 ] Completed train batch 1 metrics update 5.421 ms, 7.16 s total -Saving checkpoint at epoch 18 train batch 1 -[ 2023-09-21 06:13:32 ] Completed saving temp checkpoint 456.546 ms, 7.62 s total -[ 2023-09-21 06:13:32 ] Completed replacing temp checkpoint with checkpoint 23.140 ms, 7.64 s total -Epoch [18] :: gen_loss: 0.729, disc_loss: 0.040 -[ 2023-09-21 06:13:32 ] Completed training generator for epoch 18 63.905 ms, 7.70 s total -[ 2023-09-21 06:13:32 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 18 from eval step 0 - -[ 2023-09-21 06:13:33 ] Completed eval batch 0 to device 1,078.770 ms, 1.08 s total -[ 2023-09-21 06:13:34 ] Completed eval batch 0 forward 24.651 ms, 1.10 s total -[ 2023-09-21 06:13:34 ] Completed eval batch 0 recons_loss 0.311 ms, 1.10 s total -[ 2023-09-21 06:13:34 ] Completed eval batch 0 metrics update 0.547 ms, 1.10 s total -Saving checkpoint at epoch 18 val batch 0 -[ 2023-09-21 06:13:34 ] Completed saving temp checkpoint 421.878 ms, 1.53 s total -[ 2023-09-21 06:13:34 ] Completed replacing temp checkpoint with checkpoint 27.372 ms, 1.55 s total -[ 2023-09-21 06:13:34 ] Completed eval batch 1 to device 364.133 ms, 1.92 s total -[ 2023-09-21 06:13:34 ] Completed eval batch 1 forward 39.126 ms, 1.96 s total -[ 2023-09-21 06:13:34 ] Completed eval batch 1 recons_loss 0.329 ms, 1.96 s total -[ 2023-09-21 06:13:35 ] Completed eval batch 1 metrics update 263.351 ms, 2.22 s total -Saving checkpoint at epoch 18 val batch 1 -Epoch 18 val loss: 0.2379 -[ 2023-09-21 06:13:35 ] Completed saving temp checkpoint 420.844 ms, 2.64 s total -[ 2023-09-21 06:13:35 ] Completed replacing temp checkpoint with checkpoint 22.810 ms, 2.66 s total -[ 2023-09-21 06:13:35 ] Completed evaluating generator for epoch 18 54.664 ms, 2.72 s total - - -EPOCH :: 19 - - -[ 2023-09-21 06:13:35 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 19 from training step 0 - -[ 2023-09-21 06:13:38 ] Completed train batch 0 to device 3,042.992 ms, 3.04 s total -[ 2023-09-21 06:13:38 ] Completed train batch 0 generator forward 33.362 ms, 3.08 s total -[ 2023-09-21 06:13:38 ] Completed train batch 0 generator loss: 2.083 76.342 ms, 3.15 s total -[ 2023-09-21 06:13:39 ] Completed train batch 0 generator backward 636.184 ms, 3.79 s total -[ 2023-09-21 06:13:39 ] Completed train batch 0 discriminator loss 0.151 57.366 ms, 3.85 s total -[ 2023-09-21 06:13:39 ] Completed train batch 0 discriminator backward 118.933 ms, 3.97 s total -Epoch [19] Step [0/2], gen_loss: 0.696, disc_loss: 0.050 -[ 2023-09-21 06:13:39 ] Completed train batch 0 metrics update 5.173 ms, 3.97 s total -Saving checkpoint at epoch 19 train batch 0 -[ 2023-09-21 06:13:40 ] Completed saving temp checkpoint 405.401 ms, 4.38 s total -[ 2023-09-21 06:13:40 ] Completed replacing temp checkpoint with checkpoint 22.087 ms, 4.40 s total -[ 2023-09-21 06:13:41 ] Completed train batch 1 to device 1,430.871 ms, 5.83 s total -[ 2023-09-21 06:13:41 ] Completed train batch 1 generator forward 40.708 ms, 5.87 s total -[ 2023-09-21 06:13:41 ] Completed train batch 1 generator loss: 1.992 64.195 ms, 5.93 s total -[ 2023-09-21 06:13:42 ] Completed train batch 1 generator backward 1,245.921 ms, 7.18 s total -[ 2023-09-21 06:13:42 ] Completed train batch 1 discriminator loss 0.153 56.157 ms, 7.24 s total -[ 2023-09-21 06:13:42 ] Completed train batch 1 discriminator backward 75.378 ms, 7.31 s total -Epoch [19] Step [1/2], gen_loss: 0.666, disc_loss: 0.052 -[ 2023-09-21 06:13:42 ] Completed train batch 1 metrics update 0.859 ms, 7.31 s total -Saving checkpoint at epoch 19 train batch 1 -[ 2023-09-21 06:13:43 ] Completed saving temp checkpoint 409.781 ms, 7.72 s total -[ 2023-09-21 06:13:43 ] Completed replacing temp checkpoint with checkpoint 26.296 ms, 7.75 s total -Epoch [19] :: gen_loss: 0.681, disc_loss: 0.051 -[ 2023-09-21 06:13:43 ] Completed training generator for epoch 19 53.171 ms, 7.80 s total -[ 2023-09-21 06:13:43 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 19 from eval step 0 - -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 06:26:21 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 06:26:21 ] Completed importing Timer 0.024 ms, 0.00 s total -[ 2023-09-21 06:26:26 ] Completed importing everything else 4,907.589 ms, 4.91 s total -| distributed init (rank 5): env:// -| distributed init (rank 3): env:// -| distributed init (rank 4): env:// -| distributed init (rank 1): env:// -| distributed init (rank 0): env:// -| distributed init (rank 2): env:// -[ 2023-09-21 06:26:34 ] Completed preliminaries 7,723.475 ms, 12.63 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 06:26:34 ] Completed build datasets 10.425 ms, 12.64 s total -[ 2023-09-21 06:26:34 ] Completed build samplers 0.052 ms, 12.64 s total -[ 2023-09-21 06:26:34 ] Completed build dataloaders 0.139 ms, 12.64 s total -[ 2023-09-21 06:26:34 ] Completed generator to device 76.185 ms, 12.72 s total -[ 2023-09-21 06:26:34 ] Completed discriminator to device 7.759 ms, 12.73 s total -[ 2023-09-21 06:26:34 ] Completed loss functions 455.803 ms, 13.18 s total -[ 2023-09-21 06:26:34 ] Completed models prepped for distribution 83.918 ms, 13.27 s total -[ 2023-09-21 06:26:34 ] Completed optimizers 0.748 ms, 13.27 s total -[ 2023-09-21 06:26:34 ] Completed grad scalers 0.035 ms, 13.27 s total -[ 2023-09-21 06:26:35 ] Completed checkpoint retrieval 528.489 ms, 13.79 s total - - -EPOCH :: 19 - - -[ 2023-09-21 06:26:35 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 19 from training step 2 - -Epoch [19] :: gen_loss: 0.681, disc_loss: 0.051 -[ 2023-09-21 06:26:35 ] Completed training generator for epoch 19 138.221 ms, 0.14 s total -[ 2023-09-21 06:26:35 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 19 from eval step 0 - -[ 2023-09-21 06:26:37 ] Completed eval batch 0 to device 1,645.299 ms, 1.65 s total -[ 2023-09-21 06:26:39 ] Completed eval batch 0 forward 1,734.666 ms, 3.38 s total -[ 2023-09-21 06:26:39 ] Completed eval batch 0 recons_loss 0.349 ms, 3.38 s total -[ 2023-09-21 06:26:39 ] Completed eval batch 0 metrics update 305.799 ms, 3.69 s total -Saving checkpoint at epoch 19 val batch 0 -[ 2023-09-21 06:26:39 ] Completed saving temp checkpoint 421.883 ms, 4.11 s total -[ 2023-09-21 06:26:39 ] Completed replacing temp checkpoint with checkpoint 124.821 ms, 4.23 s total -[ 2023-09-21 06:26:39 ] Completed eval batch 1 to device 11.775 ms, 4.24 s total -[ 2023-09-21 06:26:39 ] Completed eval batch 1 forward 21.206 ms, 4.27 s total -[ 2023-09-21 06:26:39 ] Completed eval batch 1 recons_loss 0.342 ms, 4.27 s total -[ 2023-09-21 06:26:39 ] Completed eval batch 1 metrics update 0.307 ms, 4.27 s total -Saving checkpoint at epoch 19 val batch 1 -Epoch 19 val loss: 0.1705 -[ 2023-09-21 06:26:40 ] Completed saving temp checkpoint 433.925 ms, 4.70 s total -[ 2023-09-21 06:26:40 ] Completed replacing temp checkpoint with checkpoint 18.297 ms, 4.72 s total -[ 2023-09-21 06:26:40 ] Completed evaluating generator for epoch 19 40.113 ms, 4.76 s total - - -EPOCH :: 20 - - -[ 2023-09-21 06:26:40 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 20 from training step 0 - -[ 2023-09-21 06:26:44 ] Completed train batch 0 to device 3,606.015 ms, 3.61 s total -[ 2023-09-21 06:26:44 ] Completed train batch 0 generator forward 48.470 ms, 3.65 s total -[ 2023-09-21 06:26:44 ] Completed train batch 0 generator loss: 1.809 86.878 ms, 3.74 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 06:26:44 ] Completed train batch 0 generator backward 333.596 ms, 4.07 s total -[ 2023-09-21 06:26:44 ] Completed train batch 0 discriminator loss 0.165 57.774 ms, 4.13 s total -[ 2023-09-21 06:26:44 ] Completed train batch 0 discriminator backward 109.061 ms, 4.24 s total -Epoch [20] Step [0/2], gen_loss: 0.611, disc_loss: 0.055 -[ 2023-09-21 06:26:44 ] Completed train batch 0 metrics update 4.774 ms, 4.25 s total -Saving checkpoint at epoch 20 train batch 0 -[ 2023-09-21 06:26:45 ] Completed saving temp checkpoint 421.583 ms, 4.67 s total -[ 2023-09-21 06:26:45 ] Completed replacing temp checkpoint with checkpoint 17.248 ms, 4.69 s total -[ 2023-09-21 06:26:47 ] Completed train batch 1 to device 2,276.392 ms, 6.96 s total -[ 2023-09-21 06:26:47 ] Completed train batch 1 generator forward 22.244 ms, 6.98 s total -[ 2023-09-21 06:26:47 ] Completed train batch 1 generator loss: 1.595 63.413 ms, 7.05 s total -[ 2023-09-21 06:26:47 ] Completed train batch 1 generator backward 554.514 ms, 7.60 s total -[ 2023-09-21 06:26:48 ] Completed train batch 1 discriminator loss 0.166 56.843 ms, 7.66 s total -[ 2023-09-21 06:26:48 ] Completed train batch 1 discriminator backward 70.543 ms, 7.73 s total -Epoch [20] Step [1/2], gen_loss: 0.548, disc_loss: 0.055 -[ 2023-09-21 06:26:48 ] Completed train batch 1 metrics update 2.029 ms, 7.73 s total -Saving checkpoint at epoch 20 train batch 1 -[ 2023-09-21 06:26:48 ] Completed saving temp checkpoint 471.295 ms, 8.20 s total -[ 2023-09-21 06:26:48 ] Completed replacing temp checkpoint with checkpoint 21.383 ms, 8.22 s total -Epoch [20] :: gen_loss: 0.579, disc_loss: 0.055 -[ 2023-09-21 06:26:48 ] Completed training generator for epoch 20 48.211 ms, 8.27 s total -[ 2023-09-21 06:26:48 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 20 from eval step 0 - -[ 2023-09-21 06:26:49 ] Completed eval batch 0 to device 1,173.850 ms, 1.17 s total -[ 2023-09-21 06:26:49 ] Completed eval batch 0 forward 31.760 ms, 1.21 s total -[ 2023-09-21 06:26:49 ] Completed eval batch 0 recons_loss 0.385 ms, 1.21 s total -[ 2023-09-21 06:26:49 ] Completed eval batch 0 metrics update 0.681 ms, 1.21 s total -Saving checkpoint at epoch 20 val batch 0 -[ 2023-09-21 06:26:50 ] Completed saving temp checkpoint 395.185 ms, 1.60 s total -[ 2023-09-21 06:26:50 ] Completed replacing temp checkpoint with checkpoint 21.265 ms, 1.62 s total -[ 2023-09-21 06:26:50 ] Completed eval batch 1 to device 482.913 ms, 2.11 s total -[ 2023-09-21 06:26:50 ] Completed eval batch 1 forward 22.572 ms, 2.13 s total -[ 2023-09-21 06:26:50 ] Completed eval batch 1 recons_loss 0.346 ms, 2.13 s total -[ 2023-09-21 06:26:50 ] Completed eval batch 1 metrics update 172.256 ms, 2.30 s total -Saving checkpoint at epoch 20 val batch 1 -Epoch 20 val loss: 0.1752 -[ 2023-09-21 06:26:51 ] Completed saving temp checkpoint 416.413 ms, 2.72 s total -[ 2023-09-21 06:26:51 ] Completed replacing temp checkpoint with checkpoint 18.386 ms, 2.74 s total -[ 2023-09-21 06:26:51 ] Completed evaluating generator for epoch 20 47.010 ms, 2.78 s total - - -EPOCH :: 21 - - -[ 2023-09-21 06:26:51 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 21 from training step 0 - -[ 2023-09-21 06:26:54 ] Completed train batch 0 to device 3,303.875 ms, 3.30 s total -[ 2023-09-21 06:26:54 ] Completed train batch 0 generator forward 43.259 ms, 3.35 s total -[ 2023-09-21 06:26:54 ] Completed train batch 0 generator loss: 1.710 66.738 ms, 3.41 s total -[ 2023-09-21 06:26:55 ] Completed train batch 0 generator backward 595.003 ms, 4.01 s total -[ 2023-09-21 06:26:55 ] Completed train batch 0 discriminator loss 0.158 57.620 ms, 4.07 s total -[ 2023-09-21 06:26:55 ] Completed train batch 0 discriminator backward 88.025 ms, 4.15 s total -Epoch [21] Step [0/2], gen_loss: 0.575, disc_loss: 0.054 -[ 2023-09-21 06:26:55 ] Completed train batch 0 metrics update 4.955 ms, 4.16 s total -Saving checkpoint at epoch 21 train batch 0 -[ 2023-09-21 06:26:56 ] Completed saving temp checkpoint 547.404 ms, 4.71 s total -[ 2023-09-21 06:26:56 ] Completed replacing temp checkpoint with checkpoint 42.150 ms, 4.75 s total -[ 2023-09-21 06:26:57 ] Completed train batch 1 to device 1,760.675 ms, 6.51 s total -[ 2023-09-21 06:26:57 ] Completed train batch 1 generator forward 38.238 ms, 6.55 s total -[ 2023-09-21 06:26:58 ] Completed train batch 1 generator loss: 1.687 63.445 ms, 6.61 s total -[ 2023-09-21 06:26:59 ] Completed train batch 1 generator backward 1,065.147 ms, 7.68 s total -[ 2023-09-21 06:26:59 ] Completed train batch 1 discriminator loss 0.157 56.694 ms, 7.73 s total -[ 2023-09-21 06:26:59 ] Completed train batch 1 discriminator backward 69.972 ms, 7.80 s total -Epoch [21] Step [1/2], gen_loss: 0.544, disc_loss: 0.051 -[ 2023-09-21 06:26:59 ] Completed train batch 1 metrics update 1.997 ms, 7.81 s total -Saving checkpoint at epoch 21 train batch 1 -[ 2023-09-21 06:26:59 ] Completed saving temp checkpoint 714.721 ms, 8.52 s total -[ 2023-09-21 06:27:00 ] Completed replacing temp checkpoint with checkpoint 31.463 ms, 8.55 s total -Epoch [21] :: gen_loss: 0.560, disc_loss: 0.052 -[ 2023-09-21 06:27:00 ] Completed training generator for epoch 21 59.078 ms, 8.61 s total -[ 2023-09-21 06:27:00 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 21 from eval step 0 - -[ 2023-09-21 06:27:01 ] Completed eval batch 0 to device 1,159.042 ms, 1.16 s total -[ 2023-09-21 06:27:01 ] Completed eval batch 0 forward 41.340 ms, 1.20 s total -[ 2023-09-21 06:27:01 ] Completed eval batch 0 recons_loss 0.227 ms, 1.20 s total -[ 2023-09-21 06:27:01 ] Completed eval batch 0 metrics update 0.537 ms, 1.20 s total -Saving checkpoint at epoch 21 val batch 0 -[ 2023-09-21 06:27:01 ] Completed saving temp checkpoint 418.159 ms, 1.62 s total -[ 2023-09-21 06:27:01 ] Completed replacing temp checkpoint with checkpoint 25.561 ms, 1.64 s total -[ 2023-09-21 06:27:02 ] Completed eval batch 1 to device 395.809 ms, 2.04 s total -[ 2023-09-21 06:27:02 ] Completed eval batch 1 forward 27.756 ms, 2.07 s total -[ 2023-09-21 06:27:02 ] Completed eval batch 1 recons_loss 0.353 ms, 2.07 s total -[ 2023-09-21 06:27:02 ] Completed eval batch 1 metrics update 0.319 ms, 2.07 s total -Saving checkpoint at epoch 21 val batch 1 -Epoch 21 val loss: 0.1330 -[ 2023-09-21 06:27:02 ] Completed saving temp checkpoint 473.613 ms, 2.54 s total -[ 2023-09-21 06:27:02 ] Completed replacing temp checkpoint with checkpoint 25.098 ms, 2.57 s total -[ 2023-09-21 06:27:02 ] Completed evaluating generator for epoch 21 56.063 ms, 2.62 s total - - -EPOCH :: 22 - - -[ 2023-09-21 06:27:02 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 22 from training step 0 - -[ 2023-09-21 06:27:05 ] Completed train batch 0 to device 3,079.879 ms, 3.08 s total -[ 2023-09-21 06:27:05 ] Completed train batch 0 generator forward 30.760 ms, 3.11 s total -[ 2023-09-21 06:27:05 ] Completed train batch 0 generator loss: 1.689 66.481 ms, 3.18 s total -[ 2023-09-21 06:27:06 ] Completed train batch 0 generator backward 693.006 ms, 3.87 s total -[ 2023-09-21 06:27:06 ] Completed train batch 0 discriminator loss 0.149 57.735 ms, 3.93 s total -[ 2023-09-21 06:27:06 ] Completed train batch 0 discriminator backward 110.747 ms, 4.04 s total -Epoch [22] Step [0/2], gen_loss: 0.559, disc_loss: 0.049 -[ 2023-09-21 06:27:06 ] Completed train batch 0 metrics update 6.854 ms, 4.05 s total -Saving checkpoint at epoch 22 train batch 0 -[ 2023-09-21 06:27:07 ] Completed saving temp checkpoint 452.375 ms, 4.50 s total -[ 2023-09-21 06:27:07 ] Completed replacing temp checkpoint with checkpoint 24.200 ms, 4.52 s total -[ 2023-09-21 06:27:08 ] Completed train batch 1 to device 1,447.933 ms, 5.97 s total -[ 2023-09-21 06:27:08 ] Completed train batch 1 generator forward 40.817 ms, 6.01 s total -[ 2023-09-21 06:27:08 ] Completed train batch 1 generator loss: 1.550 63.452 ms, 6.07 s total -[ 2023-09-21 06:27:09 ] Completed train batch 1 generator backward 1,222.880 ms, 7.30 s total -[ 2023-09-21 06:27:10 ] Completed train batch 1 discriminator loss 0.135 56.834 ms, 7.35 s total -[ 2023-09-21 06:27:10 ] Completed train batch 1 discriminator backward 89.915 ms, 7.44 s total -Epoch [22] Step [1/2], gen_loss: 0.545, disc_loss: 0.047 -[ 2023-09-21 06:27:10 ] Completed train batch 1 metrics update 5.352 ms, 7.45 s total -Saving checkpoint at epoch 22 train batch 1 -[ 2023-09-21 06:27:10 ] Completed saving temp checkpoint 493.356 ms, 7.94 s total -[ 2023-09-21 06:27:10 ] Completed replacing temp checkpoint with checkpoint 18.979 ms, 7.96 s total -Epoch [22] :: gen_loss: 0.552, disc_loss: 0.048 -[ 2023-09-21 06:27:10 ] Completed training generator for epoch 22 56.861 ms, 8.02 s total -[ 2023-09-21 06:27:10 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 22 from eval step 0 - -[ 2023-09-21 06:27:11 ] Completed eval batch 0 to device 1,162.579 ms, 1.16 s total -[ 2023-09-21 06:27:11 ] Completed eval batch 0 forward 25.488 ms, 1.19 s total -[ 2023-09-21 06:27:11 ] Completed eval batch 0 recons_loss 0.334 ms, 1.19 s total -[ 2023-09-21 06:27:11 ] Completed eval batch 0 metrics update 1.050 ms, 1.19 s total -Saving checkpoint at epoch 22 val batch 0 -[ 2023-09-21 06:27:12 ] Completed saving temp checkpoint 445.171 ms, 1.63 s total -[ 2023-09-21 06:27:12 ] Completed replacing temp checkpoint with checkpoint 28.860 ms, 1.66 s total -[ 2023-09-21 06:27:12 ] Completed eval batch 1 to device 426.425 ms, 2.09 s total -[ 2023-09-21 06:27:12 ] Completed eval batch 1 forward 40.212 ms, 2.13 s total -[ 2023-09-21 06:27:12 ] Completed eval batch 1 recons_loss 0.595 ms, 2.13 s total -[ 2023-09-21 06:27:12 ] Completed eval batch 1 metrics update 103.060 ms, 2.23 s total -Saving checkpoint at epoch 22 val batch 1 -Epoch 22 val loss: 0.1237 -[ 2023-09-21 06:27:13 ] Completed saving temp checkpoint 468.910 ms, 2.70 s total -[ 2023-09-21 06:27:13 ] Completed replacing temp checkpoint with checkpoint 22.051 ms, 2.72 s total -[ 2023-09-21 06:27:13 ] Completed evaluating generator for epoch 22 54.260 ms, 2.78 s total - - -EPOCH :: 23 - - -[ 2023-09-21 06:27:13 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 23 from training step 0 - -[ 2023-09-21 06:27:16 ] Completed train batch 0 to device 3,157.251 ms, 3.16 s total -[ 2023-09-21 06:27:16 ] Completed train batch 0 generator forward 27.059 ms, 3.18 s total -[ 2023-09-21 06:27:16 ] Completed train batch 0 generator loss: 1.640 67.300 ms, 3.25 s total -[ 2023-09-21 06:27:17 ] Completed train batch 0 generator backward 651.430 ms, 3.90 s total -[ 2023-09-21 06:27:17 ] Completed train batch 0 discriminator loss 0.143 59.277 ms, 3.96 s total -[ 2023-09-21 06:27:17 ] Completed train batch 0 discriminator backward 95.777 ms, 4.06 s total -Epoch [23] Step [0/2], gen_loss: 0.543, disc_loss: 0.047 -[ 2023-09-21 06:27:17 ] Completed train batch 0 metrics update 5.464 ms, 4.06 s total -Saving checkpoint at epoch 23 train batch 0 -[ 2023-09-21 06:27:18 ] Completed saving temp checkpoint 477.886 ms, 4.54 s total -[ 2023-09-21 06:27:18 ] Completed replacing temp checkpoint with checkpoint 24.215 ms, 4.57 s total -[ 2023-09-21 06:27:19 ] Completed train batch 1 to device 1,536.806 ms, 6.10 s total -[ 2023-09-21 06:27:19 ] Completed train batch 1 generator forward 22.219 ms, 6.12 s total -[ 2023-09-21 06:27:19 ] Completed train batch 1 generator loss: 1.557 64.394 ms, 6.19 s total -[ 2023-09-21 06:27:21 ] Completed train batch 1 generator backward 1,388.185 ms, 7.58 s total -[ 2023-09-21 06:27:21 ] Completed train batch 1 discriminator loss 0.143 56.936 ms, 7.63 s total -[ 2023-09-21 06:27:21 ] Completed train batch 1 discriminator backward 84.745 ms, 7.72 s total -Epoch [23] Step [1/2], gen_loss: 0.526, disc_loss: 0.046 -[ 2023-09-21 06:27:21 ] Completed train batch 1 metrics update 0.900 ms, 7.72 s total -Saving checkpoint at epoch 23 train batch 1 -[ 2023-09-21 06:27:21 ] Completed saving temp checkpoint 474.567 ms, 8.19 s total -[ 2023-09-21 06:27:21 ] Completed replacing temp checkpoint with checkpoint 27.882 ms, 8.22 s total -Epoch [23] :: gen_loss: 0.535, disc_loss: 0.047 -[ 2023-09-21 06:27:21 ] Completed training generator for epoch 23 52.338 ms, 8.27 s total -[ 2023-09-21 06:27:21 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 23 from eval step 0 - -[ 2023-09-21 06:27:22 ] Completed eval batch 0 to device 1,174.747 ms, 1.17 s total -[ 2023-09-21 06:27:22 ] Completed eval batch 0 forward 42.867 ms, 1.22 s total -[ 2023-09-21 06:27:22 ] Completed eval batch 0 recons_loss 0.346 ms, 1.22 s total -[ 2023-09-21 06:27:22 ] Completed eval batch 0 metrics update 0.860 ms, 1.22 s total -Saving checkpoint at epoch 23 val batch 0 -[ 2023-09-21 06:27:23 ] Completed saving temp checkpoint 417.071 ms, 1.64 s total -[ 2023-09-21 06:27:23 ] Completed replacing temp checkpoint with checkpoint 22.505 ms, 1.66 s total -[ 2023-09-21 06:27:23 ] Completed eval batch 1 to device 436.314 ms, 2.09 s total -[ 2023-09-21 06:27:23 ] Completed eval batch 1 forward 37.607 ms, 2.13 s total -[ 2023-09-21 06:27:23 ] Completed eval batch 1 recons_loss 0.338 ms, 2.13 s total -[ 2023-09-21 06:27:24 ] Completed eval batch 1 metrics update 179.521 ms, 2.31 s total -Saving checkpoint at epoch 23 val batch 1 -Epoch 23 val loss: 0.1415 -[ 2023-09-21 06:27:24 ] Completed saving temp checkpoint 435.266 ms, 2.75 s total -[ 2023-09-21 06:27:24 ] Completed replacing temp checkpoint with checkpoint 32.591 ms, 2.78 s total -[ 2023-09-21 06:27:24 ] Completed evaluating generator for epoch 23 51.149 ms, 2.83 s total - - -EPOCH :: 24 - - -[ 2023-09-21 06:27:24 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 24 from training step 0 - -[ 2023-09-21 06:27:27 ] Completed train batch 0 to device 3,104.204 ms, 3.10 s total -[ 2023-09-21 06:27:27 ] Completed train batch 0 generator forward 26.786 ms, 3.13 s total -[ 2023-09-21 06:27:27 ] Completed train batch 0 generator loss: 1.764 66.464 ms, 3.20 s total -[ 2023-09-21 06:27:28 ] Completed train batch 0 generator backward 760.081 ms, 3.96 s total -[ 2023-09-21 06:27:28 ] Completed train batch 0 discriminator loss 0.134 57.371 ms, 4.01 s total -[ 2023-09-21 06:27:28 ] Completed train batch 0 discriminator backward 107.273 ms, 4.12 s total -Epoch [24] Step [0/2], gen_loss: 0.601, disc_loss: 0.044 -[ 2023-09-21 06:27:28 ] Completed train batch 0 metrics update 7.452 ms, 4.13 s total -Saving checkpoint at epoch 24 train batch 0 -[ 2023-09-21 06:27:29 ] Completed saving temp checkpoint 521.662 ms, 4.65 s total -[ 2023-09-21 06:27:29 ] Completed replacing temp checkpoint with checkpoint 34.244 ms, 4.69 s total -[ 2023-09-21 06:27:30 ] Completed train batch 1 to device 1,274.558 ms, 5.96 s total -[ 2023-09-21 06:27:30 ] Completed train batch 1 generator forward 42.187 ms, 6.00 s total -[ 2023-09-21 06:27:30 ] Completed train batch 1 generator loss: 1.679 65.342 ms, 6.07 s total -[ 2023-09-21 06:27:32 ] Completed train batch 1 generator backward 1,778.728 ms, 7.85 s total -[ 2023-09-21 06:27:32 ] Completed train batch 1 discriminator loss 0.127 57.495 ms, 7.90 s total -[ 2023-09-21 06:27:32 ] Completed train batch 1 discriminator backward 73.192 ms, 7.98 s total -Epoch [24] Step [1/2], gen_loss: 0.578, disc_loss: 0.043 -[ 2023-09-21 06:27:32 ] Completed train batch 1 metrics update 0.788 ms, 7.98 s total -Saving checkpoint at epoch 24 train batch 1 -[ 2023-09-21 06:27:33 ] Completed saving temp checkpoint 568.349 ms, 8.55 s total -[ 2023-09-21 06:27:33 ] Completed replacing temp checkpoint with checkpoint 33.552 ms, 8.58 s total -Epoch [24] :: gen_loss: 0.589, disc_loss: 0.044 -[ 2023-09-21 06:27:33 ] Completed training generator for epoch 24 72.822 ms, 8.65 s total -[ 2023-09-21 06:27:33 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 24 from eval step 0 - -[ 2023-09-21 06:27:34 ] Completed eval batch 0 to device 1,170.881 ms, 1.17 s total -[ 2023-09-21 06:27:34 ] Completed eval batch 0 forward 25.051 ms, 1.20 s total -[ 2023-09-21 06:27:34 ] Completed eval batch 0 recons_loss 0.329 ms, 1.20 s total -[ 2023-09-21 06:27:34 ] Completed eval batch 0 metrics update 0.640 ms, 1.20 s total -Saving checkpoint at epoch 24 val batch 0 -[ 2023-09-21 06:27:35 ] Completed saving temp checkpoint 844.709 ms, 2.04 s total -[ 2023-09-21 06:27:35 ] Completed replacing temp checkpoint with checkpoint 54.410 ms, 2.10 s total -[ 2023-09-21 06:27:35 ] Completed eval batch 1 to device 36.711 ms, 2.13 s total -[ 2023-09-21 06:27:35 ] Completed eval batch 1 forward 36.725 ms, 2.17 s total -[ 2023-09-21 06:27:35 ] Completed eval batch 1 recons_loss 0.341 ms, 2.17 s total -[ 2023-09-21 06:27:35 ] Completed eval batch 1 metrics update 0.301 ms, 2.17 s total -Saving checkpoint at epoch 24 val batch 1 -Epoch 24 val loss: 0.1357 -[ 2023-09-21 06:27:36 ] Completed saving temp checkpoint 752.492 ms, 2.92 s total -[ 2023-09-21 06:27:36 ] Completed replacing temp checkpoint with checkpoint 38.980 ms, 2.96 s total -[ 2023-09-21 06:27:36 ] Completed evaluating generator for epoch 24 59.398 ms, 3.02 s total - - -EPOCH :: 25 - - -[ 2023-09-21 06:27:36 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 25 from training step 0 - -[ 2023-09-21 06:27:39 ] Completed train batch 0 to device 3,152.129 ms, 3.15 s total -[ 2023-09-21 06:27:39 ] Completed train batch 0 generator forward 26.215 ms, 3.18 s total -[ 2023-09-21 06:27:39 ] Completed train batch 0 generator loss: 1.660 65.716 ms, 3.24 s total -[ 2023-09-21 06:27:39 ] Completed train batch 0 generator backward 315.891 ms, 3.56 s total -[ 2023-09-21 06:27:39 ] Completed train batch 0 discriminator loss 0.133 63.613 ms, 3.62 s total -[ 2023-09-21 06:27:39 ] Completed train batch 0 discriminator backward 83.277 ms, 3.71 s total -Epoch [25] Step [0/2], gen_loss: 0.544, disc_loss: 0.045 -[ 2023-09-21 06:27:39 ] Completed train batch 0 metrics update 3.741 ms, 3.71 s total -Saving checkpoint at epoch 25 train batch 0 -[ 2023-09-21 06:27:40 ] Completed saving temp checkpoint 447.755 ms, 4.16 s total -[ 2023-09-21 06:27:40 ] Completed replacing temp checkpoint with checkpoint 20.648 ms, 4.18 s total -[ 2023-09-21 06:27:42 ] Completed train batch 1 to device 1,977.280 ms, 6.16 s total -[ 2023-09-21 06:27:42 ] Completed train batch 1 generator forward 41.714 ms, 6.20 s total -[ 2023-09-21 06:27:42 ] Completed train batch 1 generator loss: 1.537 63.780 ms, 6.26 s total -[ 2023-09-21 06:27:43 ] Completed train batch 1 generator backward 666.693 ms, 6.93 s total -[ 2023-09-21 06:27:43 ] Completed train batch 1 discriminator loss 0.130 61.063 ms, 6.99 s total -[ 2023-09-21 06:27:43 ] Completed train batch 1 discriminator backward 61.070 ms, 7.05 s total -Epoch [25] Step [1/2], gen_loss: 0.535, disc_loss: 0.042 -[ 2023-09-21 06:27:43 ] Completed train batch 1 metrics update 0.637 ms, 7.05 s total -Saving checkpoint at epoch 25 train batch 1 -[ 2023-09-21 06:27:43 ] Completed saving temp checkpoint 489.122 ms, 7.54 s total -[ 2023-09-21 06:27:43 ] Completed replacing temp checkpoint with checkpoint 30.998 ms, 7.57 s total -Epoch [25] :: gen_loss: 0.539, disc_loss: 0.044 -[ 2023-09-21 06:27:43 ] Completed training generator for epoch 25 55.724 ms, 7.63 s total -[ 2023-09-21 06:27:43 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 25 from eval step 0 - -[ 2023-09-21 06:27:45 ] Completed eval batch 0 to device 1,179.917 ms, 1.18 s total -[ 2023-09-21 06:27:45 ] Completed eval batch 0 forward 31.153 ms, 1.21 s total -[ 2023-09-21 06:27:45 ] Completed eval batch 0 recons_loss 0.323 ms, 1.21 s total -[ 2023-09-21 06:27:45 ] Completed eval batch 0 metrics update 0.630 ms, 1.21 s total -Saving checkpoint at epoch 25 val batch 0 -[ 2023-09-21 06:27:45 ] Completed saving temp checkpoint 417.929 ms, 1.63 s total -[ 2023-09-21 06:27:45 ] Completed replacing temp checkpoint with checkpoint 32.709 ms, 1.66 s total -[ 2023-09-21 06:27:45 ] Completed eval batch 1 to device 434.132 ms, 2.10 s total -[ 2023-09-21 06:27:46 ] Completed eval batch 1 forward 39.890 ms, 2.14 s total -[ 2023-09-21 06:27:46 ] Completed eval batch 1 recons_loss 0.566 ms, 2.14 s total -[ 2023-09-21 06:27:46 ] Completed eval batch 1 metrics update 139.134 ms, 2.28 s total -Saving checkpoint at epoch 25 val batch 1 -Epoch 25 val loss: 0.1180 -[ 2023-09-21 06:27:46 ] Completed saving temp checkpoint 469.231 ms, 2.75 s total -[ 2023-09-21 06:27:46 ] Completed replacing temp checkpoint with checkpoint 23.951 ms, 2.77 s total -[ 2023-09-21 06:27:46 ] Completed evaluating generator for epoch 25 48.793 ms, 2.82 s total - - -EPOCH :: 26 - - -[ 2023-09-21 06:27:46 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 26 from training step 0 - -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 06:40:25 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 06:40:25 ] Completed importing Timer 0.028 ms, 0.00 s total -[ 2023-09-21 06:40:30 ] Completed importing everything else 4,714.302 ms, 4.71 s total -| distributed init (rank 4): env:// -| distributed init (rank 3): env:// -| distributed init (rank 1): env:// -| distributed init (rank 5): env:// -| distributed init (rank 0): env:// -| distributed init (rank 2): env:// -[ 2023-09-21 06:40:38 ] Completed preliminaries 7,634.946 ms, 12.35 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 06:40:38 ] Completed build datasets 11.098 ms, 12.36 s total -[ 2023-09-21 06:40:38 ] Completed build samplers 0.044 ms, 12.36 s total -[ 2023-09-21 06:40:38 ] Completed build dataloaders 0.134 ms, 12.36 s total -[ 2023-09-21 06:40:38 ] Completed generator to device 75.857 ms, 12.44 s total -[ 2023-09-21 06:40:38 ] Completed discriminator to device 6.927 ms, 12.44 s total -[ 2023-09-21 06:40:38 ] Completed loss functions 397.873 ms, 12.84 s total -[ 2023-09-21 06:40:38 ] Completed models prepped for distribution 134.753 ms, 12.98 s total -[ 2023-09-21 06:40:38 ] Completed optimizers 0.684 ms, 12.98 s total -[ 2023-09-21 06:40:38 ] Completed grad scalers 0.023 ms, 12.98 s total -[ 2023-09-21 06:40:39 ] Completed checkpoint retrieval 486.976 ms, 13.46 s total - - -EPOCH :: 25 - - -[ 2023-09-21 06:40:39 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 25 from training step 2 - -Epoch [25] :: gen_loss: 0.539, disc_loss: 0.044 -[ 2023-09-21 06:40:39 ] Completed training generator for epoch 25 135.213 ms, 0.14 s total -[ 2023-09-21 06:40:39 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 25 from eval step 2 - -[ 2023-09-21 06:40:39 ] Completed evaluating generator for epoch 25 128.298 ms, 0.13 s total - - -EPOCH :: 26 - - -[ 2023-09-21 06:40:39 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 26 from training step 0 - -[ 2023-09-21 06:40:42 ] Completed train batch 0 to device 3,463.167 ms, 3.46 s total -[ 2023-09-21 06:40:44 ] Completed train batch 0 generator forward 1,270.963 ms, 4.73 s total -[ 2023-09-21 06:40:44 ] Completed train batch 0 generator loss: 1.542 75.762 ms, 4.81 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 06:40:45 ] Completed train batch 0 generator backward 1,491.183 ms, 6.30 s total -[ 2023-09-21 06:40:45 ] Completed train batch 0 discriminator loss 0.131 56.011 ms, 6.36 s total -[ 2023-09-21 06:40:45 ] Completed train batch 0 discriminator backward 75.967 ms, 6.43 s total -Epoch [26] Step [0/2], gen_loss: 0.541, disc_loss: 0.042 -[ 2023-09-21 06:40:45 ] Completed train batch 0 metrics update 3.219 ms, 6.44 s total -Saving checkpoint at epoch 26 train batch 0 -[ 2023-09-21 06:40:46 ] Completed saving temp checkpoint 461.288 ms, 6.90 s total -[ 2023-09-21 06:40:46 ] Completed replacing temp checkpoint with checkpoint 119.659 ms, 7.02 s total -[ 2023-09-21 06:40:46 ] Completed train batch 1 to device 12.138 ms, 7.03 s total -[ 2023-09-21 06:40:46 ] Completed train batch 1 generator forward 22.221 ms, 7.05 s total -[ 2023-09-21 06:40:46 ] Completed train batch 1 generator loss: 1.540 61.811 ms, 7.11 s total -[ 2023-09-21 06:40:47 ] Completed train batch 1 generator backward 1,181.699 ms, 8.30 s total -[ 2023-09-21 06:40:47 ] Completed train batch 1 discriminator loss 0.125 56.271 ms, 8.35 s total -[ 2023-09-21 06:40:47 ] Completed train batch 1 discriminator backward 73.134 ms, 8.42 s total -Epoch [26] Step [1/2], gen_loss: 0.535, disc_loss: 0.042 -[ 2023-09-21 06:40:47 ] Completed train batch 1 metrics update 5.067 ms, 8.43 s total -Saving checkpoint at epoch 26 train batch 1 -[ 2023-09-21 06:40:48 ] Completed saving temp checkpoint 504.210 ms, 8.93 s total -[ 2023-09-21 06:40:48 ] Completed replacing temp checkpoint with checkpoint 26.556 ms, 8.96 s total -Epoch [26] :: gen_loss: 0.538, disc_loss: 0.042 -[ 2023-09-21 06:40:48 ] Completed training generator for epoch 26 46.396 ms, 9.01 s total -[ 2023-09-21 06:40:48 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 26 from eval step 0 - -[ 2023-09-21 06:40:49 ] Completed eval batch 0 to device 1,086.993 ms, 1.09 s total -[ 2023-09-21 06:40:49 ] Completed eval batch 0 forward 28.685 ms, 1.12 s total -[ 2023-09-21 06:40:49 ] Completed eval batch 0 recons_loss 0.299 ms, 1.12 s total -[ 2023-09-21 06:40:49 ] Completed eval batch 0 metrics update 0.561 ms, 1.12 s total -Saving checkpoint at epoch 26 val batch 0 -[ 2023-09-21 06:40:49 ] Completed saving temp checkpoint 379.013 ms, 1.50 s total -[ 2023-09-21 06:40:50 ] Completed replacing temp checkpoint with checkpoint 28.409 ms, 1.52 s total -[ 2023-09-21 06:40:50 ] Completed eval batch 1 to device 383.723 ms, 1.91 s total -[ 2023-09-21 06:40:50 ] Completed eval batch 1 forward 21.168 ms, 1.93 s total -[ 2023-09-21 06:40:50 ] Completed eval batch 1 recons_loss 0.324 ms, 1.93 s total -[ 2023-09-21 06:40:50 ] Completed eval batch 1 metrics update 219.131 ms, 2.15 s total -Saving checkpoint at epoch 26 val batch 1 -Epoch 26 val loss: 0.1119 -[ 2023-09-21 06:40:51 ] Completed saving temp checkpoint 401.723 ms, 2.55 s total -[ 2023-09-21 06:40:51 ] Completed replacing temp checkpoint with checkpoint 34.178 ms, 2.58 s total -[ 2023-09-21 06:40:51 ] Completed evaluating generator for epoch 26 54.337 ms, 2.64 s total - - -EPOCH :: 27 - - -[ 2023-09-21 06:40:51 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 27 from training step 0 - -[ 2023-09-21 06:40:53 ] Completed train batch 0 to device 2,863.214 ms, 2.86 s total -[ 2023-09-21 06:40:54 ] Completed train batch 0 generator forward 43.911 ms, 2.91 s total -[ 2023-09-21 06:40:54 ] Completed train batch 0 generator loss: 1.629 66.258 ms, 2.97 s total -[ 2023-09-21 06:40:55 ] Completed train batch 0 generator backward 985.470 ms, 3.96 s total -[ 2023-09-21 06:40:55 ] Completed train batch 0 discriminator loss 0.127 57.036 ms, 4.02 s total -[ 2023-09-21 06:40:55 ] Completed train batch 0 discriminator backward 99.711 ms, 4.12 s total -Epoch [27] Step [0/2], gen_loss: 0.539, disc_loss: 0.042 -[ 2023-09-21 06:40:55 ] Completed train batch 0 metrics update 6.477 ms, 4.12 s total -Saving checkpoint at epoch 27 train batch 0 -[ 2023-09-21 06:40:55 ] Completed saving temp checkpoint 482.229 ms, 4.60 s total -[ 2023-09-21 06:40:55 ] Completed replacing temp checkpoint with checkpoint 31.013 ms, 4.64 s total -[ 2023-09-21 06:40:56 ] Completed train batch 1 to device 795.688 ms, 5.43 s total -[ 2023-09-21 06:40:56 ] Completed train batch 1 generator forward 38.278 ms, 5.47 s total -[ 2023-09-21 06:40:56 ] Completed train batch 1 generator loss: 1.583 61.993 ms, 5.53 s total -[ 2023-09-21 06:40:58 ] Completed train batch 1 generator backward 1,953.289 ms, 7.48 s total -[ 2023-09-21 06:40:58 ] Completed train batch 1 discriminator loss 0.127 56.421 ms, 7.54 s total -[ 2023-09-21 06:40:58 ] Completed train batch 1 discriminator backward 77.820 ms, 7.62 s total -Epoch [27] Step [1/2], gen_loss: 0.531, disc_loss: 0.042 -[ 2023-09-21 06:40:58 ] Completed train batch 1 metrics update 1.841 ms, 7.62 s total -Saving checkpoint at epoch 27 train batch 1 -[ 2023-09-21 06:40:59 ] Completed saving temp checkpoint 756.298 ms, 8.38 s total -[ 2023-09-21 06:40:59 ] Completed replacing temp checkpoint with checkpoint 31.456 ms, 8.41 s total -Epoch [27] :: gen_loss: 0.535, disc_loss: 0.042 -[ 2023-09-21 06:40:59 ] Completed training generator for epoch 27 65.165 ms, 8.47 s total -[ 2023-09-21 06:40:59 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 27 from eval step 0 - -[ 2023-09-21 06:41:00 ] Completed eval batch 0 to device 1,138.580 ms, 1.14 s total -[ 2023-09-21 06:41:00 ] Completed eval batch 0 forward 33.419 ms, 1.17 s total -[ 2023-09-21 06:41:00 ] Completed eval batch 0 recons_loss 0.376 ms, 1.17 s total -[ 2023-09-21 06:41:00 ] Completed eval batch 0 metrics update 0.603 ms, 1.17 s total -Saving checkpoint at epoch 27 val batch 0 -[ 2023-09-21 06:41:01 ] Completed saving temp checkpoint 727.666 ms, 1.90 s total -[ 2023-09-21 06:41:01 ] Completed replacing temp checkpoint with checkpoint 27.104 ms, 1.93 s total -[ 2023-09-21 06:41:01 ] Completed eval batch 1 to device 15.856 ms, 1.94 s total -[ 2023-09-21 06:41:01 ] Completed eval batch 1 forward 21.678 ms, 1.97 s total -[ 2023-09-21 06:41:01 ] Completed eval batch 1 recons_loss 0.342 ms, 1.97 s total -[ 2023-09-21 06:41:01 ] Completed eval batch 1 metrics update 0.314 ms, 1.97 s total -Saving checkpoint at epoch 27 val batch 1 -Epoch 27 val loss: 0.1120 -[ 2023-09-21 06:41:02 ] Completed saving temp checkpoint 662.863 ms, 2.63 s total -[ 2023-09-21 06:41:02 ] Completed replacing temp checkpoint with checkpoint 39.801 ms, 2.67 s total -[ 2023-09-21 06:41:02 ] Completed evaluating generator for epoch 27 63.333 ms, 2.73 s total - - -EPOCH :: 28 - - -[ 2023-09-21 06:41:02 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 28 from training step 0 - -[ 2023-09-21 06:41:05 ] Completed train batch 0 to device 3,030.454 ms, 3.03 s total -[ 2023-09-21 06:41:05 ] Completed train batch 0 generator forward 41.913 ms, 3.07 s total -[ 2023-09-21 06:41:05 ] Completed train batch 0 generator loss: 1.668 66.251 ms, 3.14 s total -[ 2023-09-21 06:41:05 ] Completed train batch 0 generator backward 417.154 ms, 3.56 s total -[ 2023-09-21 06:41:05 ] Completed train batch 0 discriminator loss 0.132 56.211 ms, 3.61 s total -[ 2023-09-21 06:41:06 ] Completed train batch 0 discriminator backward 108.202 ms, 3.72 s total -Epoch [28] Step [0/2], gen_loss: 0.535, disc_loss: 0.043 -[ 2023-09-21 06:41:06 ] Completed train batch 0 metrics update 22.793 ms, 3.74 s total -Saving checkpoint at epoch 28 train batch 0 -[ 2023-09-21 06:41:06 ] Completed saving temp checkpoint 579.262 ms, 4.32 s total -[ 2023-09-21 06:41:06 ] Completed replacing temp checkpoint with checkpoint 22.971 ms, 4.35 s total -[ 2023-09-21 06:41:08 ] Completed train batch 1 to device 1,433.779 ms, 5.78 s total -[ 2023-09-21 06:41:08 ] Completed train batch 1 generator forward 22.257 ms, 5.80 s total -[ 2023-09-21 06:41:08 ] Completed train batch 1 generator loss: 1.533 64.441 ms, 5.87 s total -[ 2023-09-21 06:41:09 ] Completed train batch 1 generator backward 1,220.952 ms, 7.09 s total -[ 2023-09-21 06:41:09 ] Completed train batch 1 discriminator loss 0.123 56.110 ms, 7.14 s total -[ 2023-09-21 06:41:09 ] Completed train batch 1 discriminator backward 90.878 ms, 7.23 s total -Epoch [28] Step [1/2], gen_loss: 0.529, disc_loss: 0.042 -[ 2023-09-21 06:41:09 ] Completed train batch 1 metrics update 4.099 ms, 7.24 s total -Saving checkpoint at epoch 28 train batch 1 -[ 2023-09-21 06:41:10 ] Completed saving temp checkpoint 598.540 ms, 7.84 s total -[ 2023-09-21 06:41:10 ] Completed replacing temp checkpoint with checkpoint 38.369 ms, 7.87 s total -Epoch [28] :: gen_loss: 0.532, disc_loss: 0.042 -[ 2023-09-21 06:41:10 ] Completed training generator for epoch 28 65.242 ms, 7.94 s total -[ 2023-09-21 06:41:10 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 28 from eval step 0 - -[ 2023-09-21 06:41:11 ] Completed eval batch 0 to device 1,118.283 ms, 1.12 s total -[ 2023-09-21 06:41:11 ] Completed eval batch 0 forward 24.888 ms, 1.14 s total -[ 2023-09-21 06:41:11 ] Completed eval batch 0 recons_loss 0.313 ms, 1.14 s total -[ 2023-09-21 06:41:11 ] Completed eval batch 0 metrics update 0.642 ms, 1.14 s total -Saving checkpoint at epoch 28 val batch 0 -[ 2023-09-21 06:41:11 ] Completed saving temp checkpoint 519.305 ms, 1.66 s total -[ 2023-09-21 06:41:11 ] Completed replacing temp checkpoint with checkpoint 33.808 ms, 1.70 s total -[ 2023-09-21 06:41:12 ] Completed eval batch 1 to device 214.526 ms, 1.91 s total -[ 2023-09-21 06:41:12 ] Completed eval batch 1 forward 39.705 ms, 1.95 s total -[ 2023-09-21 06:41:12 ] Completed eval batch 1 recons_loss 0.580 ms, 1.95 s total -[ 2023-09-21 06:41:12 ] Completed eval batch 1 metrics update 87.338 ms, 2.04 s total -Saving checkpoint at epoch 28 val batch 1 -Epoch 28 val loss: 0.1114 -[ 2023-09-21 06:41:12 ] Completed saving temp checkpoint 580.004 ms, 2.62 s total -[ 2023-09-21 06:41:12 ] Completed replacing temp checkpoint with checkpoint 41.935 ms, 2.66 s total -[ 2023-09-21 06:41:12 ] Completed evaluating generator for epoch 28 59.099 ms, 2.72 s total - - -EPOCH :: 29 - - -[ 2023-09-21 06:41:12 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 29 from training step 0 - -[ 2023-09-21 06:41:15 ] Completed train batch 0 to device 2,776.842 ms, 2.78 s total -[ 2023-09-21 06:41:15 ] Completed train batch 0 generator forward 26.394 ms, 2.80 s total -[ 2023-09-21 06:41:15 ] Completed train batch 0 generator loss: 1.624 65.819 ms, 2.87 s total -[ 2023-09-21 06:41:16 ] Completed train batch 0 generator backward 857.939 ms, 3.73 s total -[ 2023-09-21 06:41:16 ] Completed train batch 0 discriminator loss 0.127 56.752 ms, 3.78 s total -[ 2023-09-21 06:41:16 ] Completed train batch 0 discriminator backward 113.671 ms, 3.90 s total -Epoch [29] Step [0/2], gen_loss: 0.533, disc_loss: 0.042 -[ 2023-09-21 06:41:16 ] Completed train batch 0 metrics update 5.636 ms, 3.90 s total -Saving checkpoint at epoch 29 train batch 0 -[ 2023-09-21 06:41:17 ] Completed saving temp checkpoint 577.115 ms, 4.48 s total -[ 2023-09-21 06:41:17 ] Completed replacing temp checkpoint with checkpoint 26.013 ms, 4.51 s total -[ 2023-09-21 06:41:18 ] Completed train batch 1 to device 971.227 ms, 5.48 s total -[ 2023-09-21 06:41:18 ] Completed train batch 1 generator forward 40.971 ms, 5.52 s total -[ 2023-09-21 06:41:18 ] Completed train batch 1 generator loss: 1.530 62.983 ms, 5.58 s total -[ 2023-09-21 06:41:20 ] Completed train batch 1 generator backward 1,541.150 ms, 7.12 s total -[ 2023-09-21 06:41:20 ] Completed train batch 1 discriminator loss 0.122 56.195 ms, 7.18 s total -[ 2023-09-21 06:41:20 ] Completed train batch 1 discriminator backward 90.724 ms, 7.27 s total -Epoch [29] Step [1/2], gen_loss: 0.525, disc_loss: 0.041 -[ 2023-09-21 06:41:20 ] Completed train batch 1 metrics update 5.623 ms, 7.28 s total -Saving checkpoint at epoch 29 train batch 1 -[ 2023-09-21 06:41:20 ] Completed saving temp checkpoint 567.993 ms, 7.84 s total -[ 2023-09-21 06:41:20 ] Completed replacing temp checkpoint with checkpoint 25.062 ms, 7.87 s total -Epoch [29] :: gen_loss: 0.529, disc_loss: 0.042 -[ 2023-09-21 06:41:20 ] Completed training generator for epoch 29 68.624 ms, 7.94 s total -[ 2023-09-21 06:41:20 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 29 from eval step 0 - -[ 2023-09-21 06:41:21 ] Completed eval batch 0 to device 990.118 ms, 0.99 s total -[ 2023-09-21 06:41:21 ] Completed eval batch 0 forward 25.301 ms, 1.02 s total -[ 2023-09-21 06:41:21 ] Completed eval batch 0 recons_loss 0.301 ms, 1.02 s total -[ 2023-09-21 06:41:21 ] Completed eval batch 0 metrics update 0.601 ms, 1.02 s total -Saving checkpoint at epoch 29 val batch 0 -[ 2023-09-21 06:41:22 ] Completed saving temp checkpoint 393.356 ms, 1.41 s total -[ 2023-09-21 06:41:22 ] Completed replacing temp checkpoint with checkpoint 28.632 ms, 1.44 s total -[ 2023-09-21 06:41:22 ] Completed eval batch 1 to device 360.564 ms, 1.80 s total -[ 2023-09-21 06:41:22 ] Completed eval batch 1 forward 21.284 ms, 1.82 s total -[ 2023-09-21 06:41:22 ] Completed eval batch 1 recons_loss 0.331 ms, 1.82 s total -[ 2023-09-21 06:41:23 ] Completed eval batch 1 metrics update 314.671 ms, 2.14 s total -Saving checkpoint at epoch 29 val batch 1 -Epoch 29 val loss: 0.1018 -[ 2023-09-21 06:41:23 ] Completed saving temp checkpoint 462.972 ms, 2.60 s total -[ 2023-09-21 06:41:23 ] Completed replacing temp checkpoint with checkpoint 22.868 ms, 2.62 s total -[ 2023-09-21 06:41:23 ] Completed evaluating generator for epoch 29 60.910 ms, 2.68 s total - - -EPOCH :: 30 - - -[ 2023-09-21 06:41:23 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 30 from training step 0 - -[ 2023-09-21 06:41:26 ] Completed train batch 0 to device 2,772.230 ms, 2.77 s total -[ 2023-09-21 06:41:26 ] Completed train batch 0 generator forward 25.450 ms, 2.80 s total -[ 2023-09-21 06:41:26 ] Completed train batch 0 generator loss: 1.653 64.730 ms, 2.86 s total -[ 2023-09-21 06:41:27 ] Completed train batch 0 generator backward 1,041.165 ms, 3.90 s total -[ 2023-09-21 06:41:27 ] Completed train batch 0 discriminator loss 0.124 56.405 ms, 3.96 s total -[ 2023-09-21 06:41:27 ] Completed train batch 0 discriminator backward 107.133 ms, 4.07 s total -Epoch [30] Step [0/2], gen_loss: 0.531, disc_loss: 0.041 -[ 2023-09-21 06:41:27 ] Completed train batch 0 metrics update 8.183 ms, 4.08 s total -Saving checkpoint at epoch 30 train batch 0 -[ 2023-09-21 06:41:28 ] Completed saving temp checkpoint 429.459 ms, 4.50 s total -[ 2023-09-21 06:41:28 ] Completed replacing temp checkpoint with checkpoint 32.292 ms, 4.54 s total -[ 2023-09-21 06:41:28 ] Completed train batch 1 to device 795.923 ms, 5.33 s total -[ 2023-09-21 06:41:28 ] Completed train batch 1 generator forward 40.771 ms, 5.37 s total -[ 2023-09-21 06:41:29 ] Completed train batch 1 generator loss: 1.475 61.856 ms, 5.44 s total -[ 2023-09-21 06:41:31 ] Completed train batch 1 generator backward 2,009.270 ms, 7.44 s total -[ 2023-09-21 06:41:31 ] Completed train batch 1 discriminator loss 0.129 55.948 ms, 7.50 s total -[ 2023-09-21 06:41:31 ] Completed train batch 1 discriminator backward 78.903 ms, 7.58 s total -Epoch [30] Step [1/2], gen_loss: 0.523, disc_loss: 0.041 -[ 2023-09-21 06:41:31 ] Completed train batch 1 metrics update 0.842 ms, 7.58 s total -Saving checkpoint at epoch 30 train batch 1 -[ 2023-09-21 06:41:31 ] Completed saving temp checkpoint 438.522 ms, 8.02 s total -[ 2023-09-21 06:41:31 ] Completed replacing temp checkpoint with checkpoint 29.663 ms, 8.05 s total -Epoch [30] :: gen_loss: 0.527, disc_loss: 0.041 -[ 2023-09-21 06:41:31 ] Completed training generator for epoch 30 56.105 ms, 8.10 s total -[ 2023-09-21 06:41:31 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 30 from eval step 0 - -[ 2023-09-21 06:41:32 ] Completed eval batch 0 to device 1,032.118 ms, 1.03 s total -[ 2023-09-21 06:41:32 ] Completed eval batch 0 forward 23.882 ms, 1.06 s total -[ 2023-09-21 06:41:32 ] Completed eval batch 0 recons_loss 0.236 ms, 1.06 s total -[ 2023-09-21 06:41:32 ] Completed eval batch 0 metrics update 22.668 ms, 1.08 s total -Saving checkpoint at epoch 30 val batch 0 -[ 2023-09-21 06:41:33 ] Completed saving temp checkpoint 405.074 ms, 1.48 s total -[ 2023-09-21 06:41:33 ] Completed replacing temp checkpoint with checkpoint 24.651 ms, 1.51 s total -[ 2023-09-21 06:41:33 ] Completed eval batch 1 to device 337.995 ms, 1.85 s total -[ 2023-09-21 06:41:33 ] Completed eval batch 1 forward 38.956 ms, 1.89 s total -[ 2023-09-21 06:41:33 ] Completed eval batch 1 recons_loss 0.348 ms, 1.89 s total -[ 2023-09-21 06:41:33 ] Completed eval batch 1 metrics update 362.205 ms, 2.25 s total -Saving checkpoint at epoch 30 val batch 1 -Epoch 30 val loss: 0.0929 -[ 2023-09-21 06:41:34 ] Completed saving temp checkpoint 440.899 ms, 2.69 s total -[ 2023-09-21 06:41:34 ] Completed replacing temp checkpoint with checkpoint 23.979 ms, 2.71 s total -[ 2023-09-21 06:41:34 ] Completed evaluating generator for epoch 30 73.677 ms, 2.79 s total - - -EPOCH :: 31 - - -[ 2023-09-21 06:41:34 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 31 from training step 0 - -[ 2023-09-21 06:41:37 ] Completed train batch 0 to device 2,926.802 ms, 2.93 s total -[ 2023-09-21 06:41:37 ] Completed train batch 0 generator forward 27.311 ms, 2.95 s total -[ 2023-09-21 06:41:37 ] Completed train batch 0 generator loss: 1.543 65.840 ms, 3.02 s total -[ 2023-09-21 06:41:38 ] Completed train batch 0 generator backward 848.863 ms, 3.87 s total -[ 2023-09-21 06:41:38 ] Completed train batch 0 discriminator loss 0.121 56.520 ms, 3.93 s total -[ 2023-09-21 06:41:38 ] Completed train batch 0 discriminator backward 112.160 ms, 4.04 s total -Epoch [31] Step [0/2], gen_loss: 0.523, disc_loss: 0.043 -[ 2023-09-21 06:41:38 ] Completed train batch 0 metrics update 7.135 ms, 4.04 s total -Saving checkpoint at epoch 31 train batch 0 -[ 2023-09-21 06:41:38 ] Completed saving temp checkpoint 428.031 ms, 4.47 s total -[ 2023-09-21 06:41:38 ] Completed replacing temp checkpoint with checkpoint 24.370 ms, 4.50 s total -[ 2023-09-21 06:41:40 ] Completed train batch 1 to device 1,150.628 ms, 5.65 s total -[ 2023-09-21 06:41:40 ] Completed train batch 1 generator forward 27.753 ms, 5.68 s total -[ 2023-09-21 06:41:40 ] Completed train batch 1 generator loss: 1.469 62.045 ms, 5.74 s total -[ 2023-09-21 06:41:41 ] Completed train batch 1 generator backward 1,377.699 ms, 7.12 s total -[ 2023-09-21 06:41:41 ] Completed train batch 1 discriminator loss 0.171 57.669 ms, 7.17 s total -[ 2023-09-21 06:41:41 ] Completed train batch 1 discriminator backward 65.387 ms, 7.24 s total -Epoch [31] Step [1/2], gen_loss: 0.506, disc_loss: 0.052 -[ 2023-09-21 06:41:41 ] Completed train batch 1 metrics update 2.217 ms, 7.24 s total -Saving checkpoint at epoch 31 train batch 1 -[ 2023-09-21 06:41:42 ] Completed saving temp checkpoint 440.861 ms, 7.68 s total -[ 2023-09-21 06:41:42 ] Completed replacing temp checkpoint with checkpoint 29.608 ms, 7.71 s total -Epoch [31] :: gen_loss: 0.515, disc_loss: 0.047 -[ 2023-09-21 06:41:42 ] Completed training generator for epoch 31 61.086 ms, 7.77 s total -[ 2023-09-21 06:41:42 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 31 from eval step 0 - -[ 2023-09-21 06:41:43 ] Completed eval batch 0 to device 1,112.975 ms, 1.11 s total -[ 2023-09-21 06:41:43 ] Completed eval batch 0 forward 43.529 ms, 1.16 s total -[ 2023-09-21 06:41:43 ] Completed eval batch 0 recons_loss 0.327 ms, 1.16 s total -[ 2023-09-21 06:41:43 ] Completed eval batch 0 metrics update 0.628 ms, 1.16 s total -Saving checkpoint at epoch 31 val batch 0 -[ 2023-09-21 06:41:43 ] Completed saving temp checkpoint 400.309 ms, 1.56 s total -[ 2023-09-21 06:41:43 ] Completed replacing temp checkpoint with checkpoint 33.866 ms, 1.59 s total -[ 2023-09-21 06:41:44 ] Completed eval batch 1 to device 390.629 ms, 1.98 s total -[ 2023-09-21 06:41:44 ] Completed eval batch 1 forward 39.579 ms, 2.02 s total -[ 2023-09-21 06:41:44 ] Completed eval batch 1 recons_loss 0.551 ms, 2.02 s total -[ 2023-09-21 06:41:44 ] Completed eval batch 1 metrics update 147.382 ms, 2.17 s total -Saving checkpoint at epoch 31 val batch 1 -Epoch 31 val loss: 0.0812 -[ 2023-09-21 06:41:44 ] Completed saving temp checkpoint 424.985 ms, 2.59 s total -[ 2023-09-21 06:41:44 ] Completed replacing temp checkpoint with checkpoint 28.619 ms, 2.62 s total -[ 2023-09-21 06:41:44 ] Completed evaluating generator for epoch 31 60.088 ms, 2.68 s total - - -EPOCH :: 32 - - -[ 2023-09-21 06:41:44 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 32 from training step 0 - -[ 2023-09-21 06:41:48 ] Completed train batch 0 to device 3,047.914 ms, 3.05 s total -[ 2023-09-21 06:41:48 ] Completed train batch 0 generator forward 26.374 ms, 3.07 s total -[ 2023-09-21 06:41:48 ] Completed train batch 0 generator loss: 1.476 65.333 ms, 3.14 s total -[ 2023-09-21 06:41:48 ] Completed train batch 0 generator backward 697.477 ms, 3.84 s total -[ 2023-09-21 06:41:48 ] Completed train batch 0 discriminator loss 0.183 56.656 ms, 3.89 s total -[ 2023-09-21 06:41:48 ] Completed train batch 0 discriminator backward 88.961 ms, 3.98 s total -Epoch [32] Step [0/2], gen_loss: 0.500, disc_loss: 0.067 -[ 2023-09-21 06:41:48 ] Completed train batch 0 metrics update 5.250 ms, 3.99 s total -Saving checkpoint at epoch 32 train batch 0 -[ 2023-09-21 06:41:49 ] Completed saving temp checkpoint 419.650 ms, 4.41 s total -[ 2023-09-21 06:41:49 ] Completed replacing temp checkpoint with checkpoint 26.724 ms, 4.43 s total -[ 2023-09-21 06:41:50 ] Completed train batch 1 to device 1,309.196 ms, 5.74 s total -[ 2023-09-21 06:41:50 ] Completed train batch 1 generator forward 22.098 ms, 5.77 s total -[ 2023-09-21 06:41:50 ] Completed train batch 1 generator loss: 1.451 63.639 ms, 5.83 s total -[ 2023-09-21 06:41:51 ] Completed train batch 1 generator backward 1,161.742 ms, 6.99 s total -[ 2023-09-21 06:41:52 ] Completed train batch 1 discriminator loss 0.206 56.289 ms, 7.05 s total -[ 2023-09-21 06:41:52 ] Completed train batch 1 discriminator backward 71.611 ms, 7.12 s total -Epoch [32] Step [1/2], gen_loss: 0.490, disc_loss: 0.072 -[ 2023-09-21 06:41:52 ] Completed train batch 1 metrics update 5.578 ms, 7.12 s total -Saving checkpoint at epoch 32 train batch 1 -[ 2023-09-21 06:41:52 ] Completed saving temp checkpoint 474.993 ms, 7.60 s total -[ 2023-09-21 06:41:52 ] Completed replacing temp checkpoint with checkpoint 25.553 ms, 7.63 s total -Epoch [32] :: gen_loss: 0.495, disc_loss: 0.069 -[ 2023-09-21 06:41:52 ] Completed training generator for epoch 32 68.731 ms, 7.69 s total -[ 2023-09-21 06:41:52 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 32 from eval step 0 - -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 06:53:07 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 06:53:07 ] Completed importing Timer 0.023 ms, 0.00 s total -[ 2023-09-21 06:53:12 ] Completed importing everything else 4,861.746 ms, 4.86 s total -| distributed init (rank 3): env:// -| distributed init (rank 2): env:// -| distributed init (rank 5): env:// -| distributed init (rank 0): env:// -| distributed init (rank 4): env:// -| distributed init (rank 1): env:// -[ 2023-09-21 06:53:20 ] Completed preliminaries 7,314.090 ms, 12.18 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 06:53:20 ] Completed build datasets 11.715 ms, 12.19 s total -[ 2023-09-21 06:53:20 ] Completed build samplers 0.047 ms, 12.19 s total -[ 2023-09-21 06:53:20 ] Completed build dataloaders 0.129 ms, 12.19 s total -[ 2023-09-21 06:53:20 ] Completed generator to device 95.272 ms, 12.28 s total -[ 2023-09-21 06:53:20 ] Completed discriminator to device 7.008 ms, 12.29 s total -[ 2023-09-21 06:53:20 ] Completed loss functions 373.022 ms, 12.66 s total -[ 2023-09-21 06:53:20 ] Completed models prepped for distribution 102.576 ms, 12.77 s total -[ 2023-09-21 06:53:20 ] Completed optimizers 0.671 ms, 12.77 s total -[ 2023-09-21 06:53:20 ] Completed grad scalers 0.029 ms, 12.77 s total -[ 2023-09-21 06:53:21 ] Completed checkpoint retrieval 493.675 ms, 13.26 s total - - -EPOCH :: 32 - - -[ 2023-09-21 06:53:21 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 32 from training step 2 - -Epoch [32] :: gen_loss: 0.495, disc_loss: 0.069 -[ 2023-09-21 06:53:21 ] Completed training generator for epoch 32 114.119 ms, 0.11 s total -[ 2023-09-21 06:53:21 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 32 from eval step 0 - -[ 2023-09-21 06:53:22 ] Completed eval batch 0 to device 1,543.577 ms, 1.54 s total -[ 2023-09-21 06:53:24 ] Completed eval batch 0 forward 1,545.339 ms, 3.09 s total -[ 2023-09-21 06:53:24 ] Completed eval batch 0 recons_loss 0.280 ms, 3.09 s total -[ 2023-09-21 06:53:24 ] Completed eval batch 0 metrics update 597.080 ms, 3.69 s total -Saving checkpoint at epoch 32 val batch 0 -[ 2023-09-21 06:53:25 ] Completed saving temp checkpoint 341.601 ms, 4.03 s total -[ 2023-09-21 06:53:25 ] Completed replacing temp checkpoint with checkpoint 124.077 ms, 4.15 s total -[ 2023-09-21 06:53:25 ] Completed eval batch 1 to device 16.725 ms, 4.17 s total -[ 2023-09-21 06:53:25 ] Completed eval batch 1 forward 21.114 ms, 4.19 s total -[ 2023-09-21 06:53:25 ] Completed eval batch 1 recons_loss 0.334 ms, 4.19 s total -[ 2023-09-21 06:53:25 ] Completed eval batch 1 metrics update 0.295 ms, 4.19 s total -Saving checkpoint at epoch 32 val batch 1 -Epoch 32 val loss: 0.0765 -[ 2023-09-21 06:53:25 ] Completed saving temp checkpoint 388.864 ms, 4.58 s total -[ 2023-09-21 06:53:25 ] Completed replacing temp checkpoint with checkpoint 21.574 ms, 4.60 s total -[ 2023-09-21 06:53:25 ] Completed evaluating generator for epoch 32 36.317 ms, 4.64 s total - - -EPOCH :: 33 - - -[ 2023-09-21 06:53:25 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 33 from training step 0 - -[ 2023-09-21 06:53:28 ] Completed train batch 0 to device 3,030.864 ms, 3.03 s total -[ 2023-09-21 06:53:28 ] Completed train batch 0 generator forward 36.295 ms, 3.07 s total -[ 2023-09-21 06:53:28 ] Completed train batch 0 generator loss: 1.482 76.236 ms, 3.14 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 06:53:29 ] Completed train batch 0 generator backward 846.601 ms, 3.99 s total -[ 2023-09-21 06:53:29 ] Completed train batch 0 discriminator loss 0.214 57.214 ms, 4.05 s total -[ 2023-09-21 06:53:30 ] Completed train batch 0 discriminator backward 119.646 ms, 4.17 s total -Epoch [33] Step [0/2], gen_loss: 0.494, disc_loss: 0.072 -[ 2023-09-21 06:53:30 ] Completed train batch 0 metrics update 6.875 ms, 4.17 s total -Saving checkpoint at epoch 33 train batch 0 -[ 2023-09-21 06:53:30 ] Completed saving temp checkpoint 385.520 ms, 4.56 s total -[ 2023-09-21 06:53:30 ] Completed replacing temp checkpoint with checkpoint 22.136 ms, 4.58 s total -[ 2023-09-21 06:53:32 ] Completed train batch 1 to device 1,722.468 ms, 6.30 s total -[ 2023-09-21 06:53:32 ] Completed train batch 1 generator forward 25.634 ms, 6.33 s total -[ 2023-09-21 06:53:32 ] Completed train batch 1 generator loss: 1.372 63.081 ms, 6.39 s total -[ 2023-09-21 06:53:33 ] Completed train batch 1 generator backward 840.321 ms, 7.23 s total -[ 2023-09-21 06:53:33 ] Completed train batch 1 discriminator loss 0.203 55.957 ms, 7.29 s total -[ 2023-09-21 06:53:33 ] Completed train batch 1 discriminator backward 93.029 ms, 7.38 s total -Epoch [33] Step [1/2], gen_loss: 0.491, disc_loss: 0.069 -[ 2023-09-21 06:53:33 ] Completed train batch 1 metrics update 4.242 ms, 7.39 s total -Saving checkpoint at epoch 33 train batch 1 -[ 2023-09-21 06:53:33 ] Completed saving temp checkpoint 403.596 ms, 7.79 s total -[ 2023-09-21 06:53:33 ] Completed replacing temp checkpoint with checkpoint 19.731 ms, 7.81 s total -Epoch [33] :: gen_loss: 0.493, disc_loss: 0.071 -[ 2023-09-21 06:53:33 ] Completed training generator for epoch 33 48.288 ms, 7.86 s total -[ 2023-09-21 06:53:33 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 33 from eval step 0 - -[ 2023-09-21 06:53:34 ] Completed eval batch 0 to device 1,156.707 ms, 1.16 s total -[ 2023-09-21 06:53:34 ] Completed eval batch 0 forward 24.287 ms, 1.18 s total -[ 2023-09-21 06:53:34 ] Completed eval batch 0 recons_loss 0.273 ms, 1.18 s total -[ 2023-09-21 06:53:34 ] Completed eval batch 0 metrics update 0.562 ms, 1.18 s total -Saving checkpoint at epoch 33 val batch 0 -[ 2023-09-21 06:53:35 ] Completed saving temp checkpoint 491.662 ms, 1.67 s total -[ 2023-09-21 06:53:35 ] Completed replacing temp checkpoint with checkpoint 38.420 ms, 1.71 s total -[ 2023-09-21 06:53:35 ] Completed eval batch 1 to device 316.236 ms, 2.03 s total -[ 2023-09-21 06:53:35 ] Completed eval batch 1 forward 22.106 ms, 2.05 s total -[ 2023-09-21 06:53:35 ] Completed eval batch 1 recons_loss 0.408 ms, 2.05 s total -[ 2023-09-21 06:53:36 ] Completed eval batch 1 metrics update 256.993 ms, 2.31 s total -Saving checkpoint at epoch 33 val batch 1 -Epoch 33 val loss: 0.0783 -[ 2023-09-21 06:53:36 ] Completed saving temp checkpoint 687.921 ms, 3.00 s total -[ 2023-09-21 06:53:36 ] Completed replacing temp checkpoint with checkpoint 35.277 ms, 3.03 s total -[ 2023-09-21 06:53:36 ] Completed evaluating generator for epoch 33 51.021 ms, 3.08 s total - - -EPOCH :: 34 - - -[ 2023-09-21 06:53:36 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 34 from training step 0 - -[ 2023-09-21 06:53:39 ] Completed train batch 0 to device 2,881.583 ms, 2.88 s total -[ 2023-09-21 06:53:39 ] Completed train batch 0 generator forward 24.707 ms, 2.91 s total -[ 2023-09-21 06:53:39 ] Completed train batch 0 generator loss: 1.541 65.098 ms, 2.97 s total -[ 2023-09-21 06:53:40 ] Completed train batch 0 generator backward 727.434 ms, 3.70 s total -[ 2023-09-21 06:53:40 ] Completed train batch 0 discriminator loss 0.182 57.259 ms, 3.76 s total -[ 2023-09-21 06:53:40 ] Completed train batch 0 discriminator backward 102.944 ms, 3.86 s total -Epoch [34] Step [0/2], gen_loss: 0.505, disc_loss: 0.065 -[ 2023-09-21 06:53:40 ] Completed train batch 0 metrics update 9.958 ms, 3.87 s total -Saving checkpoint at epoch 34 train batch 0 -[ 2023-09-21 06:53:41 ] Completed saving temp checkpoint 419.953 ms, 4.29 s total -[ 2023-09-21 06:53:41 ] Completed replacing temp checkpoint with checkpoint 20.192 ms, 4.31 s total -[ 2023-09-21 06:53:42 ] Completed train batch 1 to device 1,170.004 ms, 5.48 s total -[ 2023-09-21 06:53:42 ] Completed train batch 1 generator forward 40.896 ms, 5.52 s total -[ 2023-09-21 06:53:42 ] Completed train batch 1 generator loss: 1.550 61.235 ms, 5.58 s total -[ 2023-09-21 06:53:44 ] Completed train batch 1 generator backward 1,842.580 ms, 7.42 s total -[ 2023-09-21 06:53:44 ] Completed train batch 1 discriminator loss 0.186 55.682 ms, 7.48 s total -[ 2023-09-21 06:53:44 ] Completed train batch 1 discriminator backward 82.165 ms, 7.56 s total -Epoch [34] Step [1/2], gen_loss: 0.501, disc_loss: 0.062 -[ 2023-09-21 06:53:44 ] Completed train batch 1 metrics update 1.495 ms, 7.56 s total -Saving checkpoint at epoch 34 train batch 1 -[ 2023-09-21 06:53:44 ] Completed saving temp checkpoint 414.400 ms, 7.98 s total -[ 2023-09-21 06:53:44 ] Completed replacing temp checkpoint with checkpoint 19.745 ms, 8.00 s total -Epoch [34] :: gen_loss: 0.503, disc_loss: 0.063 -[ 2023-09-21 06:53:44 ] Completed training generator for epoch 34 54.274 ms, 8.05 s total -[ 2023-09-21 06:53:44 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 34 from eval step 0 - -[ 2023-09-21 06:53:45 ] Completed eval batch 0 to device 1,100.762 ms, 1.10 s total -[ 2023-09-21 06:53:45 ] Completed eval batch 0 forward 43.813 ms, 1.14 s total -[ 2023-09-21 06:53:45 ] Completed eval batch 0 recons_loss 0.360 ms, 1.14 s total -[ 2023-09-21 06:53:45 ] Completed eval batch 0 metrics update 0.655 ms, 1.15 s total -Saving checkpoint at epoch 34 val batch 0 -[ 2023-09-21 06:53:46 ] Completed saving temp checkpoint 418.936 ms, 1.56 s total -[ 2023-09-21 06:53:46 ] Completed replacing temp checkpoint with checkpoint 29.895 ms, 1.59 s total -[ 2023-09-21 06:53:46 ] Completed eval batch 1 to device 339.512 ms, 1.93 s total -[ 2023-09-21 06:53:46 ] Completed eval batch 1 forward 21.337 ms, 1.96 s total -[ 2023-09-21 06:53:46 ] Completed eval batch 1 recons_loss 0.335 ms, 1.96 s total -[ 2023-09-21 06:53:47 ] Completed eval batch 1 metrics update 332.383 ms, 2.29 s total -Saving checkpoint at epoch 34 val batch 1 -Epoch 34 val loss: 0.0768 -[ 2023-09-21 06:53:47 ] Completed saving temp checkpoint 415.019 ms, 2.70 s total -[ 2023-09-21 06:53:47 ] Completed replacing temp checkpoint with checkpoint 24.229 ms, 2.73 s total -[ 2023-09-21 06:53:47 ] Completed evaluating generator for epoch 34 52.691 ms, 2.78 s total - - -EPOCH :: 35 - - -[ 2023-09-21 06:53:47 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 35 from training step 0 - -[ 2023-09-21 06:53:50 ] Completed train batch 0 to device 3,047.354 ms, 3.05 s total -[ 2023-09-21 06:53:50 ] Completed train batch 0 generator forward 30.071 ms, 3.08 s total -[ 2023-09-21 06:53:50 ] Completed train batch 0 generator loss: 1.495 88.676 ms, 3.17 s total -[ 2023-09-21 06:53:51 ] Completed train batch 0 generator backward 678.617 ms, 3.84 s total -[ 2023-09-21 06:53:51 ] Completed train batch 0 discriminator loss 0.176 59.229 ms, 3.90 s total -[ 2023-09-21 06:53:51 ] Completed train batch 0 discriminator backward 99.949 ms, 4.00 s total -Epoch [35] Step [0/2], gen_loss: 0.506, disc_loss: 0.058 -[ 2023-09-21 06:53:51 ] Completed train batch 0 metrics update 4.300 ms, 4.01 s total -Saving checkpoint at epoch 35 train batch 0 -[ 2023-09-21 06:53:52 ] Completed saving temp checkpoint 431.631 ms, 4.44 s total -[ 2023-09-21 06:53:52 ] Completed replacing temp checkpoint with checkpoint 30.954 ms, 4.47 s total -[ 2023-09-21 06:53:53 ] Completed train batch 1 to device 1,260.500 ms, 5.73 s total -[ 2023-09-21 06:53:53 ] Completed train batch 1 generator forward 41.151 ms, 5.77 s total -[ 2023-09-21 06:53:53 ] Completed train batch 1 generator loss: 1.567 63.773 ms, 5.84 s total -[ 2023-09-21 06:53:54 ] Completed train batch 1 generator backward 1,346.190 ms, 7.18 s total -[ 2023-09-21 06:53:54 ] Completed train batch 1 discriminator loss 0.163 56.183 ms, 7.24 s total -[ 2023-09-21 06:53:54 ] Completed train batch 1 discriminator backward 74.359 ms, 7.31 s total -Epoch [35] Step [1/2], gen_loss: 0.504, disc_loss: 0.055 -[ 2023-09-21 06:53:54 ] Completed train batch 1 metrics update 0.894 ms, 7.31 s total -Saving checkpoint at epoch 35 train batch 1 -[ 2023-09-21 06:53:55 ] Completed saving temp checkpoint 405.449 ms, 7.72 s total -[ 2023-09-21 06:53:55 ] Completed replacing temp checkpoint with checkpoint 22.270 ms, 7.74 s total -Epoch [35] :: gen_loss: 0.505, disc_loss: 0.056 -[ 2023-09-21 06:53:55 ] Completed training generator for epoch 35 50.578 ms, 7.79 s total -[ 2023-09-21 06:53:55 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 35 from eval step 0 - -[ 2023-09-21 06:53:56 ] Completed eval batch 0 to device 1,051.696 ms, 1.05 s total -[ 2023-09-21 06:53:56 ] Completed eval batch 0 forward 24.921 ms, 1.08 s total -[ 2023-09-21 06:53:56 ] Completed eval batch 0 recons_loss 0.290 ms, 1.08 s total -[ 2023-09-21 06:53:56 ] Completed eval batch 0 metrics update 30.204 ms, 1.11 s total -Saving checkpoint at epoch 35 val batch 0 -[ 2023-09-21 06:53:56 ] Completed saving temp checkpoint 388.340 ms, 1.50 s total -[ 2023-09-21 06:53:56 ] Completed replacing temp checkpoint with checkpoint 34.696 ms, 1.53 s total -[ 2023-09-21 06:53:57 ] Completed eval batch 1 to device 405.081 ms, 1.94 s total -[ 2023-09-21 06:53:57 ] Completed eval batch 1 forward 39.814 ms, 1.98 s total -[ 2023-09-21 06:53:57 ] Completed eval batch 1 recons_loss 0.570 ms, 1.98 s total -[ 2023-09-21 06:53:57 ] Completed eval batch 1 metrics update 344.017 ms, 2.32 s total -Saving checkpoint at epoch 35 val batch 1 -Epoch 35 val loss: 0.0766 -[ 2023-09-21 06:53:58 ] Completed saving temp checkpoint 421.510 ms, 2.74 s total -[ 2023-09-21 06:53:58 ] Completed replacing temp checkpoint with checkpoint 15.814 ms, 2.76 s total -[ 2023-09-21 06:53:58 ] Completed evaluating generator for epoch 35 45.224 ms, 2.80 s total - - -EPOCH :: 36 - - -[ 2023-09-21 06:53:58 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 36 from training step 0 - -[ 2023-09-21 06:54:01 ] Completed train batch 0 to device 2,976.491 ms, 2.98 s total -[ 2023-09-21 06:54:01 ] Completed train batch 0 generator forward 25.112 ms, 3.00 s total -[ 2023-09-21 06:54:01 ] Completed train batch 0 generator loss: 1.483 64.761 ms, 3.07 s total -[ 2023-09-21 06:54:02 ] Completed train batch 0 generator backward 750.223 ms, 3.82 s total -[ 2023-09-21 06:54:02 ] Completed train batch 0 discriminator loss 0.163 56.859 ms, 3.87 s total -[ 2023-09-21 06:54:02 ] Completed train batch 0 discriminator backward 86.298 ms, 3.96 s total -Epoch [36] Step [0/2], gen_loss: 0.506, disc_loss: 0.053 -[ 2023-09-21 06:54:02 ] Completed train batch 0 metrics update 5.351 ms, 3.97 s total -Saving checkpoint at epoch 36 train batch 0 -[ 2023-09-21 06:54:02 ] Completed saving temp checkpoint 400.978 ms, 4.37 s total -[ 2023-09-21 06:54:02 ] Completed replacing temp checkpoint with checkpoint 24.168 ms, 4.39 s total -[ 2023-09-21 06:54:03 ] Completed train batch 1 to device 1,161.093 ms, 5.55 s total -[ 2023-09-21 06:54:03 ] Completed train batch 1 generator forward 40.788 ms, 5.59 s total -[ 2023-09-21 06:54:03 ] Completed train batch 1 generator loss: 1.513 61.635 ms, 5.65 s total -[ 2023-09-21 06:54:05 ] Completed train batch 1 generator backward 1,681.650 ms, 7.34 s total -[ 2023-09-21 06:54:05 ] Completed train batch 1 discriminator loss 0.154 55.760 ms, 7.39 s total -[ 2023-09-21 06:54:05 ] Completed train batch 1 discriminator backward 63.323 ms, 7.45 s total -Epoch [36] Step [1/2], gen_loss: 0.503, disc_loss: 0.051 -[ 2023-09-21 06:54:05 ] Completed train batch 1 metrics update 2.046 ms, 7.46 s total -Saving checkpoint at epoch 36 train batch 1 -[ 2023-09-21 06:54:06 ] Completed saving temp checkpoint 457.855 ms, 7.91 s total -[ 2023-09-21 06:54:06 ] Completed replacing temp checkpoint with checkpoint 25.195 ms, 7.94 s total -Epoch [36] :: gen_loss: 0.504, disc_loss: 0.052 -[ 2023-09-21 06:54:06 ] Completed training generator for epoch 36 61.489 ms, 8.00 s total -[ 2023-09-21 06:54:06 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 36 from eval step 0 - -[ 2023-09-21 06:54:07 ] Completed eval batch 0 to device 1,086.941 ms, 1.09 s total -[ 2023-09-21 06:54:07 ] Completed eval batch 0 forward 29.769 ms, 1.12 s total -[ 2023-09-21 06:54:07 ] Completed eval batch 0 recons_loss 0.316 ms, 1.12 s total -[ 2023-09-21 06:54:07 ] Completed eval batch 0 metrics update 0.595 ms, 1.12 s total -Saving checkpoint at epoch 36 val batch 0 -[ 2023-09-21 06:54:07 ] Completed saving temp checkpoint 423.341 ms, 1.54 s total -[ 2023-09-21 06:54:07 ] Completed replacing temp checkpoint with checkpoint 18.295 ms, 1.56 s total -[ 2023-09-21 06:54:08 ] Completed eval batch 1 to device 397.011 ms, 1.96 s total -[ 2023-09-21 06:54:08 ] Completed eval batch 1 forward 21.588 ms, 1.98 s total -[ 2023-09-21 06:54:08 ] Completed eval batch 1 recons_loss 0.345 ms, 1.98 s total -[ 2023-09-21 06:54:08 ] Completed eval batch 1 metrics update 206.117 ms, 2.18 s total -Saving checkpoint at epoch 36 val batch 1 -Epoch 36 val loss: 0.0719 -[ 2023-09-21 06:54:08 ] Completed saving temp checkpoint 426.792 ms, 2.61 s total -[ 2023-09-21 06:54:08 ] Completed replacing temp checkpoint with checkpoint 29.135 ms, 2.64 s total -[ 2023-09-21 06:54:08 ] Completed evaluating generator for epoch 36 51.482 ms, 2.69 s total - - -EPOCH :: 37 - - -[ 2023-09-21 06:54:08 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 37 from training step 0 - -[ 2023-09-21 06:54:11 ] Completed train batch 0 to device 3,043.030 ms, 3.04 s total -[ 2023-09-21 06:54:11 ] Completed train batch 0 generator forward 32.783 ms, 3.08 s total -[ 2023-09-21 06:54:12 ] Completed train batch 0 generator loss: 1.587 69.032 ms, 3.14 s total -[ 2023-09-21 06:54:12 ] Completed train batch 0 generator backward 635.842 ms, 3.78 s total -[ 2023-09-21 06:54:12 ] Completed train batch 0 discriminator loss 0.142 78.736 ms, 3.86 s total -[ 2023-09-21 06:54:12 ] Completed train batch 0 discriminator backward 76.232 ms, 3.94 s total -Epoch [37] Step [0/2], gen_loss: 0.511, disc_loss: 0.052 -[ 2023-09-21 06:54:12 ] Completed train batch 0 metrics update 24.142 ms, 3.96 s total -Saving checkpoint at epoch 37 train batch 0 -[ 2023-09-21 06:54:13 ] Completed saving temp checkpoint 440.071 ms, 4.40 s total -[ 2023-09-21 06:54:13 ] Completed replacing temp checkpoint with checkpoint 26.195 ms, 4.43 s total -[ 2023-09-21 06:54:14 ] Completed train batch 1 to device 1,338.557 ms, 5.76 s total -[ 2023-09-21 06:54:14 ] Completed train batch 1 generator forward 27.520 ms, 5.79 s total -[ 2023-09-21 06:54:14 ] Completed train batch 1 generator loss: 1.504 63.273 ms, 5.86 s total -[ 2023-09-21 06:54:15 ] Completed train batch 1 generator backward 1,076.797 ms, 6.93 s total -[ 2023-09-21 06:54:15 ] Completed train batch 1 discriminator loss 0.159 55.952 ms, 6.99 s total -[ 2023-09-21 06:54:15 ] Completed train batch 1 discriminator backward 74.880 ms, 7.06 s total -Epoch [37] Step [1/2], gen_loss: 0.504, disc_loss: 0.052 -[ 2023-09-21 06:54:15 ] Completed train batch 1 metrics update 1.965 ms, 7.07 s total -Saving checkpoint at epoch 37 train batch 1 -[ 2023-09-21 06:54:16 ] Completed saving temp checkpoint 473.460 ms, 7.54 s total -[ 2023-09-21 06:54:16 ] Completed replacing temp checkpoint with checkpoint 29.862 ms, 7.57 s total -Epoch [37] :: gen_loss: 0.508, disc_loss: 0.052 -[ 2023-09-21 06:54:16 ] Completed training generator for epoch 37 50.631 ms, 7.62 s total -[ 2023-09-21 06:54:16 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 37 from eval step 0 - -[ 2023-09-21 06:54:17 ] Completed eval batch 0 to device 1,083.109 ms, 1.08 s total -[ 2023-09-21 06:54:17 ] Completed eval batch 0 forward 35.010 ms, 1.12 s total -[ 2023-09-21 06:54:17 ] Completed eval batch 0 recons_loss 0.304 ms, 1.12 s total -[ 2023-09-21 06:54:17 ] Completed eval batch 0 metrics update 0.564 ms, 1.12 s total -Saving checkpoint at epoch 37 val batch 0 -[ 2023-09-21 06:54:18 ] Completed saving temp checkpoint 626.016 ms, 1.75 s total -[ 2023-09-21 06:54:18 ] Completed replacing temp checkpoint with checkpoint 33.807 ms, 1.78 s total -[ 2023-09-21 06:54:18 ] Completed eval batch 1 to device 192.243 ms, 1.97 s total -[ 2023-09-21 06:54:18 ] Completed eval batch 1 forward 21.367 ms, 1.99 s total -[ 2023-09-21 06:54:18 ] Completed eval batch 1 recons_loss 0.344 ms, 1.99 s total -[ 2023-09-21 06:54:18 ] Completed eval batch 1 metrics update 286.619 ms, 2.28 s total -Saving checkpoint at epoch 37 val batch 1 -Epoch 37 val loss: 0.0655 -[ 2023-09-21 06:54:19 ] Completed saving temp checkpoint 722.171 ms, 3.00 s total -[ 2023-09-21 06:54:19 ] Completed replacing temp checkpoint with checkpoint 51.861 ms, 3.05 s total -[ 2023-09-21 06:54:19 ] Completed evaluating generator for epoch 37 58.514 ms, 3.11 s total - - -EPOCH :: 38 - - -[ 2023-09-21 06:54:19 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 38 from training step 0 - -[ 2023-09-21 06:54:22 ] Completed train batch 0 to device 3,225.227 ms, 3.23 s total -[ 2023-09-21 06:54:22 ] Completed train batch 0 generator forward 26.361 ms, 3.25 s total -[ 2023-09-21 06:54:22 ] Completed train batch 0 generator loss: 1.511 65.745 ms, 3.32 s total -[ 2023-09-21 06:54:23 ] Completed train batch 0 generator backward 250.845 ms, 3.57 s total -[ 2023-09-21 06:54:23 ] Completed train batch 0 discriminator loss 0.155 56.377 ms, 3.62 s total -[ 2023-09-21 06:54:23 ] Completed train batch 0 discriminator backward 100.811 ms, 3.73 s total -Epoch [38] Step [0/2], gen_loss: 0.502, disc_loss: 0.052 -[ 2023-09-21 06:54:23 ] Completed train batch 0 metrics update 6.473 ms, 3.73 s total -Saving checkpoint at epoch 38 train batch 0 -[ 2023-09-21 06:54:23 ] Completed saving temp checkpoint 424.719 ms, 4.16 s total -[ 2023-09-21 06:54:23 ] Completed replacing temp checkpoint with checkpoint 18.065 ms, 4.17 s total -[ 2023-09-21 06:54:25 ] Completed train batch 1 to device 1,994.726 ms, 6.17 s total -[ 2023-09-21 06:54:25 ] Completed train batch 1 generator forward 22.193 ms, 6.19 s total -[ 2023-09-21 06:54:25 ] Completed train batch 1 generator loss: 1.479 63.478 ms, 6.26 s total -[ 2023-09-21 06:54:26 ] Completed train batch 1 generator backward 624.587 ms, 6.88 s total -[ 2023-09-21 06:54:26 ] Completed train batch 1 discriminator loss 0.147 56.019 ms, 6.94 s total -[ 2023-09-21 06:54:26 ] Completed train batch 1 discriminator backward 72.333 ms, 7.01 s total -Epoch [38] Step [1/2], gen_loss: 0.491, disc_loss: 0.049 -[ 2023-09-21 06:54:26 ] Completed train batch 1 metrics update 1.233 ms, 7.01 s total -Saving checkpoint at epoch 38 train batch 1 -[ 2023-09-21 06:54:27 ] Completed saving temp checkpoint 437.039 ms, 7.45 s total -[ 2023-09-21 06:54:27 ] Completed replacing temp checkpoint with checkpoint 29.549 ms, 7.48 s total -Epoch [38] :: gen_loss: 0.497, disc_loss: 0.050 -[ 2023-09-21 06:54:27 ] Completed training generator for epoch 38 48.774 ms, 7.52 s total -[ 2023-09-21 06:54:27 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 38 from eval step 0 - -[ 2023-09-21 06:54:28 ] Completed eval batch 0 to device 1,148.208 ms, 1.15 s total -[ 2023-09-21 06:54:28 ] Completed eval batch 0 forward 40.780 ms, 1.19 s total -[ 2023-09-21 06:54:28 ] Completed eval batch 0 recons_loss 0.337 ms, 1.19 s total -[ 2023-09-21 06:54:28 ] Completed eval batch 0 metrics update 0.568 ms, 1.19 s total -Saving checkpoint at epoch 38 val batch 0 -[ 2023-09-21 06:54:28 ] Completed saving temp checkpoint 425.403 ms, 1.62 s total -[ 2023-09-21 06:54:28 ] Completed replacing temp checkpoint with checkpoint 22.126 ms, 1.64 s total -[ 2023-09-21 06:54:29 ] Completed eval batch 1 to device 394.232 ms, 2.03 s total -[ 2023-09-21 06:54:29 ] Completed eval batch 1 forward 25.624 ms, 2.06 s total -[ 2023-09-21 06:54:29 ] Completed eval batch 1 recons_loss 0.371 ms, 2.06 s total -[ 2023-09-21 06:54:29 ] Completed eval batch 1 metrics update 260.318 ms, 2.32 s total -Saving checkpoint at epoch 38 val batch 1 -Epoch 38 val loss: 0.0640 -[ 2023-09-21 06:54:29 ] Completed saving temp checkpoint 455.059 ms, 2.77 s total -[ 2023-09-21 06:54:29 ] Completed replacing temp checkpoint with checkpoint 26.870 ms, 2.80 s total -[ 2023-09-21 06:54:30 ] Completed evaluating generator for epoch 38 49.693 ms, 2.85 s total - - -EPOCH :: 39 - - -[ 2023-09-21 06:54:30 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 39 from training step 0 - -[ 2023-09-21 06:54:33 ] Completed train batch 0 to device 3,026.482 ms, 3.03 s total -[ 2023-09-21 06:54:33 ] Completed train batch 0 generator forward 30.629 ms, 3.06 s total -[ 2023-09-21 06:54:33 ] Completed train batch 0 generator loss: 1.482 88.301 ms, 3.15 s total -[ 2023-09-21 06:54:33 ] Completed train batch 0 generator backward 643.514 ms, 3.79 s total -[ 2023-09-21 06:54:33 ] Completed train batch 0 discriminator loss 0.150 56.823 ms, 3.85 s total -[ 2023-09-21 06:54:33 ] Completed train batch 0 discriminator backward 125.582 ms, 3.97 s total -Epoch [39] Step [0/2], gen_loss: 0.497, disc_loss: 0.051 -[ 2023-09-21 06:54:33 ] Completed train batch 0 metrics update 12.338 ms, 3.98 s total -Saving checkpoint at epoch 39 train batch 0 -[ 2023-09-21 06:54:34 ] Completed saving temp checkpoint 689.632 ms, 4.67 s total -[ 2023-09-21 06:54:34 ] Completed replacing temp checkpoint with checkpoint 27.091 ms, 4.70 s total -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 07:05:23 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 07:05:23 ] Completed importing Timer 0.023 ms, 0.00 s total -[ 2023-09-21 07:05:28 ] Completed importing everything else 4,644.737 ms, 4.64 s total -| distributed init (rank 5): env:// -| distributed init (rank 3): env:// -| distributed init (rank 1): env:// -| distributed init (rank 0): env:// -| distributed init (rank 2): env:// -| distributed init (rank 4): env:// -[ 2023-09-21 07:05:36 ] Completed preliminaries 7,988.197 ms, 12.63 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 07:05:36 ] Completed build datasets 11.948 ms, 12.64 s total -[ 2023-09-21 07:05:36 ] Completed build samplers 0.047 ms, 12.64 s total -[ 2023-09-21 07:05:36 ] Completed build dataloaders 0.136 ms, 12.65 s total -[ 2023-09-21 07:05:36 ] Completed generator to device 88.167 ms, 12.73 s total -[ 2023-09-21 07:05:36 ] Completed discriminator to device 7.019 ms, 12.74 s total -[ 2023-09-21 07:05:37 ] Completed loss functions 390.725 ms, 13.13 s total -[ 2023-09-21 07:05:37 ] Completed models prepped for distribution 65.639 ms, 13.20 s total -[ 2023-09-21 07:05:37 ] Completed optimizers 0.690 ms, 13.20 s total -[ 2023-09-21 07:05:37 ] Completed grad scalers 0.032 ms, 13.20 s total -[ 2023-09-21 07:05:37 ] Completed checkpoint retrieval 443.243 ms, 13.64 s total - - -EPOCH :: 39 - - -[ 2023-09-21 07:05:37 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 39 from training step 1 - -[ 2023-09-21 07:05:40 ] Completed train batch 1 to device 3,022.966 ms, 3.02 s total -[ 2023-09-21 07:05:41 ] Completed train batch 1 generator forward 1,066.931 ms, 4.09 s total -[ 2023-09-21 07:05:41 ] Completed train batch 1 generator loss: 1.563 72.880 ms, 4.16 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 07:05:43 ] Completed train batch 1 generator backward 1,574.335 ms, 5.74 s total -[ 2023-09-21 07:05:43 ] Completed train batch 1 discriminator loss 0.146 55.807 ms, 5.79 s total -[ 2023-09-21 07:05:43 ] Completed train batch 1 discriminator backward 91.901 ms, 5.88 s total -Epoch [39] Step [1/2], gen_loss: 0.503, disc_loss: 0.049 -[ 2023-09-21 07:05:43 ] Completed train batch 1 metrics update 1.248 ms, 5.89 s total -Saving checkpoint at epoch 39 train batch 1 -[ 2023-09-21 07:05:43 ] Completed saving temp checkpoint 417.989 ms, 6.30 s total -[ 2023-09-21 07:05:44 ] Completed replacing temp checkpoint with checkpoint 113.759 ms, 6.42 s total -Epoch [39] :: gen_loss: 0.500, disc_loss: 0.050 -[ 2023-09-21 07:05:44 ] Completed training generator for epoch 39 39.509 ms, 6.46 s total -[ 2023-09-21 07:05:44 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 39 from eval step 0 - -[ 2023-09-21 07:05:45 ] Completed eval batch 0 to device 1,159.857 ms, 1.16 s total -[ 2023-09-21 07:05:45 ] Completed eval batch 0 forward 40.262 ms, 1.20 s total -[ 2023-09-21 07:05:45 ] Completed eval batch 0 recons_loss 0.366 ms, 1.20 s total -[ 2023-09-21 07:05:45 ] Completed eval batch 0 metrics update 0.676 ms, 1.20 s total -Saving checkpoint at epoch 39 val batch 0 -[ 2023-09-21 07:05:45 ] Completed saving temp checkpoint 396.578 ms, 1.60 s total -[ 2023-09-21 07:05:45 ] Completed replacing temp checkpoint with checkpoint 24.507 ms, 1.62 s total -[ 2023-09-21 07:05:46 ] Completed eval batch 1 to device 363.866 ms, 1.99 s total -[ 2023-09-21 07:05:46 ] Completed eval batch 1 forward 25.247 ms, 2.01 s total -[ 2023-09-21 07:05:46 ] Completed eval batch 1 recons_loss 0.377 ms, 2.01 s total -[ 2023-09-21 07:05:46 ] Completed eval batch 1 metrics update 155.394 ms, 2.17 s total -Saving checkpoint at epoch 39 val batch 1 -Epoch 39 val loss: 0.0645 -[ 2023-09-21 07:05:46 ] Completed saving temp checkpoint 433.957 ms, 2.60 s total -[ 2023-09-21 07:05:46 ] Completed replacing temp checkpoint with checkpoint 19.637 ms, 2.62 s total -[ 2023-09-21 07:05:46 ] Completed evaluating generator for epoch 39 60.775 ms, 2.68 s total - - -EPOCH :: 40 - - -[ 2023-09-21 07:05:46 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 40 from training step 0 - -[ 2023-09-21 07:05:49 ] Completed train batch 0 to device 3,154.577 ms, 3.15 s total -[ 2023-09-21 07:05:49 ] Completed train batch 0 generator forward 27.481 ms, 3.18 s total -[ 2023-09-21 07:05:50 ] Completed train batch 0 generator loss: 1.498 70.924 ms, 3.25 s total -[ 2023-09-21 07:05:50 ] Completed train batch 0 generator backward 510.772 ms, 3.76 s total -[ 2023-09-21 07:05:50 ] Completed train batch 0 discriminator loss 0.140 57.187 ms, 3.82 s total -[ 2023-09-21 07:05:50 ] Completed train batch 0 discriminator backward 107.492 ms, 3.93 s total -Epoch [40] Step [0/2], gen_loss: 0.499, disc_loss: 0.049 -[ 2023-09-21 07:05:50 ] Completed train batch 0 metrics update 5.561 ms, 3.93 s total -Saving checkpoint at epoch 40 train batch 0 -[ 2023-09-21 07:05:51 ] Completed saving temp checkpoint 453.000 ms, 4.39 s total -[ 2023-09-21 07:05:51 ] Completed replacing temp checkpoint with checkpoint 23.541 ms, 4.41 s total -[ 2023-09-21 07:05:52 ] Completed train batch 1 to device 1,805.453 ms, 6.22 s total -[ 2023-09-21 07:05:52 ] Completed train batch 1 generator forward 22.757 ms, 6.24 s total -[ 2023-09-21 07:05:53 ] Completed train batch 1 generator loss: 1.468 63.663 ms, 6.30 s total -[ 2023-09-21 07:05:53 ] Completed train batch 1 generator backward 831.206 ms, 7.13 s total -[ 2023-09-21 07:05:53 ] Completed train batch 1 discriminator loss 0.148 56.479 ms, 7.19 s total -[ 2023-09-21 07:05:54 ] Completed train batch 1 discriminator backward 76.448 ms, 7.27 s total -Epoch [40] Step [1/2], gen_loss: 0.488, disc_loss: 0.049 -[ 2023-09-21 07:05:54 ] Completed train batch 1 metrics update 0.989 ms, 7.27 s total -Saving checkpoint at epoch 40 train batch 1 -[ 2023-09-21 07:05:54 ] Completed saving temp checkpoint 530.147 ms, 7.80 s total -[ 2023-09-21 07:05:54 ] Completed replacing temp checkpoint with checkpoint 29.833 ms, 7.83 s total -Epoch [40] :: gen_loss: 0.493, disc_loss: 0.049 -[ 2023-09-21 07:05:54 ] Completed training generator for epoch 40 77.305 ms, 7.90 s total -[ 2023-09-21 07:05:54 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 40 from eval step 0 - -[ 2023-09-21 07:05:55 ] Completed eval batch 0 to device 1,141.667 ms, 1.14 s total -[ 2023-09-21 07:05:55 ] Completed eval batch 0 forward 28.488 ms, 1.17 s total -[ 2023-09-21 07:05:55 ] Completed eval batch 0 recons_loss 0.371 ms, 1.17 s total -[ 2023-09-21 07:05:55 ] Completed eval batch 0 metrics update 0.644 ms, 1.17 s total -Saving checkpoint at epoch 40 val batch 0 -[ 2023-09-21 07:05:56 ] Completed saving temp checkpoint 657.932 ms, 1.83 s total -[ 2023-09-21 07:05:56 ] Completed replacing temp checkpoint with checkpoint 38.127 ms, 1.87 s total -[ 2023-09-21 07:05:56 ] Completed eval batch 1 to device 175.778 ms, 2.04 s total -[ 2023-09-21 07:05:56 ] Completed eval batch 1 forward 24.024 ms, 2.07 s total -[ 2023-09-21 07:05:56 ] Completed eval batch 1 recons_loss 0.348 ms, 2.07 s total -[ 2023-09-21 07:05:56 ] Completed eval batch 1 metrics update 79.519 ms, 2.15 s total -Saving checkpoint at epoch 40 val batch 1 -Epoch 40 val loss: 0.0600 -[ 2023-09-21 07:05:57 ] Completed saving temp checkpoint 671.251 ms, 2.82 s total -[ 2023-09-21 07:05:57 ] Completed replacing temp checkpoint with checkpoint 41.162 ms, 2.86 s total -[ 2023-09-21 07:05:57 ] Completed evaluating generator for epoch 40 54.755 ms, 2.91 s total - - -EPOCH :: 41 - - -[ 2023-09-21 07:05:57 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 41 from training step 0 - -[ 2023-09-21 07:06:00 ] Completed train batch 0 to device 3,070.886 ms, 3.07 s total -[ 2023-09-21 07:06:00 ] Completed train batch 0 generator forward 25.109 ms, 3.10 s total -[ 2023-09-21 07:06:00 ] Completed train batch 0 generator loss: 1.535 66.550 ms, 3.16 s total -[ 2023-09-21 07:06:01 ] Completed train batch 0 generator backward 388.998 ms, 3.55 s total -[ 2023-09-21 07:06:01 ] Completed train batch 0 discriminator loss 0.145 80.970 ms, 3.63 s total -[ 2023-09-21 07:06:01 ] Completed train batch 0 discriminator backward 79.344 ms, 3.71 s total -Epoch [41] Step [0/2], gen_loss: 0.498, disc_loss: 0.048 -[ 2023-09-21 07:06:01 ] Completed train batch 0 metrics update 1.655 ms, 3.71 s total -Saving checkpoint at epoch 41 train batch 0 -[ 2023-09-21 07:06:01 ] Completed saving temp checkpoint 438.953 ms, 4.15 s total -[ 2023-09-21 07:06:01 ] Completed replacing temp checkpoint with checkpoint 19.534 ms, 4.17 s total -[ 2023-09-21 07:06:03 ] Completed train batch 1 to device 1,715.813 ms, 5.89 s total -[ 2023-09-21 07:06:03 ] Completed train batch 1 generator forward 22.355 ms, 5.91 s total -[ 2023-09-21 07:06:03 ] Completed train batch 1 generator loss: 1.472 63.750 ms, 5.97 s total -[ 2023-09-21 07:06:04 ] Completed train batch 1 generator backward 1,066.320 ms, 7.04 s total -[ 2023-09-21 07:06:04 ] Completed train batch 1 discriminator loss 0.143 69.552 ms, 7.11 s total -[ 2023-09-21 07:06:04 ] Completed train batch 1 discriminator backward 59.250 ms, 7.17 s total -Epoch [41] Step [1/2], gen_loss: 0.489, disc_loss: 0.047 -[ 2023-09-21 07:06:04 ] Completed train batch 1 metrics update 1.932 ms, 7.17 s total -Saving checkpoint at epoch 41 train batch 1 -[ 2023-09-21 07:06:05 ] Completed saving temp checkpoint 472.203 ms, 7.64 s total -[ 2023-09-21 07:06:05 ] Completed replacing temp checkpoint with checkpoint 23.048 ms, 7.67 s total -Epoch [41] :: gen_loss: 0.494, disc_loss: 0.048 -[ 2023-09-21 07:06:05 ] Completed training generator for epoch 41 59.584 ms, 7.73 s total -[ 2023-09-21 07:06:05 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 41 from eval step 0 - -[ 2023-09-21 07:06:06 ] Completed eval batch 0 to device 1,130.800 ms, 1.13 s total -[ 2023-09-21 07:06:06 ] Completed eval batch 0 forward 30.964 ms, 1.16 s total -[ 2023-09-21 07:06:06 ] Completed eval batch 0 recons_loss 0.520 ms, 1.16 s total -[ 2023-09-21 07:06:06 ] Completed eval batch 0 metrics update 1.146 ms, 1.16 s total -Saving checkpoint at epoch 41 val batch 0 -[ 2023-09-21 07:06:06 ] Completed saving temp checkpoint 480.171 ms, 1.64 s total -[ 2023-09-21 07:06:06 ] Completed replacing temp checkpoint with checkpoint 36.711 ms, 1.68 s total -[ 2023-09-21 07:06:07 ] Completed eval batch 1 to device 359.848 ms, 2.04 s total -[ 2023-09-21 07:06:07 ] Completed eval batch 1 forward 40.314 ms, 2.08 s total -[ 2023-09-21 07:06:07 ] Completed eval batch 1 recons_loss 0.593 ms, 2.08 s total -[ 2023-09-21 07:06:07 ] Completed eval batch 1 metrics update 124.310 ms, 2.21 s total -Saving checkpoint at epoch 41 val batch 1 -Epoch 41 val loss: 0.0598 -[ 2023-09-21 07:06:07 ] Completed saving temp checkpoint 475.137 ms, 2.68 s total -[ 2023-09-21 07:06:08 ] Completed replacing temp checkpoint with checkpoint 28.390 ms, 2.71 s total -[ 2023-09-21 07:06:08 ] Completed evaluating generator for epoch 41 54.527 ms, 2.76 s total - - -EPOCH :: 42 - - -[ 2023-09-21 07:06:08 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 42 from training step 0 - -[ 2023-09-21 07:06:11 ] Completed train batch 0 to device 3,121.244 ms, 3.12 s total -[ 2023-09-21 07:06:11 ] Completed train batch 0 generator forward 27.734 ms, 3.15 s total -[ 2023-09-21 07:06:11 ] Completed train batch 0 generator loss: 1.512 68.441 ms, 3.22 s total -[ 2023-09-21 07:06:11 ] Completed train batch 0 generator backward 521.588 ms, 3.74 s total -[ 2023-09-21 07:06:11 ] Completed train batch 0 discriminator loss 0.130 82.214 ms, 3.82 s total -[ 2023-09-21 07:06:11 ] Completed train batch 0 discriminator backward 82.447 ms, 3.90 s total -Epoch [42] Step [0/2], gen_loss: 0.510, disc_loss: 0.043 -[ 2023-09-21 07:06:11 ] Completed train batch 0 metrics update 0.893 ms, 3.90 s total -Saving checkpoint at epoch 42 train batch 0 -[ 2023-09-21 07:06:12 ] Completed saving temp checkpoint 474.831 ms, 4.38 s total -[ 2023-09-21 07:06:12 ] Completed replacing temp checkpoint with checkpoint 26.178 ms, 4.41 s total -[ 2023-09-21 07:06:14 ] Completed train batch 1 to device 1,555.094 ms, 5.96 s total -[ 2023-09-21 07:06:14 ] Completed train batch 1 generator forward 26.635 ms, 5.99 s total -[ 2023-09-21 07:06:14 ] Completed train batch 1 generator loss: 1.503 63.590 ms, 6.05 s total -[ 2023-09-21 07:06:15 ] Completed train batch 1 generator backward 1,271.978 ms, 7.32 s total -[ 2023-09-21 07:06:15 ] Completed train batch 1 discriminator loss 0.137 56.366 ms, 7.38 s total -[ 2023-09-21 07:06:15 ] Completed train batch 1 discriminator backward 75.974 ms, 7.46 s total -Epoch [42] Step [1/2], gen_loss: 0.505, disc_loss: 0.046 -[ 2023-09-21 07:06:15 ] Completed train batch 1 metrics update 3.972 ms, 7.46 s total -Saving checkpoint at epoch 42 train batch 1 -[ 2023-09-21 07:06:15 ] Completed saving temp checkpoint 403.848 ms, 7.86 s total -[ 2023-09-21 07:06:15 ] Completed replacing temp checkpoint with checkpoint 25.428 ms, 7.89 s total -Epoch [42] :: gen_loss: 0.507, disc_loss: 0.045 -[ 2023-09-21 07:06:16 ] Completed training generator for epoch 42 61.049 ms, 7.95 s total -[ 2023-09-21 07:06:16 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 42 from eval step 0 - -[ 2023-09-21 07:06:17 ] Completed eval batch 0 to device 1,113.964 ms, 1.11 s total -[ 2023-09-21 07:06:17 ] Completed eval batch 0 forward 34.359 ms, 1.15 s total -[ 2023-09-21 07:06:17 ] Completed eval batch 0 recons_loss 0.364 ms, 1.15 s total -[ 2023-09-21 07:06:17 ] Completed eval batch 0 metrics update 0.618 ms, 1.15 s total -Saving checkpoint at epoch 42 val batch 0 -[ 2023-09-21 07:06:17 ] Completed saving temp checkpoint 365.322 ms, 1.51 s total -[ 2023-09-21 07:06:17 ] Completed replacing temp checkpoint with checkpoint 25.831 ms, 1.54 s total -[ 2023-09-21 07:06:18 ] Completed eval batch 1 to device 507.185 ms, 2.05 s total -[ 2023-09-21 07:06:18 ] Completed eval batch 1 forward 21.934 ms, 2.07 s total -[ 2023-09-21 07:06:18 ] Completed eval batch 1 recons_loss 0.379 ms, 2.07 s total -[ 2023-09-21 07:06:18 ] Completed eval batch 1 metrics update 205.030 ms, 2.27 s total -Saving checkpoint at epoch 42 val batch 1 -Epoch 42 val loss: 0.0852 -[ 2023-09-21 07:06:18 ] Completed saving temp checkpoint 461.410 ms, 2.74 s total -[ 2023-09-21 07:06:18 ] Completed replacing temp checkpoint with checkpoint 20.019 ms, 2.76 s total -[ 2023-09-21 07:06:18 ] Completed evaluating generator for epoch 42 51.541 ms, 2.81 s total - - -EPOCH :: 43 - - -[ 2023-09-21 07:06:18 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 43 from training step 0 - -[ 2023-09-21 07:06:21 ] Completed train batch 0 to device 3,014.369 ms, 3.01 s total -[ 2023-09-21 07:06:21 ] Completed train batch 0 generator forward 30.335 ms, 3.04 s total -[ 2023-09-21 07:06:21 ] Completed train batch 0 generator loss: 1.506 66.371 ms, 3.11 s total -[ 2023-09-21 07:06:22 ] Completed train batch 0 generator backward 654.990 ms, 3.77 s total -[ 2023-09-21 07:06:22 ] Completed train batch 0 discriminator loss 0.133 57.384 ms, 3.82 s total -[ 2023-09-21 07:06:22 ] Completed train batch 0 discriminator backward 106.099 ms, 3.93 s total -Epoch [43] Step [0/2], gen_loss: 0.495, disc_loss: 0.045 -[ 2023-09-21 07:06:22 ] Completed train batch 0 metrics update 5.368 ms, 3.93 s total -Saving checkpoint at epoch 43 train batch 0 -[ 2023-09-21 07:06:23 ] Completed saving temp checkpoint 448.580 ms, 4.38 s total -[ 2023-09-21 07:06:23 ] Completed replacing temp checkpoint with checkpoint 25.432 ms, 4.41 s total -[ 2023-09-21 07:06:24 ] Completed train batch 1 to device 1,339.399 ms, 5.75 s total -[ 2023-09-21 07:06:24 ] Completed train batch 1 generator forward 40.666 ms, 5.79 s total -[ 2023-09-21 07:06:24 ] Completed train batch 1 generator loss: 1.522 63.405 ms, 5.85 s total -[ 2023-09-21 07:06:25 ] Completed train batch 1 generator backward 1,184.482 ms, 7.04 s total -[ 2023-09-21 07:06:25 ] Completed train batch 1 discriminator loss 0.127 56.162 ms, 7.09 s total -[ 2023-09-21 07:06:25 ] Completed train batch 1 discriminator backward 74.925 ms, 7.17 s total -Epoch [43] Step [1/2], gen_loss: 0.513, disc_loss: 0.043 -[ 2023-09-21 07:06:25 ] Completed train batch 1 metrics update 1.218 ms, 7.17 s total -Saving checkpoint at epoch 43 train batch 1 -[ 2023-09-21 07:06:26 ] Completed saving temp checkpoint 459.538 ms, 7.63 s total -[ 2023-09-21 07:06:26 ] Completed replacing temp checkpoint with checkpoint 28.861 ms, 7.66 s total -Epoch [43] :: gen_loss: 0.504, disc_loss: 0.044 -[ 2023-09-21 07:06:26 ] Completed training generator for epoch 43 57.037 ms, 7.71 s total -[ 2023-09-21 07:06:26 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 43 from eval step 0 - -[ 2023-09-21 07:06:27 ] Completed eval batch 0 to device 1,030.465 ms, 1.03 s total -[ 2023-09-21 07:06:27 ] Completed eval batch 0 forward 30.254 ms, 1.06 s total -[ 2023-09-21 07:06:27 ] Completed eval batch 0 recons_loss 0.281 ms, 1.06 s total -[ 2023-09-21 07:06:27 ] Completed eval batch 0 metrics update 0.583 ms, 1.06 s total -Saving checkpoint at epoch 43 val batch 0 -[ 2023-09-21 07:06:27 ] Completed saving temp checkpoint 389.758 ms, 1.45 s total -[ 2023-09-21 07:06:28 ] Completed replacing temp checkpoint with checkpoint 30.970 ms, 1.48 s total -[ 2023-09-21 07:06:28 ] Completed eval batch 1 to device 530.329 ms, 2.01 s total -[ 2023-09-21 07:06:28 ] Completed eval batch 1 forward 39.214 ms, 2.05 s total -[ 2023-09-21 07:06:28 ] Completed eval batch 1 recons_loss 0.349 ms, 2.05 s total -[ 2023-09-21 07:06:28 ] Completed eval batch 1 metrics update 149.366 ms, 2.20 s total -Saving checkpoint at epoch 43 val batch 1 -Epoch 43 val loss: 0.0800 -[ 2023-09-21 07:06:29 ] Completed saving temp checkpoint 406.484 ms, 2.61 s total -[ 2023-09-21 07:06:29 ] Completed replacing temp checkpoint with checkpoint 21.200 ms, 2.63 s total -[ 2023-09-21 07:06:29 ] Completed evaluating generator for epoch 43 53.386 ms, 2.68 s total - - -EPOCH :: 44 - - -[ 2023-09-21 07:06:29 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 44 from training step 0 - -[ 2023-09-21 07:06:32 ] Completed train batch 0 to device 3,202.604 ms, 3.20 s total -[ 2023-09-21 07:06:32 ] Completed train batch 0 generator forward 41.640 ms, 3.24 s total -[ 2023-09-21 07:06:32 ] Completed train batch 0 generator loss: 1.566 80.623 ms, 3.32 s total -[ 2023-09-21 07:06:33 ] Completed train batch 0 generator backward 525.749 ms, 3.85 s total -[ 2023-09-21 07:06:33 ] Completed train batch 0 discriminator loss 0.123 59.276 ms, 3.91 s total -[ 2023-09-21 07:06:33 ] Completed train batch 0 discriminator backward 102.375 ms, 4.01 s total -Epoch [44] Step [0/2], gen_loss: 0.521, disc_loss: 0.041 -[ 2023-09-21 07:06:33 ] Completed train batch 0 metrics update 11.493 ms, 4.02 s total -Saving checkpoint at epoch 44 train batch 0 -[ 2023-09-21 07:06:33 ] Completed saving temp checkpoint 347.530 ms, 4.37 s total -[ 2023-09-21 07:06:33 ] Completed replacing temp checkpoint with checkpoint 22.018 ms, 4.39 s total -[ 2023-09-21 07:06:35 ] Completed train batch 1 to device 1,676.775 ms, 6.07 s total -[ 2023-09-21 07:06:35 ] Completed train batch 1 generator forward 26.091 ms, 6.10 s total -[ 2023-09-21 07:06:35 ] Completed train batch 1 generator loss: 1.524 67.508 ms, 6.16 s total -[ 2023-09-21 07:06:36 ] Completed train batch 1 generator backward 955.951 ms, 7.12 s total -[ 2023-09-21 07:06:36 ] Completed train batch 1 discriminator loss 0.129 57.411 ms, 7.18 s total -[ 2023-09-21 07:06:36 ] Completed train batch 1 discriminator backward 66.774 ms, 7.24 s total -Epoch [44] Step [1/2], gen_loss: 0.511, disc_loss: 0.044 -[ 2023-09-21 07:06:36 ] Completed train batch 1 metrics update 2.074 ms, 7.25 s total -Saving checkpoint at epoch 44 train batch 1 -[ 2023-09-21 07:06:36 ] Completed saving temp checkpoint 434.741 ms, 7.68 s total -[ 2023-09-21 07:06:36 ] Completed replacing temp checkpoint with checkpoint 30.576 ms, 7.71 s total -Epoch [44] :: gen_loss: 0.516, disc_loss: 0.043 -[ 2023-09-21 07:06:36 ] Completed training generator for epoch 44 60.137 ms, 7.77 s total -[ 2023-09-21 07:06:36 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 44 from eval step 0 - -[ 2023-09-21 07:06:38 ] Completed eval batch 0 to device 1,062.463 ms, 1.06 s total -[ 2023-09-21 07:06:38 ] Completed eval batch 0 forward 29.764 ms, 1.09 s total -[ 2023-09-21 07:06:38 ] Completed eval batch 0 recons_loss 0.274 ms, 1.09 s total -[ 2023-09-21 07:06:38 ] Completed eval batch 0 metrics update 0.594 ms, 1.09 s total -Saving checkpoint at epoch 44 val batch 0 -[ 2023-09-21 07:06:38 ] Completed saving temp checkpoint 475.919 ms, 1.57 s total -[ 2023-09-21 07:06:38 ] Completed replacing temp checkpoint with checkpoint 33.217 ms, 1.60 s total -[ 2023-09-21 07:06:38 ] Completed eval batch 1 to device 384.403 ms, 1.99 s total -[ 2023-09-21 07:06:39 ] Completed eval batch 1 forward 36.944 ms, 2.02 s total -[ 2023-09-21 07:06:39 ] Completed eval batch 1 recons_loss 0.389 ms, 2.02 s total -[ 2023-09-21 07:06:39 ] Completed eval batch 1 metrics update 192.669 ms, 2.22 s total -Saving checkpoint at epoch 44 val batch 1 -Epoch 44 val loss: 0.0733 -[ 2023-09-21 07:06:39 ] Completed saving temp checkpoint 594.753 ms, 2.81 s total -[ 2023-09-21 07:06:39 ] Completed replacing temp checkpoint with checkpoint 23.059 ms, 2.83 s total -[ 2023-09-21 07:06:39 ] Completed evaluating generator for epoch 44 63.419 ms, 2.90 s total - - -EPOCH :: 45 - - -[ 2023-09-21 07:06:39 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 45 from training step 0 - -[ 2023-09-21 07:06:42 ] Completed train batch 0 to device 3,088.746 ms, 3.09 s total -[ 2023-09-21 07:06:43 ] Completed train batch 0 generator forward 29.960 ms, 3.12 s total -[ 2023-09-21 07:06:43 ] Completed train batch 0 generator loss: 1.503 66.797 ms, 3.19 s total -[ 2023-09-21 07:06:43 ] Completed train batch 0 generator backward 342.728 ms, 3.53 s total -[ 2023-09-21 07:06:43 ] Completed train batch 0 discriminator loss 0.136 58.023 ms, 3.59 s total -[ 2023-09-21 07:06:43 ] Completed train batch 0 discriminator backward 106.709 ms, 3.69 s total -Epoch [45] Step [0/2], gen_loss: 0.506, disc_loss: 0.045 -[ 2023-09-21 07:06:43 ] Completed train batch 0 metrics update 8.739 ms, 3.70 s total -Saving checkpoint at epoch 45 train batch 0 -[ 2023-09-21 07:06:44 ] Completed saving temp checkpoint 420.765 ms, 4.12 s total -[ 2023-09-21 07:06:44 ] Completed replacing temp checkpoint with checkpoint 28.805 ms, 4.15 s total -[ 2023-09-21 07:06:46 ] Completed train batch 1 to device 2,115.938 ms, 6.27 s total -[ 2023-09-21 07:06:46 ] Completed train batch 1 generator forward 22.448 ms, 6.29 s total -[ 2023-09-21 07:06:46 ] Completed train batch 1 generator loss: 1.490 63.583 ms, 6.35 s total -[ 2023-09-21 07:06:46 ] Completed train batch 1 generator backward 608.564 ms, 6.96 s total -[ 2023-09-21 07:06:46 ] Completed train batch 1 discriminator loss 0.138 56.415 ms, 7.02 s total -[ 2023-09-21 07:06:46 ] Completed train batch 1 discriminator backward 74.148 ms, 7.09 s total -Epoch [45] Step [1/2], gen_loss: 0.498, disc_loss: 0.045 -[ 2023-09-21 07:06:46 ] Completed train batch 1 metrics update 4.994 ms, 7.10 s total -Saving checkpoint at epoch 45 train batch 1 -[ 2023-09-21 07:06:47 ] Completed saving temp checkpoint 430.568 ms, 7.53 s total -[ 2023-09-21 07:06:47 ] Completed replacing temp checkpoint with checkpoint 27.656 ms, 7.56 s total -Epoch [45] :: gen_loss: 0.502, disc_loss: 0.045 -[ 2023-09-21 07:06:47 ] Completed training generator for epoch 45 60.273 ms, 7.62 s total -[ 2023-09-21 07:06:47 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 45 from eval step 0 - -[ 2023-09-21 07:06:48 ] Completed eval batch 0 to device 1,134.429 ms, 1.13 s total -[ 2023-09-21 07:06:48 ] Completed eval batch 0 forward 26.363 ms, 1.16 s total -[ 2023-09-21 07:06:48 ] Completed eval batch 0 recons_loss 0.365 ms, 1.16 s total -[ 2023-09-21 07:06:48 ] Completed eval batch 0 metrics update 0.621 ms, 1.16 s total -Saving checkpoint at epoch 45 val batch 0 -[ 2023-09-21 07:06:49 ] Completed saving temp checkpoint 377.951 ms, 1.54 s total -[ 2023-09-21 07:06:49 ] Completed replacing temp checkpoint with checkpoint 24.105 ms, 1.56 s total -[ 2023-09-21 07:06:49 ] Completed eval batch 1 to device 523.387 ms, 2.09 s total -[ 2023-09-21 07:06:49 ] Completed eval batch 1 forward 39.404 ms, 2.13 s total -[ 2023-09-21 07:06:49 ] Completed eval batch 1 recons_loss 0.343 ms, 2.13 s total -[ 2023-09-21 07:06:49 ] Completed eval batch 1 metrics update 56.165 ms, 2.18 s total -Saving checkpoint at epoch 45 val batch 1 -Epoch 45 val loss: 0.0631 -[ 2023-09-21 07:06:50 ] Completed saving temp checkpoint 604.707 ms, 2.79 s total -[ 2023-09-21 07:06:50 ] Completed replacing temp checkpoint with checkpoint 26.340 ms, 2.81 s total -[ 2023-09-21 07:06:50 ] Completed evaluating generator for epoch 45 67.911 ms, 2.88 s total - - -EPOCH :: 46 - - -[ 2023-09-21 07:06:50 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 46 from training step 0 - -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 07:17:39 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 07:17:39 ] Completed importing Timer 0.025 ms, 0.00 s total -[ 2023-09-21 07:17:44 ] Completed importing everything else 4,968.759 ms, 4.97 s total -| distributed init (rank 1): env:// -| distributed init (rank 4): env:// -| distributed init (rank 5): env:// -| distributed init (rank 3): env:// -| distributed init (rank 0): env:// -| distributed init (rank 2): env:// -[ 2023-09-21 07:17:52 ] Completed preliminaries 7,570.606 ms, 12.54 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 07:17:52 ] Completed build datasets 11.481 ms, 12.55 s total -[ 2023-09-21 07:17:52 ] Completed build samplers 0.044 ms, 12.55 s total -[ 2023-09-21 07:17:52 ] Completed build dataloaders 0.129 ms, 12.55 s total -[ 2023-09-21 07:17:52 ] Completed generator to device 77.785 ms, 12.63 s total -[ 2023-09-21 07:17:52 ] Completed discriminator to device 7.040 ms, 12.64 s total -[ 2023-09-21 07:17:52 ] Completed loss functions 401.550 ms, 13.04 s total -[ 2023-09-21 07:17:52 ] Completed models prepped for distribution 99.585 ms, 13.14 s total -[ 2023-09-21 07:17:52 ] Completed optimizers 0.682 ms, 13.14 s total -[ 2023-09-21 07:17:52 ] Completed grad scalers 0.037 ms, 13.14 s total -[ 2023-09-21 07:17:53 ] Completed checkpoint retrieval 503.083 ms, 13.64 s total - - -EPOCH :: 45 - - -[ 2023-09-21 07:17:53 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 45 from training step 2 - -Epoch [45] :: gen_loss: 0.502, disc_loss: 0.045 -[ 2023-09-21 07:17:53 ] Completed training generator for epoch 45 112.079 ms, 0.11 s total -[ 2023-09-21 07:17:53 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 45 from eval step 2 - -[ 2023-09-21 07:17:53 ] Completed evaluating generator for epoch 45 119.907 ms, 0.12 s total - - -EPOCH :: 46 - - -[ 2023-09-21 07:17:53 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 46 from training step 0 - -[ 2023-09-21 07:17:57 ] Completed train batch 0 to device 3,582.667 ms, 3.58 s total -[ 2023-09-21 07:17:58 ] Completed train batch 0 generator forward 1,551.750 ms, 5.13 s total -[ 2023-09-21 07:17:58 ] Completed train batch 0 generator loss: 1.505 81.797 ms, 5.22 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 07:18:00 ] Completed train batch 0 generator backward 1,424.238 ms, 6.64 s total -[ 2023-09-21 07:18:00 ] Completed train batch 0 discriminator loss 0.131 55.395 ms, 6.70 s total -[ 2023-09-21 07:18:00 ] Completed train batch 0 discriminator backward 101.391 ms, 6.80 s total -Epoch [46] Step [0/2], gen_loss: 0.497, disc_loss: 0.044 -[ 2023-09-21 07:18:00 ] Completed train batch 0 metrics update 6.389 ms, 6.80 s total -Saving checkpoint at epoch 46 train batch 0 -[ 2023-09-21 07:18:00 ] Completed saving temp checkpoint 444.336 ms, 7.25 s total -[ 2023-09-21 07:18:00 ] Completed replacing temp checkpoint with checkpoint 125.664 ms, 7.37 s total -[ 2023-09-21 07:18:00 ] Completed train batch 1 to device 21.719 ms, 7.40 s total -[ 2023-09-21 07:18:00 ] Completed train batch 1 generator forward 36.451 ms, 7.43 s total -[ 2023-09-21 07:18:01 ] Completed train batch 1 generator loss: 1.556 62.211 ms, 7.49 s total -[ 2023-09-21 07:18:02 ] Completed train batch 1 generator backward 1,315.460 ms, 8.81 s total -[ 2023-09-21 07:18:02 ] Completed train batch 1 discriminator loss 0.115 55.774 ms, 8.87 s total -[ 2023-09-21 07:18:02 ] Completed train batch 1 discriminator backward 71.695 ms, 8.94 s total -Epoch [46] Step [1/2], gen_loss: 0.509, disc_loss: 0.038 -[ 2023-09-21 07:18:02 ] Completed train batch 1 metrics update 0.990 ms, 8.94 s total -Saving checkpoint at epoch 46 train batch 1 -[ 2023-09-21 07:18:02 ] Completed saving temp checkpoint 446.320 ms, 9.38 s total -[ 2023-09-21 07:18:02 ] Completed replacing temp checkpoint with checkpoint 25.707 ms, 9.41 s total -Epoch [46] :: gen_loss: 0.503, disc_loss: 0.041 -[ 2023-09-21 07:18:02 ] Completed training generator for epoch 46 37.110 ms, 9.45 s total -[ 2023-09-21 07:18:02 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 46 from eval step 0 - -[ 2023-09-21 07:18:04 ] Completed eval batch 0 to device 1,133.503 ms, 1.13 s total -[ 2023-09-21 07:18:04 ] Completed eval batch 0 forward 24.993 ms, 1.16 s total -[ 2023-09-21 07:18:04 ] Completed eval batch 0 recons_loss 0.339 ms, 1.16 s total -[ 2023-09-21 07:18:04 ] Completed eval batch 0 metrics update 0.614 ms, 1.16 s total -Saving checkpoint at epoch 46 val batch 0 -[ 2023-09-21 07:18:04 ] Completed saving temp checkpoint 380.225 ms, 1.54 s total -[ 2023-09-21 07:18:04 ] Completed replacing temp checkpoint with checkpoint 28.548 ms, 1.57 s total -[ 2023-09-21 07:18:04 ] Completed eval batch 1 to device 426.021 ms, 1.99 s total -[ 2023-09-21 07:18:05 ] Completed eval batch 1 forward 40.887 ms, 2.04 s total -[ 2023-09-21 07:18:05 ] Completed eval batch 1 recons_loss 0.498 ms, 2.04 s total -[ 2023-09-21 07:18:05 ] Completed eval batch 1 metrics update 193.611 ms, 2.23 s total -Saving checkpoint at epoch 46 val batch 1 -Epoch 46 val loss: 0.0676 -[ 2023-09-21 07:18:05 ] Completed saving temp checkpoint 427.385 ms, 2.66 s total -[ 2023-09-21 07:18:05 ] Completed replacing temp checkpoint with checkpoint 23.389 ms, 2.68 s total -[ 2023-09-21 07:18:05 ] Completed evaluating generator for epoch 46 47.534 ms, 2.73 s total - - -EPOCH :: 47 - - -[ 2023-09-21 07:18:05 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 47 from training step 0 - -[ 2023-09-21 07:18:08 ] Completed train batch 0 to device 3,171.520 ms, 3.17 s total -[ 2023-09-21 07:18:08 ] Completed train batch 0 generator forward 27.453 ms, 3.20 s total -[ 2023-09-21 07:18:08 ] Completed train batch 0 generator loss: 1.459 67.114 ms, 3.27 s total -[ 2023-09-21 07:18:09 ] Completed train batch 0 generator backward 662.138 ms, 3.93 s total -[ 2023-09-21 07:18:09 ] Completed train batch 0 discriminator loss 0.135 57.341 ms, 3.99 s total -[ 2023-09-21 07:18:09 ] Completed train batch 0 discriminator backward 112.715 ms, 4.10 s total -Epoch [47] Step [0/2], gen_loss: 0.498, disc_loss: 0.045 -[ 2023-09-21 07:18:09 ] Completed train batch 0 metrics update 3.998 ms, 4.10 s total -Saving checkpoint at epoch 47 train batch 0 -[ 2023-09-21 07:18:10 ] Completed saving temp checkpoint 423.948 ms, 4.53 s total -[ 2023-09-21 07:18:10 ] Completed replacing temp checkpoint with checkpoint 29.372 ms, 4.56 s total -[ 2023-09-21 07:18:11 ] Completed train batch 1 to device 1,505.213 ms, 6.06 s total -[ 2023-09-21 07:18:11 ] Completed train batch 1 generator forward 39.398 ms, 6.10 s total -[ 2023-09-21 07:18:11 ] Completed train batch 1 generator loss: 1.515 67.652 ms, 6.17 s total -[ 2023-09-21 07:18:13 ] Completed train batch 1 generator backward 1,239.204 ms, 7.41 s total -[ 2023-09-21 07:18:13 ] Completed train batch 1 discriminator loss 0.131 55.819 ms, 7.46 s total -[ 2023-09-21 07:18:13 ] Completed train batch 1 discriminator backward 79.449 ms, 7.54 s total -Epoch [47] Step [1/2], gen_loss: 0.497, disc_loss: 0.044 -[ 2023-09-21 07:18:13 ] Completed train batch 1 metrics update 0.880 ms, 7.54 s total -Saving checkpoint at epoch 47 train batch 1 -[ 2023-09-21 07:18:13 ] Completed saving temp checkpoint 477.477 ms, 8.02 s total -[ 2023-09-21 07:18:13 ] Completed replacing temp checkpoint with checkpoint 31.007 ms, 8.05 s total -Epoch [47] :: gen_loss: 0.497, disc_loss: 0.045 -[ 2023-09-21 07:18:13 ] Completed training generator for epoch 47 52.838 ms, 8.10 s total -[ 2023-09-21 07:18:13 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 47 from eval step 0 - -[ 2023-09-21 07:18:14 ] Completed eval batch 0 to device 1,158.970 ms, 1.16 s total -[ 2023-09-21 07:18:15 ] Completed eval batch 0 forward 43.657 ms, 1.20 s total -[ 2023-09-21 07:18:15 ] Completed eval batch 0 recons_loss 0.369 ms, 1.20 s total -[ 2023-09-21 07:18:15 ] Completed eval batch 0 metrics update 0.862 ms, 1.20 s total -Saving checkpoint at epoch 47 val batch 0 -[ 2023-09-21 07:18:15 ] Completed saving temp checkpoint 388.957 ms, 1.59 s total -[ 2023-09-21 07:18:15 ] Completed replacing temp checkpoint with checkpoint 18.538 ms, 1.61 s total -[ 2023-09-21 07:18:15 ] Completed eval batch 1 to device 460.485 ms, 2.07 s total -[ 2023-09-21 07:18:15 ] Completed eval batch 1 forward 37.754 ms, 2.11 s total -[ 2023-09-21 07:18:15 ] Completed eval batch 1 recons_loss 0.343 ms, 2.11 s total -[ 2023-09-21 07:18:16 ] Completed eval batch 1 metrics update 166.735 ms, 2.28 s total -Saving checkpoint at epoch 47 val batch 1 -Epoch 47 val loss: 0.0661 -[ 2023-09-21 07:18:16 ] Completed saving temp checkpoint 445.364 ms, 2.72 s total -[ 2023-09-21 07:18:16 ] Completed replacing temp checkpoint with checkpoint 21.592 ms, 2.74 s total -[ 2023-09-21 07:18:16 ] Completed evaluating generator for epoch 47 45.692 ms, 2.79 s total - - -EPOCH :: 48 - - -[ 2023-09-21 07:18:16 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 48 from training step 0 - -[ 2023-09-21 07:18:19 ] Completed train batch 0 to device 3,198.751 ms, 3.20 s total -[ 2023-09-21 07:18:19 ] Completed train batch 0 generator forward 28.901 ms, 3.23 s total -[ 2023-09-21 07:18:19 ] Completed train batch 0 generator loss: 1.539 66.530 ms, 3.29 s total -[ 2023-09-21 07:18:20 ] Completed train batch 0 generator backward 685.345 ms, 3.98 s total -[ 2023-09-21 07:18:20 ] Completed train batch 0 discriminator loss 0.129 56.877 ms, 4.04 s total -[ 2023-09-21 07:18:20 ] Completed train batch 0 discriminator backward 116.290 ms, 4.15 s total -Epoch [48] Step [0/2], gen_loss: 0.494, disc_loss: 0.042 -[ 2023-09-21 07:18:20 ] Completed train batch 0 metrics update 5.067 ms, 4.16 s total -Saving checkpoint at epoch 48 train batch 0 -[ 2023-09-21 07:18:21 ] Completed saving temp checkpoint 431.673 ms, 4.59 s total -[ 2023-09-21 07:18:21 ] Completed replacing temp checkpoint with checkpoint 30.364 ms, 4.62 s total -[ 2023-09-21 07:18:22 ] Completed train batch 1 to device 1,746.709 ms, 6.37 s total -[ 2023-09-21 07:18:23 ] Completed train batch 1 generator forward 40.592 ms, 6.41 s total -[ 2023-09-21 07:18:23 ] Completed train batch 1 generator loss: 1.543 63.809 ms, 6.47 s total -[ 2023-09-21 07:18:24 ] Completed train batch 1 generator backward 977.688 ms, 7.45 s total -[ 2023-09-21 07:18:24 ] Completed train batch 1 discriminator loss 0.125 61.987 ms, 7.51 s total -[ 2023-09-21 07:18:24 ] Completed train batch 1 discriminator backward 65.804 ms, 7.58 s total -Epoch [48] Step [1/2], gen_loss: 0.499, disc_loss: 0.039 -[ 2023-09-21 07:18:24 ] Completed train batch 1 metrics update 0.577 ms, 7.58 s total -Saving checkpoint at epoch 48 train batch 1 -[ 2023-09-21 07:18:24 ] Completed saving temp checkpoint 686.801 ms, 8.26 s total -[ 2023-09-21 07:18:24 ] Completed replacing temp checkpoint with checkpoint 23.168 ms, 8.29 s total -Epoch [48] :: gen_loss: 0.497, disc_loss: 0.041 -[ 2023-09-21 07:18:24 ] Completed training generator for epoch 48 49.808 ms, 8.34 s total -[ 2023-09-21 07:18:24 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 48 from eval step 0 - -[ 2023-09-21 07:18:26 ] Completed eval batch 0 to device 1,076.696 ms, 1.08 s total -[ 2023-09-21 07:18:26 ] Completed eval batch 0 forward 37.554 ms, 1.11 s total -[ 2023-09-21 07:18:26 ] Completed eval batch 0 recons_loss 0.365 ms, 1.11 s total -[ 2023-09-21 07:18:26 ] Completed eval batch 0 metrics update 0.736 ms, 1.12 s total -Saving checkpoint at epoch 48 val batch 0 -[ 2023-09-21 07:18:26 ] Completed saving temp checkpoint 521.569 ms, 1.64 s total -[ 2023-09-21 07:18:26 ] Completed replacing temp checkpoint with checkpoint 24.817 ms, 1.66 s total -[ 2023-09-21 07:18:26 ] Completed eval batch 1 to device 270.174 ms, 1.93 s total -[ 2023-09-21 07:18:26 ] Completed eval batch 1 forward 38.461 ms, 1.97 s total -[ 2023-09-21 07:18:26 ] Completed eval batch 1 recons_loss 0.355 ms, 1.97 s total -[ 2023-09-21 07:18:27 ] Completed eval batch 1 metrics update 101.511 ms, 2.07 s total -Saving checkpoint at epoch 48 val batch 1 -Epoch 48 val loss: 0.0680 -[ 2023-09-21 07:18:27 ] Completed saving temp checkpoint 453.318 ms, 2.53 s total -[ 2023-09-21 07:18:27 ] Completed replacing temp checkpoint with checkpoint 23.442 ms, 2.55 s total -[ 2023-09-21 07:18:27 ] Completed evaluating generator for epoch 48 46.677 ms, 2.60 s total - - -EPOCH :: 49 - - -[ 2023-09-21 07:18:27 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 49 from training step 0 - -[ 2023-09-21 07:18:30 ] Completed train batch 0 to device 3,243.125 ms, 3.24 s total -[ 2023-09-21 07:18:30 ] Completed train batch 0 generator forward 30.685 ms, 3.27 s total -[ 2023-09-21 07:18:30 ] Completed train batch 0 generator loss: 1.524 65.423 ms, 3.34 s total -[ 2023-09-21 07:18:31 ] Completed train batch 0 generator backward 513.364 ms, 3.85 s total -[ 2023-09-21 07:18:31 ] Completed train batch 0 discriminator loss 0.114 56.994 ms, 3.91 s total -[ 2023-09-21 07:18:31 ] Completed train batch 0 discriminator backward 102.851 ms, 4.01 s total -Epoch [49] Step [0/2], gen_loss: 0.503, disc_loss: 0.038 -[ 2023-09-21 07:18:31 ] Completed train batch 0 metrics update 7.010 ms, 4.02 s total -Saving checkpoint at epoch 49 train batch 0 -[ 2023-09-21 07:18:31 ] Completed saving temp checkpoint 442.037 ms, 4.46 s total -[ 2023-09-21 07:18:32 ] Completed replacing temp checkpoint with checkpoint 22.113 ms, 4.48 s total -[ 2023-09-21 07:18:33 ] Completed train batch 1 to device 1,926.042 ms, 6.41 s total -[ 2023-09-21 07:18:33 ] Completed train batch 1 generator forward 40.658 ms, 6.45 s total -[ 2023-09-21 07:18:34 ] Completed train batch 1 generator loss: 1.434 63.699 ms, 6.51 s total -[ 2023-09-21 07:18:34 ] Completed train batch 1 generator backward 717.045 ms, 7.23 s total -[ 2023-09-21 07:18:34 ] Completed train batch 1 discriminator loss 0.117 55.496 ms, 7.29 s total -[ 2023-09-21 07:18:34 ] Completed train batch 1 discriminator backward 82.731 ms, 7.37 s total -Epoch [49] Step [1/2], gen_loss: 0.495, disc_loss: 0.039 -[ 2023-09-21 07:18:34 ] Completed train batch 1 metrics update 2.122 ms, 7.37 s total -Saving checkpoint at epoch 49 train batch 1 -[ 2023-09-21 07:18:35 ] Completed saving temp checkpoint 440.995 ms, 7.81 s total -[ 2023-09-21 07:18:35 ] Completed replacing temp checkpoint with checkpoint 23.117 ms, 7.84 s total -Epoch [49] :: gen_loss: 0.499, disc_loss: 0.039 -[ 2023-09-21 07:18:35 ] Completed training generator for epoch 49 49.593 ms, 7.89 s total -[ 2023-09-21 07:18:35 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 49 from eval step 0 - -[ 2023-09-21 07:18:36 ] Completed eval batch 0 to device 1,158.535 ms, 1.16 s total -[ 2023-09-21 07:18:36 ] Completed eval batch 0 forward 43.990 ms, 1.20 s total -[ 2023-09-21 07:18:36 ] Completed eval batch 0 recons_loss 0.308 ms, 1.20 s total -[ 2023-09-21 07:18:36 ] Completed eval batch 0 metrics update 0.590 ms, 1.20 s total -Saving checkpoint at epoch 49 val batch 0 -[ 2023-09-21 07:18:37 ] Completed saving temp checkpoint 451.297 ms, 1.65 s total -[ 2023-09-21 07:18:37 ] Completed replacing temp checkpoint with checkpoint 34.371 ms, 1.69 s total -[ 2023-09-21 07:18:37 ] Completed eval batch 1 to device 376.646 ms, 2.07 s total -[ 2023-09-21 07:18:37 ] Completed eval batch 1 forward 39.838 ms, 2.11 s total -[ 2023-09-21 07:18:37 ] Completed eval batch 1 recons_loss 0.581 ms, 2.11 s total -[ 2023-09-21 07:18:37 ] Completed eval batch 1 metrics update 76.644 ms, 2.18 s total -Saving checkpoint at epoch 49 val batch 1 -Epoch 49 val loss: 0.0695 -[ 2023-09-21 07:18:38 ] Completed saving temp checkpoint 489.921 ms, 2.67 s total -[ 2023-09-21 07:18:38 ] Completed replacing temp checkpoint with checkpoint 26.766 ms, 2.70 s total -[ 2023-09-21 07:18:38 ] Completed evaluating generator for epoch 49 50.867 ms, 2.75 s total - - -EPOCH :: 50 - - -[ 2023-09-21 07:18:38 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 50 from training step 0 - -[ 2023-09-21 07:18:41 ] Completed train batch 0 to device 3,091.808 ms, 3.09 s total -[ 2023-09-21 07:18:41 ] Completed train batch 0 generator forward 43.706 ms, 3.14 s total -[ 2023-09-21 07:18:41 ] Completed train batch 0 generator loss: 1.412 66.672 ms, 3.20 s total -[ 2023-09-21 07:18:41 ] Completed train batch 0 generator backward 570.518 ms, 3.77 s total -[ 2023-09-21 07:18:41 ] Completed train batch 0 discriminator loss 0.114 56.686 ms, 3.83 s total -[ 2023-09-21 07:18:42 ] Completed train batch 0 discriminator backward 115.430 ms, 3.94 s total -Epoch [50] Step [0/2], gen_loss: 0.492, disc_loss: 0.038 -[ 2023-09-21 07:18:42 ] Completed train batch 0 metrics update 21.686 ms, 3.97 s total -Saving checkpoint at epoch 50 train batch 0 -[ 2023-09-21 07:18:42 ] Completed saving temp checkpoint 462.535 ms, 4.43 s total -[ 2023-09-21 07:18:42 ] Completed replacing temp checkpoint with checkpoint 24.433 ms, 4.45 s total -[ 2023-09-21 07:18:44 ] Completed train batch 1 to device 1,519.181 ms, 5.97 s total -[ 2023-09-21 07:18:44 ] Completed train batch 1 generator forward 22.244 ms, 5.99 s total -[ 2023-09-21 07:18:44 ] Completed train batch 1 generator loss: 1.486 64.842 ms, 6.06 s total -[ 2023-09-21 07:18:45 ] Completed train batch 1 generator backward 1,323.729 ms, 7.38 s total -[ 2023-09-21 07:18:45 ] Completed train batch 1 discriminator loss 0.105 55.889 ms, 7.44 s total -[ 2023-09-21 07:18:45 ] Completed train batch 1 discriminator backward 81.862 ms, 7.52 s total -Epoch [50] Step [1/2], gen_loss: 0.492, disc_loss: 0.036 -[ 2023-09-21 07:18:45 ] Completed train batch 1 metrics update 3.746 ms, 7.52 s total -Saving checkpoint at epoch 50 train batch 1 -[ 2023-09-21 07:18:46 ] Completed saving temp checkpoint 507.008 ms, 8.03 s total -[ 2023-09-21 07:18:46 ] Completed replacing temp checkpoint with checkpoint 31.222 ms, 8.06 s total -Epoch [50] :: gen_loss: 0.492, disc_loss: 0.037 -[ 2023-09-21 07:18:46 ] Completed training generator for epoch 50 53.388 ms, 8.12 s total -[ 2023-09-21 07:18:46 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 50 from eval step 0 - -[ 2023-09-21 07:18:47 ] Completed eval batch 0 to device 1,143.930 ms, 1.14 s total -[ 2023-09-21 07:18:47 ] Completed eval batch 0 forward 40.634 ms, 1.18 s total -[ 2023-09-21 07:18:47 ] Completed eval batch 0 recons_loss 0.367 ms, 1.18 s total -[ 2023-09-21 07:18:47 ] Completed eval batch 0 metrics update 0.882 ms, 1.19 s total -Saving checkpoint at epoch 50 val batch 0 -[ 2023-09-21 07:18:47 ] Completed saving temp checkpoint 439.918 ms, 1.63 s total -[ 2023-09-21 07:18:47 ] Completed replacing temp checkpoint with checkpoint 35.674 ms, 1.66 s total -[ 2023-09-21 07:18:48 ] Completed eval batch 1 to device 344.780 ms, 2.01 s total -[ 2023-09-21 07:18:48 ] Completed eval batch 1 forward 22.965 ms, 2.03 s total -[ 2023-09-21 07:18:48 ] Completed eval batch 1 recons_loss 0.349 ms, 2.03 s total -[ 2023-09-21 07:18:48 ] Completed eval batch 1 metrics update 194.772 ms, 2.22 s total -Saving checkpoint at epoch 50 val batch 1 -Epoch 50 val loss: 0.0658 -[ 2023-09-21 07:18:48 ] Completed saving temp checkpoint 446.743 ms, 2.67 s total -[ 2023-09-21 07:18:48 ] Completed replacing temp checkpoint with checkpoint 24.729 ms, 2.70 s total -[ 2023-09-21 07:18:49 ] Completed evaluating generator for epoch 50 47.328 ms, 2.74 s total - - -EPOCH :: 51 - - -[ 2023-09-21 07:18:49 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 51 from training step 0 - -[ 2023-09-21 07:18:52 ] Completed train batch 0 to device 3,098.718 ms, 3.10 s total -[ 2023-09-21 07:18:52 ] Completed train batch 0 generator forward 27.786 ms, 3.13 s total -[ 2023-09-21 07:18:52 ] Completed train batch 0 generator loss: 1.465 67.238 ms, 3.19 s total -[ 2023-09-21 07:18:52 ] Completed train batch 0 generator backward 663.835 ms, 3.86 s total -[ 2023-09-21 07:18:52 ] Completed train batch 0 discriminator loss 0.101 57.713 ms, 3.92 s total -[ 2023-09-21 07:18:53 ] Completed train batch 0 discriminator backward 98.640 ms, 4.01 s total -Epoch [51] Step [0/2], gen_loss: 0.490, disc_loss: 0.034 -[ 2023-09-21 07:18:53 ] Completed train batch 0 metrics update 3.209 ms, 4.02 s total -Saving checkpoint at epoch 51 train batch 0 -[ 2023-09-21 07:18:53 ] Completed saving temp checkpoint 457.031 ms, 4.47 s total -[ 2023-09-21 07:18:53 ] Completed replacing temp checkpoint with checkpoint 25.404 ms, 4.50 s total -[ 2023-09-21 07:18:54 ] Completed train batch 1 to device 1,277.134 ms, 5.78 s total -[ 2023-09-21 07:18:54 ] Completed train batch 1 generator forward 22.341 ms, 5.80 s total -[ 2023-09-21 07:18:54 ] Completed train batch 1 generator loss: 1.461 64.777 ms, 5.86 s total -[ 2023-09-21 07:18:56 ] Completed train batch 1 generator backward 1,492.982 ms, 7.36 s total -[ 2023-09-21 07:18:56 ] Completed train batch 1 discriminator loss 0.102 55.982 ms, 7.41 s total -[ 2023-09-21 07:18:56 ] Completed train batch 1 discriminator backward 79.054 ms, 7.49 s total -Epoch [51] Step [1/2], gen_loss: 0.494, disc_loss: 0.034 -[ 2023-09-21 07:18:56 ] Completed train batch 1 metrics update 4.894 ms, 7.50 s total -Saving checkpoint at epoch 51 train batch 1 -[ 2023-09-21 07:18:56 ] Completed saving temp checkpoint 423.821 ms, 7.92 s total -[ 2023-09-21 07:18:56 ] Completed replacing temp checkpoint with checkpoint 19.746 ms, 7.94 s total -Epoch [51] :: gen_loss: 0.492, disc_loss: 0.034 -[ 2023-09-21 07:18:57 ] Completed training generator for epoch 51 52.641 ms, 7.99 s total -[ 2023-09-21 07:18:57 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 51 from eval step 0 - -[ 2023-09-21 07:18:58 ] Completed eval batch 0 to device 1,112.959 ms, 1.11 s total -[ 2023-09-21 07:18:58 ] Completed eval batch 0 forward 25.001 ms, 1.14 s total -[ 2023-09-21 07:18:58 ] Completed eval batch 0 recons_loss 0.288 ms, 1.14 s total -[ 2023-09-21 07:18:58 ] Completed eval batch 0 metrics update 6.023 ms, 1.14 s total -Saving checkpoint at epoch 51 val batch 0 -[ 2023-09-21 07:18:58 ] Completed saving temp checkpoint 328.733 ms, 1.47 s total -[ 2023-09-21 07:18:58 ] Completed replacing temp checkpoint with checkpoint 20.929 ms, 1.49 s total -[ 2023-09-21 07:18:59 ] Completed eval batch 1 to device 516.116 ms, 2.01 s total -[ 2023-09-21 07:18:59 ] Completed eval batch 1 forward 40.689 ms, 2.05 s total -[ 2023-09-21 07:18:59 ] Completed eval batch 1 recons_loss 0.516 ms, 2.05 s total -[ 2023-09-21 07:18:59 ] Completed eval batch 1 metrics update 338.164 ms, 2.39 s total -Saving checkpoint at epoch 51 val batch 1 -Epoch 51 val loss: 0.0606 -[ 2023-09-21 07:18:59 ] Completed saving temp checkpoint 370.937 ms, 2.76 s total -[ 2023-09-21 07:18:59 ] Completed replacing temp checkpoint with checkpoint 20.192 ms, 2.78 s total -[ 2023-09-21 07:18:59 ] Completed evaluating generator for epoch 51 46.069 ms, 2.83 s total - - -EPOCH :: 52 - - -[ 2023-09-21 07:18:59 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 52 from training step 0 - -[ 2023-09-21 07:19:03 ] Completed train batch 0 to device 3,186.127 ms, 3.19 s total -[ 2023-09-21 07:19:03 ] Completed train batch 0 generator forward 40.662 ms, 3.23 s total -[ 2023-09-21 07:19:03 ] Completed train batch 0 generator loss: 1.502 66.561 ms, 3.29 s total -[ 2023-09-21 07:19:03 ] Completed train batch 0 generator backward 632.934 ms, 3.93 s total -[ 2023-09-21 07:19:03 ] Completed train batch 0 discriminator loss 0.106 57.209 ms, 3.98 s total -[ 2023-09-21 07:19:03 ] Completed train batch 0 discriminator backward 100.358 ms, 4.08 s total -Epoch [52] Step [0/2], gen_loss: 0.491, disc_loss: 0.036 -[ 2023-09-21 07:19:03 ] Completed train batch 0 metrics update 1.102 ms, 4.08 s total -Saving checkpoint at epoch 52 train batch 0 -[ 2023-09-21 07:19:04 ] Completed saving temp checkpoint 490.269 ms, 4.58 s total -[ 2023-09-21 07:19:04 ] Completed replacing temp checkpoint with checkpoint 21.697 ms, 4.60 s total -[ 2023-09-21 07:19:05 ] Completed train batch 1 to device 1,395.165 ms, 5.99 s total -[ 2023-09-21 07:19:05 ] Completed train batch 1 generator forward 40.023 ms, 6.03 s total -[ 2023-09-21 07:19:05 ] Completed train batch 1 generator loss: 1.489 63.631 ms, 6.10 s total -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 07:31:22 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 07:31:22 ] Completed importing Timer 0.025 ms, 0.00 s total -[ 2023-09-21 07:31:27 ] Completed importing everything else 4,582.563 ms, 4.58 s total -| distributed init (rank 1): env:// -| distributed init (rank 2): env:// -| distributed init (rank 3): env:// -| distributed init (rank 0): env:// -| distributed init (rank 4): env:// -| distributed init (rank 5): env:// -[ 2023-09-21 07:31:34 ] Completed preliminaries 7,361.249 ms, 11.94 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 07:31:34 ] Completed build datasets 10.783 ms, 11.95 s total -[ 2023-09-21 07:31:34 ] Completed build samplers 0.044 ms, 11.95 s total -[ 2023-09-21 07:31:34 ] Completed build dataloaders 0.141 ms, 11.95 s total -[ 2023-09-21 07:31:34 ] Completed generator to device 80.119 ms, 12.03 s total -[ 2023-09-21 07:31:34 ] Completed discriminator to device 7.039 ms, 12.04 s total -[ 2023-09-21 07:31:35 ] Completed loss functions 412.549 ms, 12.45 s total -[ 2023-09-21 07:31:35 ] Completed models prepped for distribution 122.574 ms, 12.58 s total -[ 2023-09-21 07:31:35 ] Completed optimizers 0.694 ms, 12.58 s total -[ 2023-09-21 07:31:35 ] Completed grad scalers 0.032 ms, 12.58 s total -[ 2023-09-21 07:31:35 ] Completed checkpoint retrieval 501.205 ms, 13.08 s total - - -EPOCH :: 52 - - -[ 2023-09-21 07:31:35 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 52 from training step 1 - -[ 2023-09-21 07:31:39 ] Completed train batch 1 to device 3,715.823 ms, 3.72 s total -[ 2023-09-21 07:31:40 ] Completed train batch 1 generator forward 1,082.362 ms, 4.80 s total -[ 2023-09-21 07:31:40 ] Completed train batch 1 generator loss: 1.516 73.703 ms, 4.87 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 07:31:41 ] Completed train batch 1 generator backward 1,041.506 ms, 5.91 s total -[ 2023-09-21 07:31:41 ] Completed train batch 1 discriminator loss 0.102 72.417 ms, 5.99 s total -[ 2023-09-21 07:31:41 ] Completed train batch 1 discriminator backward 67.522 ms, 6.05 s total -Epoch [52] Step [1/2], gen_loss: 0.486, disc_loss: 0.034 -[ 2023-09-21 07:31:41 ] Completed train batch 1 metrics update 5.455 ms, 6.06 s total -Saving checkpoint at epoch 52 train batch 1 -[ 2023-09-21 07:31:42 ] Completed saving temp checkpoint 427.516 ms, 6.49 s total -[ 2023-09-21 07:31:42 ] Completed replacing temp checkpoint with checkpoint 119.743 ms, 6.61 s total -Epoch [52] :: gen_loss: 0.488, disc_loss: 0.035 -[ 2023-09-21 07:31:42 ] Completed training generator for epoch 52 55.153 ms, 6.66 s total -[ 2023-09-21 07:31:42 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 52 from eval step 0 - -[ 2023-09-21 07:31:43 ] Completed eval batch 0 to device 1,082.955 ms, 1.08 s total -[ 2023-09-21 07:31:43 ] Completed eval batch 0 forward 25.241 ms, 1.11 s total -[ 2023-09-21 07:31:43 ] Completed eval batch 0 recons_loss 0.293 ms, 1.11 s total -[ 2023-09-21 07:31:43 ] Completed eval batch 0 metrics update 0.638 ms, 1.11 s total -Saving checkpoint at epoch 52 val batch 0 -[ 2023-09-21 07:31:43 ] Completed saving temp checkpoint 453.675 ms, 1.56 s total -[ 2023-09-21 07:31:43 ] Completed replacing temp checkpoint with checkpoint 26.976 ms, 1.59 s total -[ 2023-09-21 07:31:44 ] Completed eval batch 1 to device 346.263 ms, 1.94 s total -[ 2023-09-21 07:31:44 ] Completed eval batch 1 forward 40.800 ms, 1.98 s total -[ 2023-09-21 07:31:44 ] Completed eval batch 1 recons_loss 0.502 ms, 1.98 s total -[ 2023-09-21 07:31:44 ] Completed eval batch 1 metrics update 185.617 ms, 2.16 s total -Saving checkpoint at epoch 52 val batch 1 -Epoch 52 val loss: 0.0605 -[ 2023-09-21 07:31:44 ] Completed saving temp checkpoint 430.936 ms, 2.59 s total -[ 2023-09-21 07:31:45 ] Completed replacing temp checkpoint with checkpoint 29.670 ms, 2.62 s total -[ 2023-09-21 07:31:45 ] Completed evaluating generator for epoch 52 47.591 ms, 2.67 s total - - -EPOCH :: 53 - - -[ 2023-09-21 07:31:45 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 53 from training step 0 - -[ 2023-09-21 07:31:48 ] Completed train batch 0 to device 3,057.031 ms, 3.06 s total -[ 2023-09-21 07:31:48 ] Completed train batch 0 generator forward 41.932 ms, 3.10 s total -[ 2023-09-21 07:31:48 ] Completed train batch 0 generator loss: 1.452 64.831 ms, 3.16 s total -[ 2023-09-21 07:31:49 ] Completed train batch 0 generator backward 799.631 ms, 3.96 s total -[ 2023-09-21 07:31:49 ] Completed train batch 0 discriminator loss 0.097 56.349 ms, 4.02 s total -[ 2023-09-21 07:31:49 ] Completed train batch 0 discriminator backward 111.225 ms, 4.13 s total -Epoch [53] Step [0/2], gen_loss: 0.480, disc_loss: 0.032 -[ 2023-09-21 07:31:49 ] Completed train batch 0 metrics update 6.817 ms, 4.14 s total -Saving checkpoint at epoch 53 train batch 0 -[ 2023-09-21 07:31:49 ] Completed saving temp checkpoint 362.368 ms, 4.50 s total -[ 2023-09-21 07:31:49 ] Completed replacing temp checkpoint with checkpoint 25.178 ms, 4.53 s total -[ 2023-09-21 07:31:50 ] Completed train batch 1 to device 1,186.412 ms, 5.71 s total -[ 2023-09-21 07:31:50 ] Completed train batch 1 generator forward 39.419 ms, 5.75 s total -[ 2023-09-21 07:31:50 ] Completed train batch 1 generator loss: 1.461 63.212 ms, 5.81 s total -[ 2023-09-21 07:31:52 ] Completed train batch 1 generator backward 1,574.853 ms, 7.39 s total -[ 2023-09-21 07:31:52 ] Completed train batch 1 discriminator loss 0.077 56.092 ms, 7.45 s total -[ 2023-09-21 07:31:52 ] Completed train batch 1 discriminator backward 83.733 ms, 7.53 s total -Epoch [53] Step [1/2], gen_loss: 0.491, disc_loss: 0.027 -[ 2023-09-21 07:31:52 ] Completed train batch 1 metrics update 4.809 ms, 7.53 s total -Saving checkpoint at epoch 53 train batch 1 -[ 2023-09-21 07:31:52 ] Completed saving temp checkpoint 355.193 ms, 7.89 s total -[ 2023-09-21 07:31:52 ] Completed replacing temp checkpoint with checkpoint 26.752 ms, 7.92 s total -Epoch [53] :: gen_loss: 0.485, disc_loss: 0.029 -[ 2023-09-21 07:31:53 ] Completed training generator for epoch 53 50.006 ms, 7.97 s total -[ 2023-09-21 07:31:53 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 53 from eval step 0 - -[ 2023-09-21 07:31:54 ] Completed eval batch 0 to device 1,161.867 ms, 1.16 s total -[ 2023-09-21 07:31:54 ] Completed eval batch 0 forward 26.562 ms, 1.19 s total -[ 2023-09-21 07:31:54 ] Completed eval batch 0 recons_loss 0.282 ms, 1.19 s total -[ 2023-09-21 07:31:54 ] Completed eval batch 0 metrics update 21.130 ms, 1.21 s total -Saving checkpoint at epoch 53 val batch 0 -[ 2023-09-21 07:31:54 ] Completed saving temp checkpoint 354.099 ms, 1.56 s total -[ 2023-09-21 07:31:54 ] Completed replacing temp checkpoint with checkpoint 21.856 ms, 1.59 s total -[ 2023-09-21 07:31:55 ] Completed eval batch 1 to device 507.408 ms, 2.09 s total -[ 2023-09-21 07:31:55 ] Completed eval batch 1 forward 26.756 ms, 2.12 s total -[ 2023-09-21 07:31:55 ] Completed eval batch 1 recons_loss 0.339 ms, 2.12 s total -[ 2023-09-21 07:31:55 ] Completed eval batch 1 metrics update 344.941 ms, 2.47 s total -Saving checkpoint at epoch 53 val batch 1 -Epoch 53 val loss: 0.0632 -[ 2023-09-21 07:31:55 ] Completed saving temp checkpoint 423.668 ms, 2.89 s total -[ 2023-09-21 07:31:55 ] Completed replacing temp checkpoint with checkpoint 31.304 ms, 2.92 s total -[ 2023-09-21 07:31:56 ] Completed evaluating generator for epoch 53 47.242 ms, 2.97 s total - - -EPOCH :: 54 - - -[ 2023-09-21 07:31:56 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 54 from training step 0 - -[ 2023-09-21 07:31:59 ] Completed train batch 0 to device 3,236.233 ms, 3.24 s total -[ 2023-09-21 07:31:59 ] Completed train batch 0 generator forward 42.852 ms, 3.28 s total -[ 2023-09-21 07:31:59 ] Completed train batch 0 generator loss: 1.436 66.407 ms, 3.35 s total -[ 2023-09-21 07:31:59 ] Completed train batch 0 generator backward 597.975 ms, 3.94 s total -[ 2023-09-21 07:32:00 ] Completed train batch 0 discriminator loss 0.088 57.759 ms, 4.00 s total -[ 2023-09-21 07:32:00 ] Completed train batch 0 discriminator backward 100.365 ms, 4.10 s total -Epoch [54] Step [0/2], gen_loss: 0.473, disc_loss: 0.030 -[ 2023-09-21 07:32:00 ] Completed train batch 0 metrics update 6.141 ms, 4.11 s total -Saving checkpoint at epoch 54 train batch 0 -[ 2023-09-21 07:32:00 ] Completed saving temp checkpoint 466.110 ms, 4.57 s total -[ 2023-09-21 07:32:00 ] Completed replacing temp checkpoint with checkpoint 17.600 ms, 4.59 s total -[ 2023-09-21 07:32:02 ] Completed train batch 1 to device 1,562.122 ms, 6.15 s total -[ 2023-09-21 07:32:02 ] Completed train batch 1 generator forward 22.465 ms, 6.18 s total -[ 2023-09-21 07:32:02 ] Completed train batch 1 generator loss: 1.433 63.437 ms, 6.24 s total -[ 2023-09-21 07:32:03 ] Completed train batch 1 generator backward 1,351.680 ms, 7.59 s total -[ 2023-09-21 07:32:03 ] Completed train batch 1 discriminator loss 0.092 56.155 ms, 7.65 s total -[ 2023-09-21 07:32:03 ] Completed train batch 1 discriminator backward 84.751 ms, 7.73 s total -Epoch [54] Step [1/2], gen_loss: 0.482, disc_loss: 0.031 -[ 2023-09-21 07:32:03 ] Completed train batch 1 metrics update 5.064 ms, 7.74 s total -Saving checkpoint at epoch 54 train batch 1 -[ 2023-09-21 07:32:04 ] Completed saving temp checkpoint 476.878 ms, 8.21 s total -[ 2023-09-21 07:32:04 ] Completed replacing temp checkpoint with checkpoint 21.124 ms, 8.24 s total -Epoch [54] :: gen_loss: 0.478, disc_loss: 0.031 -[ 2023-09-21 07:32:04 ] Completed training generator for epoch 54 53.952 ms, 8.29 s total -[ 2023-09-21 07:32:04 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 54 from eval step 0 - -[ 2023-09-21 07:32:05 ] Completed eval batch 0 to device 1,174.192 ms, 1.17 s total -[ 2023-09-21 07:32:05 ] Completed eval batch 0 forward 32.634 ms, 1.21 s total -[ 2023-09-21 07:32:05 ] Completed eval batch 0 recons_loss 0.350 ms, 1.21 s total -[ 2023-09-21 07:32:05 ] Completed eval batch 0 metrics update 0.605 ms, 1.21 s total -Saving checkpoint at epoch 54 val batch 0 -[ 2023-09-21 07:32:05 ] Completed saving temp checkpoint 464.844 ms, 1.67 s total -[ 2023-09-21 07:32:06 ] Completed replacing temp checkpoint with checkpoint 27.932 ms, 1.70 s total -[ 2023-09-21 07:32:06 ] Completed eval batch 1 to device 360.109 ms, 2.06 s total -[ 2023-09-21 07:32:06 ] Completed eval batch 1 forward 27.371 ms, 2.09 s total -[ 2023-09-21 07:32:06 ] Completed eval batch 1 recons_loss 0.358 ms, 2.09 s total -[ 2023-09-21 07:32:06 ] Completed eval batch 1 metrics update 113.961 ms, 2.20 s total -Saving checkpoint at epoch 54 val batch 1 -Epoch 54 val loss: 0.0662 -[ 2023-09-21 07:32:06 ] Completed saving temp checkpoint 497.844 ms, 2.70 s total -[ 2023-09-21 07:32:07 ] Completed replacing temp checkpoint with checkpoint 28.102 ms, 2.73 s total -[ 2023-09-21 07:32:07 ] Completed evaluating generator for epoch 54 49.991 ms, 2.78 s total - - -EPOCH :: 55 - - -[ 2023-09-21 07:32:07 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 55 from training step 0 - -[ 2023-09-21 07:32:10 ] Completed train batch 0 to device 3,367.543 ms, 3.37 s total -[ 2023-09-21 07:32:10 ] Completed train batch 0 generator forward 30.456 ms, 3.40 s total -[ 2023-09-21 07:32:10 ] Completed train batch 0 generator loss: 1.432 67.141 ms, 3.47 s total -[ 2023-09-21 07:32:10 ] Completed train batch 0 generator backward 272.765 ms, 3.74 s total -[ 2023-09-21 07:32:10 ] Completed train batch 0 discriminator loss 0.108 57.625 ms, 3.80 s total -[ 2023-09-21 07:32:10 ] Completed train batch 0 discriminator backward 117.722 ms, 3.91 s total -Epoch [55] Step [0/2], gen_loss: 0.474, disc_loss: 0.035 -[ 2023-09-21 07:32:10 ] Completed train batch 0 metrics update 0.888 ms, 3.91 s total -Saving checkpoint at epoch 55 train batch 0 -[ 2023-09-21 07:32:11 ] Completed saving temp checkpoint 480.805 ms, 4.39 s total -[ 2023-09-21 07:32:11 ] Completed replacing temp checkpoint with checkpoint 32.019 ms, 4.43 s total -[ 2023-09-21 07:32:13 ] Completed train batch 1 to device 2,091.206 ms, 6.52 s total -[ 2023-09-21 07:32:13 ] Completed train batch 1 generator forward 40.803 ms, 6.56 s total -[ 2023-09-21 07:32:13 ] Completed train batch 1 generator loss: 1.413 63.515 ms, 6.62 s total -[ 2023-09-21 07:32:14 ] Completed train batch 1 generator backward 602.175 ms, 7.22 s total -[ 2023-09-21 07:32:14 ] Completed train batch 1 discriminator loss 0.102 63.794 ms, 7.29 s total -[ 2023-09-21 07:32:14 ] Completed train batch 1 discriminator backward 72.864 ms, 7.36 s total -Epoch [55] Step [1/2], gen_loss: 0.484, disc_loss: 0.034 -[ 2023-09-21 07:32:14 ] Completed train batch 1 metrics update 4.747 ms, 7.37 s total -Saving checkpoint at epoch 55 train batch 1 -[ 2023-09-21 07:32:14 ] Completed saving temp checkpoint 494.559 ms, 7.86 s total -[ 2023-09-21 07:32:14 ] Completed replacing temp checkpoint with checkpoint 29.797 ms, 7.89 s total -Epoch [55] :: gen_loss: 0.479, disc_loss: 0.035 -[ 2023-09-21 07:32:15 ] Completed training generator for epoch 55 52.486 ms, 7.94 s total -[ 2023-09-21 07:32:15 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 55 from eval step 0 - -[ 2023-09-21 07:32:16 ] Completed eval batch 0 to device 1,179.965 ms, 1.18 s total -[ 2023-09-21 07:32:16 ] Completed eval batch 0 forward 30.921 ms, 1.21 s total -[ 2023-09-21 07:32:16 ] Completed eval batch 0 recons_loss 0.356 ms, 1.21 s total -[ 2023-09-21 07:32:16 ] Completed eval batch 0 metrics update 0.714 ms, 1.21 s total -Saving checkpoint at epoch 55 val batch 0 -[ 2023-09-21 07:32:16 ] Completed saving temp checkpoint 450.441 ms, 1.66 s total -[ 2023-09-21 07:32:16 ] Completed replacing temp checkpoint with checkpoint 25.264 ms, 1.69 s total -[ 2023-09-21 07:32:17 ] Completed eval batch 1 to device 429.650 ms, 2.12 s total -[ 2023-09-21 07:32:17 ] Completed eval batch 1 forward 21.403 ms, 2.14 s total -[ 2023-09-21 07:32:17 ] Completed eval batch 1 recons_loss 0.333 ms, 2.14 s total -[ 2023-09-21 07:32:17 ] Completed eval batch 1 metrics update 66.810 ms, 2.21 s total -Saving checkpoint at epoch 55 val batch 1 -Epoch 55 val loss: 0.0626 -[ 2023-09-21 07:32:17 ] Completed saving temp checkpoint 465.885 ms, 2.67 s total -[ 2023-09-21 07:32:17 ] Completed replacing temp checkpoint with checkpoint 22.262 ms, 2.69 s total -[ 2023-09-21 07:32:17 ] Completed evaluating generator for epoch 55 47.652 ms, 2.74 s total - - -EPOCH :: 56 - - -[ 2023-09-21 07:32:17 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 56 from training step 0 - -[ 2023-09-21 07:32:21 ] Completed train batch 0 to device 3,291.007 ms, 3.29 s total -[ 2023-09-21 07:32:21 ] Completed train batch 0 generator forward 26.777 ms, 3.32 s total -[ 2023-09-21 07:32:21 ] Completed train batch 0 generator loss: 1.442 66.833 ms, 3.38 s total -[ 2023-09-21 07:32:21 ] Completed train batch 0 generator backward 439.205 ms, 3.82 s total -[ 2023-09-21 07:32:21 ] Completed train batch 0 discriminator loss 0.096 79.973 ms, 3.90 s total -[ 2023-09-21 07:32:21 ] Completed train batch 0 discriminator backward 75.560 ms, 3.98 s total -Epoch [56] Step [0/2], gen_loss: 0.480, disc_loss: 0.032 -[ 2023-09-21 07:32:21 ] Completed train batch 0 metrics update 3.804 ms, 3.98 s total -Saving checkpoint at epoch 56 train batch 0 -[ 2023-09-21 07:32:22 ] Completed saving temp checkpoint 492.375 ms, 4.48 s total -[ 2023-09-21 07:32:22 ] Completed replacing temp checkpoint with checkpoint 18.551 ms, 4.49 s total -[ 2023-09-21 07:32:24 ] Completed train batch 1 to device 1,837.028 ms, 6.33 s total -[ 2023-09-21 07:32:24 ] Completed train batch 1 generator forward 23.946 ms, 6.36 s total -[ 2023-09-21 07:32:24 ] Completed train batch 1 generator loss: 1.440 65.893 ms, 6.42 s total -[ 2023-09-21 07:32:25 ] Completed train batch 1 generator backward 934.150 ms, 7.36 s total -[ 2023-09-21 07:32:25 ] Completed train batch 1 discriminator loss 0.087 56.890 ms, 7.41 s total -[ 2023-09-21 07:32:25 ] Completed train batch 1 discriminator backward 79.433 ms, 7.49 s total -Epoch [56] Step [1/2], gen_loss: 0.480, disc_loss: 0.030 -[ 2023-09-21 07:32:25 ] Completed train batch 1 metrics update 0.494 ms, 7.49 s total -Saving checkpoint at epoch 56 train batch 1 -[ 2023-09-21 07:32:25 ] Completed saving temp checkpoint 455.991 ms, 7.95 s total -[ 2023-09-21 07:32:25 ] Completed replacing temp checkpoint with checkpoint 20.489 ms, 7.97 s total -Epoch [56] :: gen_loss: 0.480, disc_loss: 0.031 -[ 2023-09-21 07:32:25 ] Completed training generator for epoch 56 51.575 ms, 8.02 s total -[ 2023-09-21 07:32:25 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 56 from eval step 0 - -[ 2023-09-21 07:32:26 ] Completed eval batch 0 to device 1,175.478 ms, 1.18 s total -[ 2023-09-21 07:32:26 ] Completed eval batch 0 forward 29.751 ms, 1.21 s total -[ 2023-09-21 07:32:26 ] Completed eval batch 0 recons_loss 0.282 ms, 1.21 s total -[ 2023-09-21 07:32:26 ] Completed eval batch 0 metrics update 0.579 ms, 1.21 s total -Saving checkpoint at epoch 56 val batch 0 -[ 2023-09-21 07:32:27 ] Completed saving temp checkpoint 445.294 ms, 1.65 s total -[ 2023-09-21 07:32:27 ] Completed replacing temp checkpoint with checkpoint 26.569 ms, 1.68 s total -[ 2023-09-21 07:32:28 ] Completed eval batch 1 to device 544.718 ms, 2.22 s total -[ 2023-09-21 07:32:28 ] Completed eval batch 1 forward 39.835 ms, 2.26 s total -[ 2023-09-21 07:32:28 ] Completed eval batch 1 recons_loss 0.430 ms, 2.26 s total -[ 2023-09-21 07:32:28 ] Completed eval batch 1 metrics update 0.394 ms, 2.26 s total -Saving checkpoint at epoch 56 val batch 1 -Epoch 56 val loss: 0.0629 -[ 2023-09-21 07:32:28 ] Completed saving temp checkpoint 470.549 ms, 2.73 s total -[ 2023-09-21 07:32:28 ] Completed replacing temp checkpoint with checkpoint 22.174 ms, 2.76 s total -[ 2023-09-21 07:32:28 ] Completed evaluating generator for epoch 56 50.576 ms, 2.81 s total - - -EPOCH :: 57 - - -[ 2023-09-21 07:32:28 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 57 from training step 0 - -[ 2023-09-21 07:32:31 ] Completed train batch 0 to device 3,220.045 ms, 3.22 s total -[ 2023-09-21 07:32:31 ] Completed train batch 0 generator forward 33.912 ms, 3.25 s total -[ 2023-09-21 07:32:31 ] Completed train batch 0 generator loss: 1.455 69.046 ms, 3.32 s total -[ 2023-09-21 07:32:32 ] Completed train batch 0 generator backward 439.825 ms, 3.76 s total -[ 2023-09-21 07:32:32 ] Completed train batch 0 discriminator loss 0.078 57.996 ms, 3.82 s total -[ 2023-09-21 07:32:32 ] Completed train batch 0 discriminator backward 105.143 ms, 3.93 s total -Epoch [57] Step [0/2], gen_loss: 0.474, disc_loss: 0.027 -[ 2023-09-21 07:32:32 ] Completed train batch 0 metrics update 22.851 ms, 3.95 s total -Saving checkpoint at epoch 57 train batch 0 -[ 2023-09-21 07:32:32 ] Completed saving temp checkpoint 422.642 ms, 4.37 s total -[ 2023-09-21 07:32:32 ] Completed replacing temp checkpoint with checkpoint 31.574 ms, 4.40 s total -[ 2023-09-21 07:32:34 ] Completed train batch 1 to device 1,794.883 ms, 6.20 s total -[ 2023-09-21 07:32:34 ] Completed train batch 1 generator forward 39.268 ms, 6.24 s total -[ 2023-09-21 07:32:34 ] Completed train batch 1 generator loss: 1.446 63.523 ms, 6.30 s total -[ 2023-09-21 07:32:35 ] Completed train batch 1 generator backward 891.363 ms, 7.19 s total -[ 2023-09-21 07:32:35 ] Completed train batch 1 discriminator loss 0.075 56.258 ms, 7.25 s total -[ 2023-09-21 07:32:35 ] Completed train batch 1 discriminator backward 73.408 ms, 7.32 s total -Epoch [57] Step [1/2], gen_loss: 0.479, disc_loss: 0.024 -[ 2023-09-21 07:32:35 ] Completed train batch 1 metrics update 5.492 ms, 7.33 s total -Saving checkpoint at epoch 57 train batch 1 -[ 2023-09-21 07:32:36 ] Completed saving temp checkpoint 450.072 ms, 7.78 s total -[ 2023-09-21 07:32:36 ] Completed replacing temp checkpoint with checkpoint 16.748 ms, 7.79 s total -Epoch [57] :: gen_loss: 0.476, disc_loss: 0.026 -[ 2023-09-21 07:32:36 ] Completed training generator for epoch 57 46.521 ms, 7.84 s total -[ 2023-09-21 07:32:36 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 57 from eval step 0 - -[ 2023-09-21 07:32:37 ] Completed eval batch 0 to device 1,157.684 ms, 1.16 s total -[ 2023-09-21 07:32:37 ] Completed eval batch 0 forward 42.979 ms, 1.20 s total -[ 2023-09-21 07:32:37 ] Completed eval batch 0 recons_loss 0.305 ms, 1.20 s total -[ 2023-09-21 07:32:37 ] Completed eval batch 0 metrics update 0.607 ms, 1.20 s total -Saving checkpoint at epoch 57 val batch 0 -[ 2023-09-21 07:32:37 ] Completed saving temp checkpoint 361.704 ms, 1.56 s total -[ 2023-09-21 07:32:38 ] Completed replacing temp checkpoint with checkpoint 23.432 ms, 1.59 s total -[ 2023-09-21 07:32:38 ] Completed eval batch 1 to device 477.549 ms, 2.06 s total -[ 2023-09-21 07:32:38 ] Completed eval batch 1 forward 38.800 ms, 2.10 s total -[ 2023-09-21 07:32:38 ] Completed eval batch 1 recons_loss 0.337 ms, 2.10 s total -[ 2023-09-21 07:32:38 ] Completed eval batch 1 metrics update 118.013 ms, 2.22 s total -Saving checkpoint at epoch 57 val batch 1 -Epoch 57 val loss: 0.0653 -[ 2023-09-21 07:32:39 ] Completed saving temp checkpoint 404.053 ms, 2.63 s total -[ 2023-09-21 07:32:39 ] Completed replacing temp checkpoint with checkpoint 19.728 ms, 2.65 s total -[ 2023-09-21 07:32:39 ] Completed evaluating generator for epoch 57 44.980 ms, 2.69 s total - - -EPOCH :: 58 - - -[ 2023-09-21 07:32:39 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 58 from training step 0 - -[ 2023-09-21 07:32:42 ] Completed train batch 0 to device 3,167.107 ms, 3.17 s total -[ 2023-09-21 07:32:42 ] Completed train batch 0 generator forward 42.651 ms, 3.21 s total -[ 2023-09-21 07:32:42 ] Completed train batch 0 generator loss: 1.480 66.479 ms, 3.28 s total -[ 2023-09-21 07:32:43 ] Completed train batch 0 generator backward 644.372 ms, 3.92 s total -[ 2023-09-21 07:32:43 ] Completed train batch 0 discriminator loss 0.068 58.494 ms, 3.98 s total -[ 2023-09-21 07:32:43 ] Completed train batch 0 discriminator backward 106.858 ms, 4.09 s total -Epoch [58] Step [0/2], gen_loss: 0.488, disc_loss: 0.022 -[ 2023-09-21 07:32:43 ] Completed train batch 0 metrics update 5.353 ms, 4.09 s total -Saving checkpoint at epoch 58 train batch 0 -[ 2023-09-21 07:32:43 ] Completed saving temp checkpoint 494.629 ms, 4.59 s total -[ 2023-09-21 07:32:43 ] Completed replacing temp checkpoint with checkpoint 27.751 ms, 4.61 s total -[ 2023-09-21 07:32:45 ] Completed train batch 1 to device 1,418.980 ms, 6.03 s total -[ 2023-09-21 07:32:45 ] Completed train batch 1 generator forward 26.940 ms, 6.06 s total -[ 2023-09-21 07:32:45 ] Completed train batch 1 generator loss: 1.406 63.258 ms, 6.12 s total -[ 2023-09-21 07:32:46 ] Completed train batch 1 generator backward 1,285.841 ms, 7.41 s total -[ 2023-09-21 07:32:46 ] Completed train batch 1 discriminator loss 0.080 55.964 ms, 7.46 s total -[ 2023-09-21 07:32:46 ] Completed train batch 1 discriminator backward 80.841 ms, 7.55 s total -Epoch [58] Step [1/2], gen_loss: 0.470, disc_loss: 0.027 -[ 2023-09-21 07:32:46 ] Completed train batch 1 metrics update 1.650 ms, 7.55 s total -Saving checkpoint at epoch 58 train batch 1 -[ 2023-09-21 07:32:47 ] Completed saving temp checkpoint 447.198 ms, 7.99 s total -[ 2023-09-21 07:32:47 ] Completed replacing temp checkpoint with checkpoint 22.292 ms, 8.02 s total -Epoch [58] :: gen_loss: 0.479, disc_loss: 0.024 -[ 2023-09-21 07:32:47 ] Completed training generator for epoch 58 51.166 ms, 8.07 s total -[ 2023-09-21 07:32:47 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 58 from eval step 0 - -[ 2023-09-21 07:32:48 ] Completed eval batch 0 to device 1,147.716 ms, 1.15 s total -[ 2023-09-21 07:32:48 ] Completed eval batch 0 forward 24.761 ms, 1.17 s total -[ 2023-09-21 07:32:48 ] Completed eval batch 0 recons_loss 0.324 ms, 1.17 s total -[ 2023-09-21 07:32:48 ] Completed eval batch 0 metrics update 0.565 ms, 1.17 s total -Saving checkpoint at epoch 58 val batch 0 -[ 2023-09-21 07:32:48 ] Completed saving temp checkpoint 456.028 ms, 1.63 s total -[ 2023-09-21 07:32:48 ] Completed replacing temp checkpoint with checkpoint 37.985 ms, 1.67 s total -[ 2023-09-21 07:32:49 ] Completed eval batch 1 to device 425.747 ms, 2.09 s total -[ 2023-09-21 07:32:49 ] Completed eval batch 1 forward 37.867 ms, 2.13 s total -[ 2023-09-21 07:32:49 ] Completed eval batch 1 recons_loss 0.333 ms, 2.13 s total -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 07:45:07 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 07:45:07 ] Completed importing Timer 0.023 ms, 0.00 s total -[ 2023-09-21 07:45:11 ] Completed importing everything else 4,701.407 ms, 4.70 s total -| distributed init (rank 0): env:// -| distributed init (rank 4): env:// -| distributed init (rank 1): env:// -| distributed init (rank 3): env:// -| distributed init (rank 5): env:// -| distributed init (rank 2): env:// -[ 2023-09-21 07:45:19 ] Completed preliminaries 7,421.430 ms, 12.12 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 07:45:19 ] Completed build datasets 11.123 ms, 12.13 s total -[ 2023-09-21 07:45:19 ] Completed build samplers 0.044 ms, 12.13 s total -[ 2023-09-21 07:45:19 ] Completed build dataloaders 0.135 ms, 12.13 s total -[ 2023-09-21 07:45:19 ] Completed generator to device 76.535 ms, 12.21 s total -[ 2023-09-21 07:45:19 ] Completed discriminator to device 7.310 ms, 12.22 s total -[ 2023-09-21 07:45:19 ] Completed loss functions 441.427 ms, 12.66 s total -[ 2023-09-21 07:45:19 ] Completed models prepped for distribution 92.613 ms, 12.75 s total -[ 2023-09-21 07:45:19 ] Completed optimizers 0.712 ms, 12.75 s total -[ 2023-09-21 07:45:19 ] Completed grad scalers 0.032 ms, 12.75 s total -[ 2023-09-21 07:45:20 ] Completed checkpoint retrieval 482.754 ms, 13.24 s total - - -EPOCH :: 58 - - -[ 2023-09-21 07:45:20 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 58 from training step 2 - -Epoch [58] :: gen_loss: 0.479, disc_loss: 0.024 -[ 2023-09-21 07:45:20 ] Completed training generator for epoch 58 134.396 ms, 0.13 s total -[ 2023-09-21 07:45:20 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 58 from eval step 1 - -[ 2023-09-21 07:45:21 ] Completed eval batch 1 to device 1,160.486 ms, 1.16 s total -[ 2023-09-21 07:45:22 ] Completed eval batch 1 forward 1,185.613 ms, 2.35 s total -[ 2023-09-21 07:45:22 ] Completed eval batch 1 recons_loss 0.266 ms, 2.35 s total -[ 2023-09-21 07:45:23 ] Completed eval batch 1 metrics update 686.940 ms, 3.03 s total -Saving checkpoint at epoch 58 val batch 1 -Epoch 58 val loss: 0.0666 -[ 2023-09-21 07:45:24 ] Completed saving temp checkpoint 458.228 ms, 3.49 s total -[ 2023-09-21 07:45:24 ] Completed replacing temp checkpoint with checkpoint 121.169 ms, 3.61 s total -[ 2023-09-21 07:45:24 ] Completed evaluating generator for epoch 58 41.033 ms, 3.65 s total - - -EPOCH :: 59 - - -[ 2023-09-21 07:45:24 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 59 from training step 0 - -[ 2023-09-21 07:45:27 ] Completed train batch 0 to device 2,990.165 ms, 2.99 s total -[ 2023-09-21 07:45:27 ] Completed train batch 0 generator forward 44.006 ms, 3.03 s total -[ 2023-09-21 07:45:27 ] Completed train batch 0 generator loss: 1.461 87.789 ms, 3.12 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 07:45:28 ] Completed train batch 0 generator backward 830.807 ms, 3.95 s total -[ 2023-09-21 07:45:28 ] Completed train batch 0 discriminator loss 0.078 59.709 ms, 4.01 s total -[ 2023-09-21 07:45:28 ] Completed train batch 0 discriminator backward 105.937 ms, 4.12 s total -Epoch [59] Step [0/2], gen_loss: 0.480, disc_loss: 0.027 -[ 2023-09-21 07:45:28 ] Completed train batch 0 metrics update 4.922 ms, 4.12 s total -Saving checkpoint at epoch 59 train batch 0 -[ 2023-09-21 07:45:28 ] Completed saving temp checkpoint 452.953 ms, 4.58 s total -[ 2023-09-21 07:45:28 ] Completed replacing temp checkpoint with checkpoint 20.875 ms, 4.60 s total -[ 2023-09-21 07:45:29 ] Completed train batch 1 to device 1,228.240 ms, 5.83 s total -[ 2023-09-21 07:45:30 ] Completed train batch 1 generator forward 39.438 ms, 5.86 s total -[ 2023-09-21 07:45:30 ] Completed train batch 1 generator loss: 1.463 66.931 ms, 5.93 s total -[ 2023-09-21 07:45:31 ] Completed train batch 1 generator backward 1,537.025 ms, 7.47 s total -[ 2023-09-21 07:45:31 ] Completed train batch 1 discriminator loss 0.083 56.339 ms, 7.53 s total -[ 2023-09-21 07:45:31 ] Completed train batch 1 discriminator backward 81.416 ms, 7.61 s total -Epoch [59] Step [1/2], gen_loss: 0.488, disc_loss: 0.028 -[ 2023-09-21 07:45:31 ] Completed train batch 1 metrics update 1.985 ms, 7.61 s total -Saving checkpoint at epoch 59 train batch 1 -[ 2023-09-21 07:45:32 ] Completed saving temp checkpoint 416.804 ms, 8.03 s total -[ 2023-09-21 07:45:32 ] Completed replacing temp checkpoint with checkpoint 18.221 ms, 8.04 s total -Epoch [59] :: gen_loss: 0.484, disc_loss: 0.027 -[ 2023-09-21 07:45:32 ] Completed training generator for epoch 59 55.324 ms, 8.10 s total -[ 2023-09-21 07:45:32 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 59 from eval step 0 - -[ 2023-09-21 07:45:33 ] Completed eval batch 0 to device 1,058.731 ms, 1.06 s total -[ 2023-09-21 07:45:33 ] Completed eval batch 0 forward 26.831 ms, 1.09 s total -[ 2023-09-21 07:45:33 ] Completed eval batch 0 recons_loss 0.240 ms, 1.09 s total -[ 2023-09-21 07:45:33 ] Completed eval batch 0 metrics update 88.189 ms, 1.17 s total -Saving checkpoint at epoch 59 val batch 0 -[ 2023-09-21 07:45:33 ] Completed saving temp checkpoint 545.752 ms, 1.72 s total -[ 2023-09-21 07:45:34 ] Completed replacing temp checkpoint with checkpoint 34.526 ms, 1.75 s total -[ 2023-09-21 07:45:34 ] Completed eval batch 1 to device 205.842 ms, 1.96 s total -[ 2023-09-21 07:45:34 ] Completed eval batch 1 forward 21.535 ms, 1.98 s total -[ 2023-09-21 07:45:34 ] Completed eval batch 1 recons_loss 0.399 ms, 1.98 s total -[ 2023-09-21 07:45:34 ] Completed eval batch 1 metrics update 446.929 ms, 2.43 s total -Saving checkpoint at epoch 59 val batch 1 -Epoch 59 val loss: 0.0712 -[ 2023-09-21 07:45:35 ] Completed saving temp checkpoint 808.282 ms, 3.24 s total -[ 2023-09-21 07:45:35 ] Completed replacing temp checkpoint with checkpoint 39.370 ms, 3.28 s total -[ 2023-09-21 07:45:35 ] Completed evaluating generator for epoch 59 51.388 ms, 3.33 s total - - -EPOCH :: 60 - - -[ 2023-09-21 07:45:35 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 60 from training step 0 - -[ 2023-09-21 07:45:38 ] Completed train batch 0 to device 3,226.665 ms, 3.23 s total -[ 2023-09-21 07:45:38 ] Completed train batch 0 generator forward 32.167 ms, 3.26 s total -[ 2023-09-21 07:45:38 ] Completed train batch 0 generator loss: 1.462 66.825 ms, 3.33 s total -[ 2023-09-21 07:45:39 ] Completed train batch 0 generator backward 217.508 ms, 3.54 s total -[ 2023-09-21 07:45:39 ] Completed train batch 0 discriminator loss 0.079 57.793 ms, 3.60 s total -[ 2023-09-21 07:45:39 ] Completed train batch 0 discriminator backward 104.056 ms, 3.71 s total -Epoch [60] Step [0/2], gen_loss: 0.478, disc_loss: 0.027 -[ 2023-09-21 07:45:39 ] Completed train batch 0 metrics update 5.776 ms, 3.71 s total -Saving checkpoint at epoch 60 train batch 0 -[ 2023-09-21 07:45:39 ] Completed saving temp checkpoint 543.796 ms, 4.25 s total -[ 2023-09-21 07:45:39 ] Completed replacing temp checkpoint with checkpoint 34.551 ms, 4.29 s total -[ 2023-09-21 07:45:41 ] Completed train batch 1 to device 1,707.821 ms, 6.00 s total -[ 2023-09-21 07:45:41 ] Completed train batch 1 generator forward 22.338 ms, 6.02 s total -[ 2023-09-21 07:45:41 ] Completed train batch 1 generator loss: 1.384 63.586 ms, 6.08 s total -[ 2023-09-21 07:45:42 ] Completed train batch 1 generator backward 1,107.085 ms, 7.19 s total -[ 2023-09-21 07:45:42 ] Completed train batch 1 discriminator loss 0.079 56.983 ms, 7.25 s total -[ 2023-09-21 07:45:42 ] Completed train batch 1 discriminator backward 72.179 ms, 7.32 s total -Epoch [60] Step [1/2], gen_loss: 0.474, disc_loss: 0.027 -[ 2023-09-21 07:45:42 ] Completed train batch 1 metrics update 5.178 ms, 7.32 s total -Saving checkpoint at epoch 60 train batch 1 -[ 2023-09-21 07:45:43 ] Completed saving temp checkpoint 528.335 ms, 7.85 s total -[ 2023-09-21 07:45:43 ] Completed replacing temp checkpoint with checkpoint 28.988 ms, 7.88 s total -Epoch [60] :: gen_loss: 0.476, disc_loss: 0.027 -[ 2023-09-21 07:45:43 ] Completed training generator for epoch 60 64.573 ms, 7.95 s total -[ 2023-09-21 07:45:43 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 60 from eval step 0 - -[ 2023-09-21 07:45:44 ] Completed eval batch 0 to device 1,140.787 ms, 1.14 s total -[ 2023-09-21 07:45:44 ] Completed eval batch 0 forward 25.814 ms, 1.17 s total -[ 2023-09-21 07:45:44 ] Completed eval batch 0 recons_loss 0.354 ms, 1.17 s total -[ 2023-09-21 07:45:44 ] Completed eval batch 0 metrics update 0.795 ms, 1.17 s total -Saving checkpoint at epoch 60 val batch 0 -[ 2023-09-21 07:45:45 ] Completed saving temp checkpoint 508.877 ms, 1.68 s total -[ 2023-09-21 07:45:45 ] Completed replacing temp checkpoint with checkpoint 22.637 ms, 1.70 s total -[ 2023-09-21 07:45:45 ] Completed eval batch 1 to device 339.669 ms, 2.04 s total -[ 2023-09-21 07:45:45 ] Completed eval batch 1 forward 39.374 ms, 2.08 s total -[ 2023-09-21 07:45:45 ] Completed eval batch 1 recons_loss 0.342 ms, 2.08 s total -[ 2023-09-21 07:45:45 ] Completed eval batch 1 metrics update 44.845 ms, 2.12 s total -Saving checkpoint at epoch 60 val batch 1 -Epoch 60 val loss: 0.0698 -[ 2023-09-21 07:45:46 ] Completed saving temp checkpoint 542.991 ms, 2.67 s total -[ 2023-09-21 07:45:46 ] Completed replacing temp checkpoint with checkpoint 20.076 ms, 2.69 s total -[ 2023-09-21 07:45:46 ] Completed evaluating generator for epoch 60 71.870 ms, 2.76 s total - - -EPOCH :: 61 - - -[ 2023-09-21 07:45:46 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 61 from training step 0 - -[ 2023-09-21 07:45:49 ] Completed train batch 0 to device 3,104.430 ms, 3.10 s total -[ 2023-09-21 07:45:49 ] Completed train batch 0 generator forward 26.351 ms, 3.13 s total -[ 2023-09-21 07:45:49 ] Completed train batch 0 generator loss: 1.411 66.235 ms, 3.20 s total -[ 2023-09-21 07:45:50 ] Completed train batch 0 generator backward 562.204 ms, 3.76 s total -[ 2023-09-21 07:45:50 ] Completed train batch 0 discriminator loss 0.080 58.874 ms, 3.82 s total -[ 2023-09-21 07:45:50 ] Completed train batch 0 discriminator backward 78.518 ms, 3.90 s total -Epoch [61] Step [0/2], gen_loss: 0.474, disc_loss: 0.026 -[ 2023-09-21 07:45:50 ] Completed train batch 0 metrics update 8.176 ms, 3.90 s total -Saving checkpoint at epoch 61 train batch 0 -[ 2023-09-21 07:45:50 ] Completed saving temp checkpoint 499.624 ms, 4.40 s total -[ 2023-09-21 07:45:50 ] Completed replacing temp checkpoint with checkpoint 33.839 ms, 4.44 s total -[ 2023-09-21 07:45:52 ] Completed train batch 1 to device 1,878.233 ms, 6.32 s total -[ 2023-09-21 07:45:52 ] Completed train batch 1 generator forward 40.009 ms, 6.36 s total -[ 2023-09-21 07:45:52 ] Completed train batch 1 generator loss: 1.403 63.335 ms, 6.42 s total -[ 2023-09-21 07:45:53 ] Completed train batch 1 generator backward 787.777 ms, 7.21 s total -[ 2023-09-21 07:45:53 ] Completed train batch 1 discriminator loss 0.074 57.788 ms, 7.27 s total -[ 2023-09-21 07:45:53 ] Completed train batch 1 discriminator backward 63.063 ms, 7.33 s total -Epoch [61] Step [1/2], gen_loss: 0.471, disc_loss: 0.025 -[ 2023-09-21 07:45:53 ] Completed train batch 1 metrics update 3.695 ms, 7.33 s total -Saving checkpoint at epoch 61 train batch 1 -[ 2023-09-21 07:45:54 ] Completed saving temp checkpoint 546.667 ms, 7.88 s total -[ 2023-09-21 07:45:54 ] Completed replacing temp checkpoint with checkpoint 22.648 ms, 7.90 s total -Epoch [61] :: gen_loss: 0.473, disc_loss: 0.025 -[ 2023-09-21 07:45:54 ] Completed training generator for epoch 61 60.812 ms, 7.96 s total -[ 2023-09-21 07:45:54 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 61 from eval step 0 - -[ 2023-09-21 07:45:55 ] Completed eval batch 0 to device 1,134.550 ms, 1.13 s total -[ 2023-09-21 07:45:55 ] Completed eval batch 0 forward 25.163 ms, 1.16 s total -[ 2023-09-21 07:45:55 ] Completed eval batch 0 recons_loss 0.338 ms, 1.16 s total -[ 2023-09-21 07:45:55 ] Completed eval batch 0 metrics update 0.724 ms, 1.16 s total -Saving checkpoint at epoch 61 val batch 0 -[ 2023-09-21 07:45:55 ] Completed saving temp checkpoint 502.528 ms, 1.66 s total -[ 2023-09-21 07:45:55 ] Completed replacing temp checkpoint with checkpoint 35.492 ms, 1.70 s total -[ 2023-09-21 07:45:56 ] Completed eval batch 1 to device 318.622 ms, 2.02 s total -[ 2023-09-21 07:45:56 ] Completed eval batch 1 forward 29.398 ms, 2.05 s total -[ 2023-09-21 07:45:56 ] Completed eval batch 1 recons_loss 0.340 ms, 2.05 s total -[ 2023-09-21 07:45:56 ] Completed eval batch 1 metrics update 144.972 ms, 2.19 s total -Saving checkpoint at epoch 61 val batch 1 -Epoch 61 val loss: 0.0690 -[ 2023-09-21 07:45:57 ] Completed saving temp checkpoint 612.369 ms, 2.80 s total -[ 2023-09-21 07:45:57 ] Completed replacing temp checkpoint with checkpoint 44.678 ms, 2.85 s total -[ 2023-09-21 07:45:57 ] Completed evaluating generator for epoch 61 53.910 ms, 2.90 s total - - -EPOCH :: 62 - - -[ 2023-09-21 07:45:57 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 62 from training step 0 - -[ 2023-09-21 07:46:00 ] Completed train batch 0 to device 3,155.076 ms, 3.16 s total -[ 2023-09-21 07:46:00 ] Completed train batch 0 generator forward 31.637 ms, 3.19 s total -[ 2023-09-21 07:46:00 ] Completed train batch 0 generator loss: 1.414 67.051 ms, 3.25 s total -[ 2023-09-21 07:46:00 ] Completed train batch 0 generator backward 477.182 ms, 3.73 s total -[ 2023-09-21 07:46:00 ] Completed train batch 0 discriminator loss 0.071 58.203 ms, 3.79 s total -[ 2023-09-21 07:46:01 ] Completed train batch 0 discriminator backward 105.591 ms, 3.89 s total -Epoch [62] Step [0/2], gen_loss: 0.473, disc_loss: 0.023 -[ 2023-09-21 07:46:01 ] Completed train batch 0 metrics update 5.719 ms, 3.90 s total -Saving checkpoint at epoch 62 train batch 0 -[ 2023-09-21 07:46:01 ] Completed saving temp checkpoint 786.041 ms, 4.69 s total -[ 2023-09-21 07:46:01 ] Completed replacing temp checkpoint with checkpoint 49.905 ms, 4.74 s total -[ 2023-09-21 07:46:03 ] Completed train batch 1 to device 1,574.301 ms, 6.31 s total -[ 2023-09-21 07:46:03 ] Completed train batch 1 generator forward 40.052 ms, 6.35 s total -[ 2023-09-21 07:46:03 ] Completed train batch 1 generator loss: 1.415 63.315 ms, 6.41 s total -[ 2023-09-21 07:46:04 ] Completed train batch 1 generator backward 685.567 ms, 7.10 s total -[ 2023-09-21 07:46:04 ] Completed train batch 1 discriminator loss 0.070 56.231 ms, 7.16 s total -[ 2023-09-21 07:46:04 ] Completed train batch 1 discriminator backward 80.251 ms, 7.24 s total -Epoch [62] Step [1/2], gen_loss: 0.472, disc_loss: 0.024 -[ 2023-09-21 07:46:04 ] Completed train batch 1 metrics update 5.489 ms, 7.24 s total -Saving checkpoint at epoch 62 train batch 1 -[ 2023-09-21 07:46:06 ] Completed saving temp checkpoint 1,710.586 ms, 8.95 s total -[ 2023-09-21 07:46:06 ] Completed replacing temp checkpoint with checkpoint 55.588 ms, 9.01 s total -Epoch [62] :: gen_loss: 0.472, disc_loss: 0.024 -[ 2023-09-21 07:46:06 ] Completed training generator for epoch 62 74.363 ms, 9.08 s total -[ 2023-09-21 07:46:06 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 62 from eval step 0 - -[ 2023-09-21 07:46:07 ] Completed eval batch 0 to device 989.026 ms, 0.99 s total -[ 2023-09-21 07:46:07 ] Completed eval batch 0 forward 42.185 ms, 1.03 s total -[ 2023-09-21 07:46:07 ] Completed eval batch 0 recons_loss 0.208 ms, 1.03 s total -[ 2023-09-21 07:46:07 ] Completed eval batch 0 metrics update 0.473 ms, 1.03 s total -Saving checkpoint at epoch 62 val batch 0 -[ 2023-09-21 07:46:08 ] Completed saving temp checkpoint 1,644.197 ms, 2.68 s total -[ 2023-09-21 07:46:08 ] Completed replacing temp checkpoint with checkpoint 61.229 ms, 2.74 s total -[ 2023-09-21 07:46:08 ] Completed eval batch 1 to device 16.731 ms, 2.75 s total -[ 2023-09-21 07:46:09 ] Completed eval batch 1 forward 21.193 ms, 2.78 s total -[ 2023-09-21 07:46:09 ] Completed eval batch 1 recons_loss 0.324 ms, 2.78 s total -[ 2023-09-21 07:46:09 ] Completed eval batch 1 metrics update 0.298 ms, 2.78 s total -Saving checkpoint at epoch 62 val batch 1 -Epoch 62 val loss: 0.0711 -[ 2023-09-21 07:46:10 ] Completed saving temp checkpoint 1,690.169 ms, 4.47 s total -[ 2023-09-21 07:46:10 ] Completed replacing temp checkpoint with checkpoint 44.283 ms, 4.51 s total -[ 2023-09-21 07:46:10 ] Completed evaluating generator for epoch 62 50.871 ms, 4.56 s total - - -EPOCH :: 63 - - -[ 2023-09-21 07:46:10 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 63 from training step 0 - -[ 2023-09-21 07:46:13 ] Completed train batch 0 to device 3,157.389 ms, 3.16 s total -[ 2023-09-21 07:46:13 ] Completed train batch 0 generator forward 34.018 ms, 3.19 s total -[ 2023-09-21 07:46:14 ] Completed train batch 0 generator loss: 1.409 66.339 ms, 3.26 s total -[ 2023-09-21 07:46:14 ] Completed train batch 0 generator backward 148.786 ms, 3.41 s total -[ 2023-09-21 07:46:14 ] Completed train batch 0 discriminator loss 0.069 58.824 ms, 3.47 s total -[ 2023-09-21 07:46:14 ] Completed train batch 0 discriminator backward 81.808 ms, 3.55 s total -Epoch [63] Step [0/2], gen_loss: 0.471, disc_loss: 0.023 -[ 2023-09-21 07:46:14 ] Completed train batch 0 metrics update 9.314 ms, 3.56 s total -Saving checkpoint at epoch 63 train batch 0 -[ 2023-09-21 07:46:16 ] Completed saving temp checkpoint 1,687.748 ms, 5.24 s total -[ 2023-09-21 07:46:16 ] Completed replacing temp checkpoint with checkpoint 64.970 ms, 5.31 s total -[ 2023-09-21 07:46:16 ] Completed train batch 1 to device 505.510 ms, 5.81 s total -[ 2023-09-21 07:46:16 ] Completed train batch 1 generator forward 24.161 ms, 5.84 s total -[ 2023-09-21 07:46:16 ] Completed train batch 1 generator loss: 1.418 63.298 ms, 5.90 s total -[ 2023-09-21 07:46:16 ] Completed train batch 1 generator backward 215.178 ms, 6.12 s total -[ 2023-09-21 07:46:16 ] Completed train batch 1 discriminator loss 0.067 56.222 ms, 6.17 s total -[ 2023-09-21 07:46:17 ] Completed train batch 1 discriminator backward 68.521 ms, 6.24 s total -Epoch [63] Step [1/2], gen_loss: 0.473, disc_loss: 0.023 -[ 2023-09-21 07:46:17 ] Completed train batch 1 metrics update 5.025 ms, 6.25 s total -Saving checkpoint at epoch 63 train batch 1 -[ 2023-09-21 07:46:18 ] Completed saving temp checkpoint 1,714.156 ms, 7.96 s total -[ 2023-09-21 07:46:18 ] Completed replacing temp checkpoint with checkpoint 47.317 ms, 8.01 s total -Epoch [63] :: gen_loss: 0.472, disc_loss: 0.023 -[ 2023-09-21 07:46:18 ] Completed training generator for epoch 63 55.577 ms, 8.06 s total -[ 2023-09-21 07:46:18 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 63 from eval step 0 - -[ 2023-09-21 07:46:19 ] Completed eval batch 0 to device 980.702 ms, 0.98 s total -[ 2023-09-21 07:46:19 ] Completed eval batch 0 forward 23.724 ms, 1.00 s total -[ 2023-09-21 07:46:19 ] Completed eval batch 0 recons_loss 0.216 ms, 1.00 s total -[ 2023-09-21 07:46:19 ] Completed eval batch 0 metrics update 0.486 ms, 1.01 s total -Saving checkpoint at epoch 63 val batch 0 -[ 2023-09-21 07:46:20 ] Completed saving temp checkpoint 1,067.760 ms, 2.07 s total -[ 2023-09-21 07:46:21 ] Completed replacing temp checkpoint with checkpoint 62.519 ms, 2.14 s total -[ 2023-09-21 07:46:21 ] Completed eval batch 1 to device 18.476 ms, 2.15 s total -[ 2023-09-21 07:46:21 ] Completed eval batch 1 forward 26.240 ms, 2.18 s total -[ 2023-09-21 07:46:21 ] Completed eval batch 1 recons_loss 0.331 ms, 2.18 s total -[ 2023-09-21 07:46:21 ] Completed eval batch 1 metrics update 0.307 ms, 2.18 s total -Saving checkpoint at epoch 63 val batch 1 -Epoch 63 val loss: 0.0709 -[ 2023-09-21 07:46:21 ] Completed saving temp checkpoint 348.221 ms, 2.53 s total -[ 2023-09-21 07:46:21 ] Completed replacing temp checkpoint with checkpoint 26.025 ms, 2.56 s total -[ 2023-09-21 07:46:21 ] Completed evaluating generator for epoch 63 49.744 ms, 2.60 s total - - -EPOCH :: 64 - - -[ 2023-09-21 07:46:21 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 64 from training step 0 - -[ 2023-09-21 07:46:24 ] Completed train batch 0 to device 3,190.575 ms, 3.19 s total -[ 2023-09-21 07:46:24 ] Completed train batch 0 generator forward 26.732 ms, 3.22 s total -[ 2023-09-21 07:46:24 ] Completed train batch 0 generator loss: 1.402 66.087 ms, 3.28 s total -[ 2023-09-21 07:46:25 ] Completed train batch 0 generator backward 705.525 ms, 3.99 s total -[ 2023-09-21 07:46:25 ] Completed train batch 0 discriminator loss 0.072 56.929 ms, 4.05 s total -[ 2023-09-21 07:46:25 ] Completed train batch 0 discriminator backward 97.464 ms, 4.14 s total -Epoch [64] Step [0/2], gen_loss: 0.478, disc_loss: 0.024 -[ 2023-09-21 07:46:25 ] Completed train batch 0 metrics update 3.122 ms, 4.15 s total -Saving checkpoint at epoch 64 train batch 0 -[ 2023-09-21 07:46:26 ] Completed saving temp checkpoint 381.033 ms, 4.53 s total -[ 2023-09-21 07:46:26 ] Completed replacing temp checkpoint with checkpoint 26.860 ms, 4.55 s total -[ 2023-09-21 07:46:27 ] Completed train batch 1 to device 1,587.706 ms, 6.14 s total -[ 2023-09-21 07:46:27 ] Completed train batch 1 generator forward 23.380 ms, 6.17 s total -[ 2023-09-21 07:46:27 ] Completed train batch 1 generator loss: 1.411 65.257 ms, 6.23 s total -[ 2023-09-21 07:46:29 ] Completed train batch 1 generator backward 1,317.843 ms, 7.55 s total -[ 2023-09-21 07:46:29 ] Completed train batch 1 discriminator loss 0.077 56.531 ms, 7.61 s total -[ 2023-09-21 07:46:29 ] Completed train batch 1 discriminator backward 88.860 ms, 7.69 s total -Epoch [64] Step [1/2], gen_loss: 0.466, disc_loss: 0.026 -[ 2023-09-21 07:46:29 ] Completed train batch 1 metrics update 5.524 ms, 7.70 s total -Saving checkpoint at epoch 64 train batch 1 -[ 2023-09-21 07:46:29 ] Completed saving temp checkpoint 359.173 ms, 8.06 s total -[ 2023-09-21 07:46:29 ] Completed replacing temp checkpoint with checkpoint 18.111 ms, 8.08 s total -Epoch [64] :: gen_loss: 0.472, disc_loss: 0.025 -[ 2023-09-21 07:46:29 ] Completed training generator for epoch 64 55.378 ms, 8.13 s total -[ 2023-09-21 07:46:29 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 64 from eval step 0 - -[ 2023-09-21 07:46:30 ] Completed eval batch 0 to device 1,195.489 ms, 1.20 s total -[ 2023-09-21 07:46:30 ] Completed eval batch 0 forward 31.788 ms, 1.23 s total -[ 2023-09-21 07:46:30 ] Completed eval batch 0 recons_loss 0.290 ms, 1.23 s total -[ 2023-09-21 07:46:30 ] Completed eval batch 0 metrics update 0.572 ms, 1.23 s total -Saving checkpoint at epoch 64 val batch 0 -[ 2023-09-21 07:46:31 ] Completed saving temp checkpoint 392.608 ms, 1.62 s total -[ 2023-09-21 07:46:31 ] Completed replacing temp checkpoint with checkpoint 19.662 ms, 1.64 s total -[ 2023-09-21 07:46:31 ] Completed eval batch 1 to device 626.789 ms, 2.27 s total -[ 2023-09-21 07:46:31 ] Completed eval batch 1 forward 39.927 ms, 2.31 s total -[ 2023-09-21 07:46:31 ] Completed eval batch 1 recons_loss 0.553 ms, 2.31 s total -[ 2023-09-21 07:46:31 ] Completed eval batch 1 metrics update 67.580 ms, 2.38 s total -Saving checkpoint at epoch 64 val batch 1 -Epoch 64 val loss: 0.0644 -[ 2023-09-21 07:46:32 ] Completed saving temp checkpoint 639.560 ms, 3.01 s total -[ 2023-09-21 07:46:32 ] Completed replacing temp checkpoint with checkpoint 28.445 ms, 3.04 s total -[ 2023-09-21 07:46:32 ] Completed evaluating generator for epoch 64 53.124 ms, 3.10 s total - - -EPOCH :: 65 - - -[ 2023-09-21 07:46:32 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 65 from training step 0 - -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 07:57:22 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 07:57:22 ] Completed importing Timer 0.024 ms, 0.00 s total -[ 2023-09-21 07:57:27 ] Completed importing everything else 4,755.735 ms, 4.76 s total -| distributed init (rank 1): env:// -| distributed init (rank 0): env:// -| distributed init (rank 2): env:// -| distributed init (rank 5): env:// -| distributed init (rank 4): env:// -| distributed init (rank 3): env:// -[ 2023-09-21 07:57:34 ] Completed preliminaries 7,242.016 ms, 12.00 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 07:57:34 ] Completed build datasets 11.795 ms, 12.01 s total -[ 2023-09-21 07:57:34 ] Completed build samplers 0.044 ms, 12.01 s total -[ 2023-09-21 07:57:34 ] Completed build dataloaders 0.133 ms, 12.01 s total -[ 2023-09-21 07:57:34 ] Completed generator to device 78.040 ms, 12.09 s total -[ 2023-09-21 07:57:34 ] Completed discriminator to device 7.159 ms, 12.09 s total -[ 2023-09-21 07:57:35 ] Completed loss functions 456.607 ms, 12.55 s total -[ 2023-09-21 07:57:35 ] Completed models prepped for distribution 88.907 ms, 12.64 s total -[ 2023-09-21 07:57:35 ] Completed optimizers 0.669 ms, 12.64 s total -[ 2023-09-21 07:57:35 ] Completed grad scalers 0.032 ms, 12.64 s total -[ 2023-09-21 07:57:36 ] Completed checkpoint retrieval 515.327 ms, 13.16 s total - - -EPOCH :: 64 - - -[ 2023-09-21 07:57:36 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 64 from training step 2 - -Epoch [64] :: gen_loss: 0.472, disc_loss: 0.025 -[ 2023-09-21 07:57:36 ] Completed training generator for epoch 64 126.429 ms, 0.13 s total -[ 2023-09-21 07:57:36 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 64 from eval step 2 - -[ 2023-09-21 07:57:36 ] Completed evaluating generator for epoch 64 123.818 ms, 0.12 s total - - -EPOCH :: 65 - - -[ 2023-09-21 07:57:36 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 65 from training step 0 - -[ 2023-09-21 07:57:40 ] Completed train batch 0 to device 3,924.223 ms, 3.92 s total -[ 2023-09-21 07:57:41 ] Completed train batch 0 generator forward 1,639.195 ms, 5.56 s total -[ 2023-09-21 07:57:41 ] Completed train batch 0 generator loss: 1.421 78.163 ms, 5.64 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 07:57:42 ] Completed train batch 0 generator backward 535.484 ms, 6.18 s total -[ 2023-09-21 07:57:42 ] Completed train batch 0 discriminator loss 0.081 62.398 ms, 6.24 s total -[ 2023-09-21 07:57:42 ] Completed train batch 0 discriminator backward 88.723 ms, 6.33 s total -Epoch [65] Step [0/2], gen_loss: 0.463, disc_loss: 0.026 -[ 2023-09-21 07:57:42 ] Completed train batch 0 metrics update 4.782 ms, 6.33 s total -Saving checkpoint at epoch 65 train batch 0 -[ 2023-09-21 07:57:43 ] Completed saving temp checkpoint 410.913 ms, 6.74 s total -[ 2023-09-21 07:57:43 ] Completed replacing temp checkpoint with checkpoint 126.369 ms, 6.87 s total -[ 2023-09-21 07:57:43 ] Completed train batch 1 to device 255.134 ms, 7.13 s total -[ 2023-09-21 07:57:43 ] Completed train batch 1 generator forward 22.289 ms, 7.15 s total -[ 2023-09-21 07:57:43 ] Completed train batch 1 generator loss: 1.374 63.400 ms, 7.21 s total -[ 2023-09-21 07:57:44 ] Completed train batch 1 generator backward 1,153.316 ms, 8.36 s total -[ 2023-09-21 07:57:44 ] Completed train batch 1 discriminator loss 0.079 56.747 ms, 8.42 s total -[ 2023-09-21 07:57:44 ] Completed train batch 1 discriminator backward 69.588 ms, 8.49 s total -Epoch [65] Step [1/2], gen_loss: 0.460, disc_loss: 0.027 -[ 2023-09-21 07:57:44 ] Completed train batch 1 metrics update 1.912 ms, 8.49 s total -Saving checkpoint at epoch 65 train batch 1 -[ 2023-09-21 07:57:45 ] Completed saving temp checkpoint 442.456 ms, 8.94 s total -[ 2023-09-21 07:57:45 ] Completed replacing temp checkpoint with checkpoint 18.870 ms, 8.95 s total -Epoch [65] :: gen_loss: 0.461, disc_loss: 0.027 -[ 2023-09-21 07:57:45 ] Completed training generator for epoch 65 38.525 ms, 8.99 s total -[ 2023-09-21 07:57:45 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 65 from eval step 0 - -[ 2023-09-21 07:57:46 ] Completed eval batch 0 to device 1,167.462 ms, 1.17 s total -[ 2023-09-21 07:57:46 ] Completed eval batch 0 forward 36.284 ms, 1.20 s total -[ 2023-09-21 07:57:46 ] Completed eval batch 0 recons_loss 0.272 ms, 1.20 s total -[ 2023-09-21 07:57:46 ] Completed eval batch 0 metrics update 0.561 ms, 1.20 s total -Saving checkpoint at epoch 65 val batch 0 -[ 2023-09-21 07:57:46 ] Completed saving temp checkpoint 468.114 ms, 1.67 s total -[ 2023-09-21 07:57:46 ] Completed replacing temp checkpoint with checkpoint 30.477 ms, 1.70 s total -[ 2023-09-21 07:57:47 ] Completed eval batch 1 to device 449.251 ms, 2.15 s total -[ 2023-09-21 07:57:47 ] Completed eval batch 1 forward 40.457 ms, 2.19 s total -[ 2023-09-21 07:57:47 ] Completed eval batch 1 recons_loss 0.665 ms, 2.19 s total -[ 2023-09-21 07:57:47 ] Completed eval batch 1 metrics update 84.523 ms, 2.28 s total -Saving checkpoint at epoch 65 val batch 1 -Epoch 65 val loss: 0.0624 -[ 2023-09-21 07:57:48 ] Completed saving temp checkpoint 466.139 ms, 2.74 s total -[ 2023-09-21 07:57:48 ] Completed replacing temp checkpoint with checkpoint 25.501 ms, 2.77 s total -[ 2023-09-21 07:57:48 ] Completed evaluating generator for epoch 65 51.879 ms, 2.82 s total - - -EPOCH :: 66 - - -[ 2023-09-21 07:57:48 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 66 from training step 0 - -[ 2023-09-21 07:57:51 ] Completed train batch 0 to device 3,071.725 ms, 3.07 s total -[ 2023-09-21 07:57:51 ] Completed train batch 0 generator forward 27.393 ms, 3.10 s total -[ 2023-09-21 07:57:51 ] Completed train batch 0 generator loss: 1.368 65.656 ms, 3.16 s total -[ 2023-09-21 07:57:52 ] Completed train batch 0 generator backward 802.108 ms, 3.97 s total -[ 2023-09-21 07:57:52 ] Completed train batch 0 discriminator loss 0.081 58.141 ms, 4.03 s total -[ 2023-09-21 07:57:52 ] Completed train batch 0 discriminator backward 87.451 ms, 4.11 s total -Epoch [66] Step [0/2], gen_loss: 0.452, disc_loss: 0.026 -[ 2023-09-21 07:57:52 ] Completed train batch 0 metrics update 5.418 ms, 4.12 s total -Saving checkpoint at epoch 66 train batch 0 -[ 2023-09-21 07:57:52 ] Completed saving temp checkpoint 430.365 ms, 4.55 s total -[ 2023-09-21 07:57:52 ] Completed replacing temp checkpoint with checkpoint 23.926 ms, 4.57 s total -[ 2023-09-21 07:57:54 ] Completed train batch 1 to device 1,373.882 ms, 5.95 s total -[ 2023-09-21 07:57:54 ] Completed train batch 1 generator forward 39.977 ms, 5.99 s total -[ 2023-09-21 07:57:54 ] Completed train batch 1 generator loss: 1.324 64.688 ms, 6.05 s total -[ 2023-09-21 07:57:55 ] Completed train batch 1 generator backward 1,565.706 ms, 7.62 s total -[ 2023-09-21 07:57:55 ] Completed train batch 1 discriminator loss 0.073 57.426 ms, 7.67 s total -[ 2023-09-21 07:57:55 ] Completed train batch 1 discriminator backward 90.599 ms, 7.76 s total -Epoch [66] Step [1/2], gen_loss: 0.458, disc_loss: 0.025 -[ 2023-09-21 07:57:55 ] Completed train batch 1 metrics update 5.459 ms, 7.77 s total -Saving checkpoint at epoch 66 train batch 1 -[ 2023-09-21 07:57:56 ] Completed saving temp checkpoint 389.900 ms, 8.16 s total -[ 2023-09-21 07:57:56 ] Completed replacing temp checkpoint with checkpoint 20.658 ms, 8.18 s total -Epoch [66] :: gen_loss: 0.455, disc_loss: 0.025 -[ 2023-09-21 07:57:56 ] Completed training generator for epoch 66 48.393 ms, 8.23 s total -[ 2023-09-21 07:57:56 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 66 from eval step 0 - -[ 2023-09-21 07:57:57 ] Completed eval batch 0 to device 1,189.272 ms, 1.19 s total -[ 2023-09-21 07:57:57 ] Completed eval batch 0 forward 26.415 ms, 1.22 s total -[ 2023-09-21 07:57:57 ] Completed eval batch 0 recons_loss 0.339 ms, 1.22 s total -[ 2023-09-21 07:57:57 ] Completed eval batch 0 metrics update 0.607 ms, 1.22 s total -Saving checkpoint at epoch 66 val batch 0 -[ 2023-09-21 07:57:57 ] Completed saving temp checkpoint 396.969 ms, 1.61 s total -[ 2023-09-21 07:57:57 ] Completed replacing temp checkpoint with checkpoint 23.353 ms, 1.64 s total -[ 2023-09-21 07:57:58 ] Completed eval batch 1 to device 630.587 ms, 2.27 s total -[ 2023-09-21 07:57:58 ] Completed eval batch 1 forward 21.279 ms, 2.29 s total -[ 2023-09-21 07:57:58 ] Completed eval batch 1 recons_loss 0.344 ms, 2.29 s total -[ 2023-09-21 07:57:58 ] Completed eval batch 1 metrics update 33.533 ms, 2.32 s total -Saving checkpoint at epoch 66 val batch 1 -Epoch 66 val loss: 0.0673 -[ 2023-09-21 07:57:59 ] Completed saving temp checkpoint 385.726 ms, 2.71 s total -[ 2023-09-21 07:57:59 ] Completed replacing temp checkpoint with checkpoint 19.797 ms, 2.73 s total -[ 2023-09-21 07:57:59 ] Completed evaluating generator for epoch 66 53.046 ms, 2.78 s total - - -EPOCH :: 67 - - -[ 2023-09-21 07:57:59 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 67 from training step 0 - -[ 2023-09-21 07:58:02 ] Completed train batch 0 to device 3,165.403 ms, 3.17 s total -[ 2023-09-21 07:58:02 ] Completed train batch 0 generator forward 41.164 ms, 3.21 s total -[ 2023-09-21 07:58:02 ] Completed train batch 0 generator loss: 1.308 90.277 ms, 3.30 s total -[ 2023-09-21 07:58:03 ] Completed train batch 0 generator backward 652.949 ms, 3.95 s total -[ 2023-09-21 07:58:03 ] Completed train batch 0 discriminator loss 0.094 58.870 ms, 4.01 s total -[ 2023-09-21 07:58:03 ] Completed train batch 0 discriminator backward 91.831 ms, 4.10 s total -Epoch [67] Step [0/2], gen_loss: 0.441, disc_loss: 0.031 -[ 2023-09-21 07:58:03 ] Completed train batch 0 metrics update 24.993 ms, 4.13 s total -Saving checkpoint at epoch 67 train batch 0 -[ 2023-09-21 07:58:03 ] Completed saving temp checkpoint 350.222 ms, 4.48 s total -[ 2023-09-21 07:58:03 ] Completed replacing temp checkpoint with checkpoint 19.123 ms, 4.49 s total -[ 2023-09-21 07:58:05 ] Completed train batch 1 to device 1,575.093 ms, 6.07 s total -[ 2023-09-21 07:58:05 ] Completed train batch 1 generator forward 38.875 ms, 6.11 s total -[ 2023-09-21 07:58:05 ] Completed train batch 1 generator loss: 1.315 63.681 ms, 6.17 s total -[ 2023-09-21 07:58:06 ] Completed train batch 1 generator backward 1,151.062 ms, 7.32 s total -[ 2023-09-21 07:58:06 ] Completed train batch 1 discriminator loss 0.130 56.855 ms, 7.38 s total -[ 2023-09-21 07:58:06 ] Completed train batch 1 discriminator backward 89.108 ms, 7.47 s total -Epoch [67] Step [1/2], gen_loss: 0.441, disc_loss: 0.043 -[ 2023-09-21 07:58:06 ] Completed train batch 1 metrics update 3.902 ms, 7.47 s total -Saving checkpoint at epoch 67 train batch 1 -[ 2023-09-21 07:58:07 ] Completed saving temp checkpoint 448.778 ms, 7.92 s total -[ 2023-09-21 07:58:07 ] Completed replacing temp checkpoint with checkpoint 20.324 ms, 7.94 s total -Epoch [67] :: gen_loss: 0.441, disc_loss: 0.037 -[ 2023-09-21 07:58:07 ] Completed training generator for epoch 67 48.801 ms, 7.99 s total -[ 2023-09-21 07:58:07 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 67 from eval step 0 - -[ 2023-09-21 07:58:08 ] Completed eval batch 0 to device 1,131.437 ms, 1.13 s total -[ 2023-09-21 07:58:08 ] Completed eval batch 0 forward 31.435 ms, 1.16 s total -[ 2023-09-21 07:58:08 ] Completed eval batch 0 recons_loss 0.317 ms, 1.16 s total -[ 2023-09-21 07:58:08 ] Completed eval batch 0 metrics update 0.594 ms, 1.16 s total -Saving checkpoint at epoch 67 val batch 0 -[ 2023-09-21 07:58:08 ] Completed saving temp checkpoint 423.934 ms, 1.59 s total -[ 2023-09-21 07:58:08 ] Completed replacing temp checkpoint with checkpoint 30.295 ms, 1.62 s total -[ 2023-09-21 07:58:09 ] Completed eval batch 1 to device 553.525 ms, 2.17 s total -[ 2023-09-21 07:58:09 ] Completed eval batch 1 forward 39.437 ms, 2.21 s total -[ 2023-09-21 07:58:09 ] Completed eval batch 1 recons_loss 0.335 ms, 2.21 s total -[ 2023-09-21 07:58:09 ] Completed eval batch 1 metrics update 161.407 ms, 2.37 s total -Saving checkpoint at epoch 67 val batch 1 -Epoch 67 val loss: 0.0727 -[ 2023-09-21 07:58:09 ] Completed saving temp checkpoint 481.884 ms, 2.85 s total -[ 2023-09-21 07:58:09 ] Completed replacing temp checkpoint with checkpoint 26.666 ms, 2.88 s total -[ 2023-09-21 07:58:10 ] Completed evaluating generator for epoch 67 47.122 ms, 2.93 s total - - -EPOCH :: 68 - - -[ 2023-09-21 07:58:10 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 68 from training step 0 - -[ 2023-09-21 07:58:13 ] Completed train batch 0 to device 3,201.496 ms, 3.20 s total -[ 2023-09-21 07:58:13 ] Completed train batch 0 generator forward 33.746 ms, 3.24 s total -[ 2023-09-21 07:58:13 ] Completed train batch 0 generator loss: 1.332 66.054 ms, 3.30 s total -[ 2023-09-21 07:58:13 ] Completed train batch 0 generator backward 531.775 ms, 3.83 s total -[ 2023-09-21 07:58:13 ] Completed train batch 0 discriminator loss 0.115 57.361 ms, 3.89 s total -[ 2023-09-21 07:58:14 ] Completed train batch 0 discriminator backward 114.072 ms, 4.00 s total -Epoch [68] Step [0/2], gen_loss: 0.438, disc_loss: 0.039 -[ 2023-09-21 07:58:14 ] Completed train batch 0 metrics update 0.990 ms, 4.01 s total -Saving checkpoint at epoch 68 train batch 0 -[ 2023-09-21 07:58:14 ] Completed saving temp checkpoint 524.974 ms, 4.53 s total -[ 2023-09-21 07:58:14 ] Completed replacing temp checkpoint with checkpoint 24.865 ms, 4.56 s total -[ 2023-09-21 07:58:16 ] Completed train batch 1 to device 1,746.830 ms, 6.30 s total -[ 2023-09-21 07:58:16 ] Completed train batch 1 generator forward 40.431 ms, 6.34 s total -[ 2023-09-21 07:58:16 ] Completed train batch 1 generator loss: 1.448 63.751 ms, 6.41 s total -[ 2023-09-21 07:58:17 ] Completed train batch 1 generator backward 868.392 ms, 7.27 s total -[ 2023-09-21 07:58:17 ] Completed train batch 1 discriminator loss 0.093 56.961 ms, 7.33 s total -[ 2023-09-21 07:58:17 ] Completed train batch 1 discriminator backward 71.342 ms, 7.40 s total -Epoch [68] Step [1/2], gen_loss: 0.458, disc_loss: 0.035 -[ 2023-09-21 07:58:17 ] Completed train batch 1 metrics update 3.233 ms, 7.41 s total -Saving checkpoint at epoch 68 train batch 1 -[ 2023-09-21 07:58:17 ] Completed saving temp checkpoint 510.881 ms, 7.92 s total -[ 2023-09-21 07:58:17 ] Completed replacing temp checkpoint with checkpoint 20.210 ms, 7.94 s total -Epoch [68] :: gen_loss: 0.448, disc_loss: 0.037 -[ 2023-09-21 07:58:18 ] Completed training generator for epoch 68 65.064 ms, 8.00 s total -[ 2023-09-21 07:58:18 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 68 from eval step 0 - -[ 2023-09-21 07:58:19 ] Completed eval batch 0 to device 1,071.219 ms, 1.07 s total -[ 2023-09-21 07:58:19 ] Completed eval batch 0 forward 23.817 ms, 1.10 s total -[ 2023-09-21 07:58:19 ] Completed eval batch 0 recons_loss 0.229 ms, 1.10 s total -[ 2023-09-21 07:58:19 ] Completed eval batch 0 metrics update 0.534 ms, 1.10 s total -Saving checkpoint at epoch 68 val batch 0 -[ 2023-09-21 07:58:19 ] Completed saving temp checkpoint 446.660 ms, 1.54 s total -[ 2023-09-21 07:58:19 ] Completed replacing temp checkpoint with checkpoint 36.085 ms, 1.58 s total -[ 2023-09-21 07:58:20 ] Completed eval batch 1 to device 508.880 ms, 2.09 s total -[ 2023-09-21 07:58:20 ] Completed eval batch 1 forward 21.433 ms, 2.11 s total -[ 2023-09-21 07:58:20 ] Completed eval batch 1 recons_loss 0.332 ms, 2.11 s total -[ 2023-09-21 07:58:20 ] Completed eval batch 1 metrics update 66.442 ms, 2.18 s total -Saving checkpoint at epoch 68 val batch 1 -Epoch 68 val loss: 0.0640 -[ 2023-09-21 07:58:20 ] Completed saving temp checkpoint 487.707 ms, 2.66 s total -[ 2023-09-21 07:58:20 ] Completed replacing temp checkpoint with checkpoint 21.877 ms, 2.69 s total -[ 2023-09-21 07:58:20 ] Completed evaluating generator for epoch 68 51.097 ms, 2.74 s total - - -EPOCH :: 69 - - -[ 2023-09-21 07:58:20 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 69 from training step 0 - -[ 2023-09-21 07:58:23 ] Completed train batch 0 to device 2,991.578 ms, 2.99 s total -[ 2023-09-21 07:58:23 ] Completed train batch 0 generator forward 27.041 ms, 3.02 s total -[ 2023-09-21 07:58:23 ] Completed train batch 0 generator loss: 1.299 69.722 ms, 3.09 s total -[ 2023-09-21 07:58:24 ] Completed train batch 0 generator backward 662.208 ms, 3.75 s total -[ 2023-09-21 07:58:24 ] Completed train batch 0 discriminator loss 0.135 58.301 ms, 3.81 s total -[ 2023-09-21 07:58:24 ] Completed train batch 0 discriminator backward 86.715 ms, 3.90 s total -Epoch [69] Step [0/2], gen_loss: 0.446, disc_loss: 0.044 -[ 2023-09-21 07:58:24 ] Completed train batch 0 metrics update 5.229 ms, 3.90 s total -Saving checkpoint at epoch 69 train batch 0 -[ 2023-09-21 07:58:25 ] Completed saving temp checkpoint 448.222 ms, 4.35 s total -[ 2023-09-21 07:58:25 ] Completed replacing temp checkpoint with checkpoint 24.982 ms, 4.37 s total -[ 2023-09-21 07:58:27 ] Completed train batch 1 to device 1,865.326 ms, 6.24 s total -[ 2023-09-21 07:58:27 ] Completed train batch 1 generator forward 26.428 ms, 6.27 s total -[ 2023-09-21 07:58:27 ] Completed train batch 1 generator loss: 1.357 63.457 ms, 6.33 s total -[ 2023-09-21 07:58:28 ] Completed train batch 1 generator backward 1,011.093 ms, 7.34 s total -[ 2023-09-21 07:58:28 ] Completed train batch 1 discriminator loss 0.128 62.968 ms, 7.40 s total -[ 2023-09-21 07:58:28 ] Completed train batch 1 discriminator backward 58.608 ms, 7.46 s total -Epoch [69] Step [1/2], gen_loss: 0.457, disc_loss: 0.043 -[ 2023-09-21 07:58:28 ] Completed train batch 1 metrics update 0.795 ms, 7.46 s total -Saving checkpoint at epoch 69 train batch 1 -[ 2023-09-21 07:58:28 ] Completed saving temp checkpoint 489.358 ms, 7.95 s total -[ 2023-09-21 07:58:28 ] Completed replacing temp checkpoint with checkpoint 22.628 ms, 7.97 s total -Epoch [69] :: gen_loss: 0.451, disc_loss: 0.044 -[ 2023-09-21 07:58:28 ] Completed training generator for epoch 69 50.364 ms, 8.03 s total -[ 2023-09-21 07:58:28 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 69 from eval step 0 - -[ 2023-09-21 07:58:29 ] Completed eval batch 0 to device 1,160.171 ms, 1.16 s total -[ 2023-09-21 07:58:29 ] Completed eval batch 0 forward 43.831 ms, 1.20 s total -[ 2023-09-21 07:58:29 ] Completed eval batch 0 recons_loss 0.379 ms, 1.20 s total -[ 2023-09-21 07:58:29 ] Completed eval batch 0 metrics update 0.732 ms, 1.21 s total -Saving checkpoint at epoch 69 val batch 0 -[ 2023-09-21 07:58:30 ] Completed saving temp checkpoint 488.500 ms, 1.69 s total -[ 2023-09-21 07:58:30 ] Completed replacing temp checkpoint with checkpoint 36.871 ms, 1.73 s total -[ 2023-09-21 07:58:30 ] Completed eval batch 1 to device 333.524 ms, 2.06 s total -[ 2023-09-21 07:58:30 ] Completed eval batch 1 forward 36.796 ms, 2.10 s total -[ 2023-09-21 07:58:30 ] Completed eval batch 1 recons_loss 0.583 ms, 2.10 s total -[ 2023-09-21 07:58:30 ] Completed eval batch 1 metrics update 100.548 ms, 2.20 s total -Saving checkpoint at epoch 69 val batch 1 -Epoch 69 val loss: 0.0655 -[ 2023-09-21 07:58:31 ] Completed saving temp checkpoint 477.221 ms, 2.68 s total -[ 2023-09-21 07:58:31 ] Completed replacing temp checkpoint with checkpoint 23.654 ms, 2.70 s total -[ 2023-09-21 07:58:31 ] Completed evaluating generator for epoch 69 66.408 ms, 2.77 s total - - -EPOCH :: 70 - - -[ 2023-09-21 07:58:31 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 70 from training step 0 - -[ 2023-09-21 07:58:34 ] Completed train batch 0 to device 3,141.693 ms, 3.14 s total -[ 2023-09-21 07:58:34 ] Completed train batch 0 generator forward 27.267 ms, 3.17 s total -[ 2023-09-21 07:58:34 ] Completed train batch 0 generator loss: 1.375 64.658 ms, 3.23 s total -[ 2023-09-21 07:58:35 ] Completed train batch 0 generator backward 568.531 ms, 3.80 s total -[ 2023-09-21 07:58:35 ] Completed train batch 0 discriminator loss 0.108 57.565 ms, 3.86 s total -[ 2023-09-21 07:58:35 ] Completed train batch 0 discriminator backward 88.898 ms, 3.95 s total -Epoch [70] Step [0/2], gen_loss: 0.463, disc_loss: 0.036 -[ 2023-09-21 07:58:35 ] Completed train batch 0 metrics update 3.056 ms, 3.95 s total -Saving checkpoint at epoch 70 train batch 0 -[ 2023-09-21 07:58:35 ] Completed saving temp checkpoint 461.149 ms, 4.41 s total -[ 2023-09-21 07:58:35 ] Completed replacing temp checkpoint with checkpoint 20.304 ms, 4.43 s total -[ 2023-09-21 07:58:37 ] Completed train batch 1 to device 1,659.915 ms, 6.09 s total -[ 2023-09-21 07:58:37 ] Completed train batch 1 generator forward 22.275 ms, 6.12 s total -[ 2023-09-21 07:58:37 ] Completed train batch 1 generator loss: 1.347 63.017 ms, 6.18 s total -[ 2023-09-21 07:58:38 ] Completed train batch 1 generator backward 805.908 ms, 6.98 s total -[ 2023-09-21 07:58:38 ] Completed train batch 1 discriminator loss 0.109 56.588 ms, 7.04 s total -[ 2023-09-21 07:58:38 ] Completed train batch 1 discriminator backward 75.112 ms, 7.12 s total -Epoch [70] Step [1/2], gen_loss: 0.452, disc_loss: 0.036 -[ 2023-09-21 07:58:38 ] Completed train batch 1 metrics update 4.685 ms, 7.12 s total -Saving checkpoint at epoch 70 train batch 1 -[ 2023-09-21 07:58:39 ] Completed saving temp checkpoint 395.437 ms, 7.52 s total -[ 2023-09-21 07:58:39 ] Completed replacing temp checkpoint with checkpoint 22.855 ms, 7.54 s total -Epoch [70] :: gen_loss: 0.458, disc_loss: 0.036 -[ 2023-09-21 07:58:39 ] Completed training generator for epoch 70 60.674 ms, 7.60 s total -[ 2023-09-21 07:58:39 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 70 from eval step 0 - -[ 2023-09-21 07:58:40 ] Completed eval batch 0 to device 1,074.515 ms, 1.07 s total -[ 2023-09-21 07:58:40 ] Completed eval batch 0 forward 23.678 ms, 1.10 s total -[ 2023-09-21 07:58:40 ] Completed eval batch 0 recons_loss 0.220 ms, 1.10 s total -[ 2023-09-21 07:58:40 ] Completed eval batch 0 metrics update 0.588 ms, 1.10 s total -Saving checkpoint at epoch 70 val batch 0 -[ 2023-09-21 07:58:40 ] Completed saving temp checkpoint 297.500 ms, 1.40 s total -[ 2023-09-21 07:58:40 ] Completed replacing temp checkpoint with checkpoint 24.150 ms, 1.42 s total -[ 2023-09-21 07:58:41 ] Completed eval batch 1 to device 532.310 ms, 1.95 s total -[ 2023-09-21 07:58:41 ] Completed eval batch 1 forward 21.677 ms, 1.97 s total -[ 2023-09-21 07:58:41 ] Completed eval batch 1 recons_loss 0.373 ms, 1.98 s total -[ 2023-09-21 07:58:41 ] Completed eval batch 1 metrics update 320.625 ms, 2.30 s total -Saving checkpoint at epoch 70 val batch 1 -Epoch 70 val loss: 0.0747 -[ 2023-09-21 07:58:41 ] Completed saving temp checkpoint 401.100 ms, 2.70 s total -[ 2023-09-21 07:58:41 ] Completed replacing temp checkpoint with checkpoint 17.812 ms, 2.71 s total -[ 2023-09-21 07:58:41 ] Completed evaluating generator for epoch 70 62.267 ms, 2.78 s total - - -EPOCH :: 71 - - -[ 2023-09-21 07:58:41 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 71 from training step 0 - -[ 2023-09-21 07:58:45 ] Completed train batch 0 to device 3,124.482 ms, 3.12 s total -[ 2023-09-21 07:58:45 ] Completed train batch 0 generator forward 34.011 ms, 3.16 s total -[ 2023-09-21 07:58:45 ] Completed train batch 0 generator loss: 1.362 66.355 ms, 3.22 s total -[ 2023-09-21 07:58:45 ] Completed train batch 0 generator backward 554.264 ms, 3.78 s total -[ 2023-09-21 07:58:45 ] Completed train batch 0 discriminator loss 0.106 57.635 ms, 3.84 s total -[ 2023-09-21 07:58:45 ] Completed train batch 0 discriminator backward 103.922 ms, 3.94 s total -Epoch [71] Step [0/2], gen_loss: 0.459, disc_loss: 0.035 -[ 2023-09-21 07:58:45 ] Completed train batch 0 metrics update 5.941 ms, 3.95 s total -Saving checkpoint at epoch 71 train batch 0 -[ 2023-09-21 07:58:46 ] Completed saving temp checkpoint 433.877 ms, 4.38 s total -[ 2023-09-21 07:58:46 ] Completed replacing temp checkpoint with checkpoint 30.553 ms, 4.41 s total -[ 2023-09-21 07:58:47 ] Completed train batch 1 to device 1,551.116 ms, 5.96 s total -[ 2023-09-21 07:58:47 ] Completed train batch 1 generator forward 22.154 ms, 5.98 s total -[ 2023-09-21 07:58:47 ] Completed train batch 1 generator loss: 1.350 63.407 ms, 6.05 s total -[ 2023-09-21 07:58:49 ] Completed train batch 1 generator backward 1,346.798 ms, 7.39 s total -[ 2023-09-21 07:58:49 ] Completed train batch 1 discriminator loss 0.100 56.385 ms, 7.45 s total -[ 2023-09-21 07:58:49 ] Completed train batch 1 discriminator backward 88.573 ms, 7.54 s total -Epoch [71] Step [1/2], gen_loss: 0.452, disc_loss: 0.034 -[ 2023-09-21 07:58:49 ] Completed train batch 1 metrics update 4.402 ms, 7.54 s total -Saving checkpoint at epoch 71 train batch 1 -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 08:11:13 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 08:11:13 ] Completed importing Timer 0.026 ms, 0.00 s total -[ 2023-09-21 08:11:18 ] Completed importing everything else 4,614.973 ms, 4.61 s total -| distributed init (rank 3): env:// -| distributed init (rank 0): env:// -| distributed init (rank 4): env:// -| distributed init (rank 2): env:// -| distributed init (rank 1): env:// -| distributed init (rank 5): env:// -[ 2023-09-21 08:11:26 ] Completed preliminaries 7,857.885 ms, 12.47 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 08:11:26 ] Completed build datasets 12.437 ms, 12.49 s total -[ 2023-09-21 08:11:26 ] Completed build samplers 0.042 ms, 12.49 s total -[ 2023-09-21 08:11:26 ] Completed build dataloaders 0.127 ms, 12.49 s total -[ 2023-09-21 08:11:26 ] Completed generator to device 90.771 ms, 12.58 s total -[ 2023-09-21 08:11:26 ] Completed discriminator to device 7.047 ms, 12.58 s total -[ 2023-09-21 08:11:26 ] Completed loss functions 396.861 ms, 12.98 s total -[ 2023-09-21 08:11:26 ] Completed models prepped for distribution 100.575 ms, 13.08 s total -[ 2023-09-21 08:11:26 ] Completed optimizers 0.656 ms, 13.08 s total -[ 2023-09-21 08:11:26 ] Completed grad scalers 0.019 ms, 13.08 s total -[ 2023-09-21 08:11:27 ] Completed checkpoint retrieval 437.787 ms, 13.52 s total - - -EPOCH :: 71 - - -[ 2023-09-21 08:11:27 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 71 from training step 1 - -[ 2023-09-21 08:11:30 ] Completed train batch 1 to device 3,541.980 ms, 3.54 s total -[ 2023-09-21 08:11:31 ] Completed train batch 1 generator forward 1,088.314 ms, 4.63 s total -[ 2023-09-21 08:11:31 ] Completed train batch 1 generator loss: 1.296 77.965 ms, 4.71 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 08:11:32 ] Completed train batch 1 generator backward 794.755 ms, 5.50 s total -[ 2023-09-21 08:11:32 ] Completed train batch 1 discriminator loss 0.100 55.693 ms, 5.56 s total -[ 2023-09-21 08:11:32 ] Completed train batch 1 discriminator backward 87.523 ms, 5.65 s total -Epoch [71] Step [1/2], gen_loss: 0.455, disc_loss: 0.034 -[ 2023-09-21 08:11:32 ] Completed train batch 1 metrics update 5.483 ms, 5.65 s total -Saving checkpoint at epoch 71 train batch 1 -[ 2023-09-21 08:11:33 ] Completed saving temp checkpoint 487.428 ms, 6.14 s total -[ 2023-09-21 08:11:33 ] Completed replacing temp checkpoint with checkpoint 131.364 ms, 6.27 s total -Epoch [71] :: gen_loss: 0.457, disc_loss: 0.035 -[ 2023-09-21 08:11:33 ] Completed training generator for epoch 71 36.094 ms, 6.31 s total -[ 2023-09-21 08:11:33 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 71 from eval step 0 - -[ 2023-09-21 08:11:34 ] Completed eval batch 0 to device 1,090.389 ms, 1.09 s total -[ 2023-09-21 08:11:34 ] Completed eval batch 0 forward 42.589 ms, 1.13 s total -[ 2023-09-21 08:11:34 ] Completed eval batch 0 recons_loss 0.347 ms, 1.13 s total -[ 2023-09-21 08:11:34 ] Completed eval batch 0 metrics update 0.626 ms, 1.13 s total -Saving checkpoint at epoch 71 val batch 0 -[ 2023-09-21 08:11:35 ] Completed saving temp checkpoint 446.639 ms, 1.58 s total -[ 2023-09-21 08:11:35 ] Completed replacing temp checkpoint with checkpoint 30.902 ms, 1.61 s total -[ 2023-09-21 08:11:35 ] Completed eval batch 1 to device 293.216 ms, 1.90 s total -[ 2023-09-21 08:11:35 ] Completed eval batch 1 forward 40.259 ms, 1.94 s total -[ 2023-09-21 08:11:35 ] Completed eval batch 1 recons_loss 0.532 ms, 1.95 s total -[ 2023-09-21 08:11:35 ] Completed eval batch 1 metrics update 151.727 ms, 2.10 s total -Saving checkpoint at epoch 71 val batch 1 -Epoch 71 val loss: 0.0817 -[ 2023-09-21 08:11:35 ] Completed saving temp checkpoint 439.025 ms, 2.54 s total -[ 2023-09-21 08:11:35 ] Completed replacing temp checkpoint with checkpoint 20.708 ms, 2.56 s total -[ 2023-09-21 08:11:36 ] Completed evaluating generator for epoch 71 52.904 ms, 2.61 s total - - -EPOCH :: 72 - - -[ 2023-09-21 08:11:36 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 72 from training step 0 - -[ 2023-09-21 08:11:39 ] Completed train batch 0 to device 2,988.386 ms, 2.99 s total -[ 2023-09-21 08:11:39 ] Completed train batch 0 generator forward 28.979 ms, 3.02 s total -[ 2023-09-21 08:11:39 ] Completed train batch 0 generator loss: 1.326 67.123 ms, 3.08 s total -[ 2023-09-21 08:11:39 ] Completed train batch 0 generator backward 818.970 ms, 3.90 s total -[ 2023-09-21 08:11:40 ] Completed train batch 0 discriminator loss 0.099 57.118 ms, 3.96 s total -[ 2023-09-21 08:11:40 ] Completed train batch 0 discriminator backward 99.349 ms, 4.06 s total -Epoch [72] Step [0/2], gen_loss: 0.445, disc_loss: 0.033 -[ 2023-09-21 08:11:40 ] Completed train batch 0 metrics update 28.022 ms, 4.09 s total -Saving checkpoint at epoch 72 train batch 0 -[ 2023-09-21 08:11:40 ] Completed saving temp checkpoint 377.349 ms, 4.47 s total -[ 2023-09-21 08:11:40 ] Completed replacing temp checkpoint with checkpoint 30.498 ms, 4.50 s total -[ 2023-09-21 08:11:41 ] Completed train batch 1 to device 1,297.884 ms, 5.79 s total -[ 2023-09-21 08:11:41 ] Completed train batch 1 generator forward 22.817 ms, 5.82 s total -[ 2023-09-21 08:11:41 ] Completed train batch 1 generator loss: 1.306 63.894 ms, 5.88 s total -[ 2023-09-21 08:11:43 ] Completed train batch 1 generator backward 1,630.351 ms, 7.51 s total -[ 2023-09-21 08:11:43 ] Completed train batch 1 discriminator loss 0.096 56.381 ms, 7.57 s total -[ 2023-09-21 08:11:43 ] Completed train batch 1 discriminator backward 85.316 ms, 7.65 s total -Epoch [72] Step [1/2], gen_loss: 0.447, disc_loss: 0.032 -[ 2023-09-21 08:11:43 ] Completed train batch 1 metrics update 2.208 ms, 7.65 s total -Saving checkpoint at epoch 72 train batch 1 -[ 2023-09-21 08:11:44 ] Completed saving temp checkpoint 408.053 ms, 8.06 s total -[ 2023-09-21 08:11:44 ] Completed replacing temp checkpoint with checkpoint 22.185 ms, 8.08 s total -Epoch [72] :: gen_loss: 0.446, disc_loss: 0.032 -[ 2023-09-21 08:11:44 ] Completed training generator for epoch 72 50.336 ms, 8.14 s total -[ 2023-09-21 08:11:44 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 72 from eval step 0 - -[ 2023-09-21 08:11:45 ] Completed eval batch 0 to device 1,195.798 ms, 1.20 s total -[ 2023-09-21 08:11:45 ] Completed eval batch 0 forward 24.609 ms, 1.22 s total -[ 2023-09-21 08:11:45 ] Completed eval batch 0 recons_loss 0.268 ms, 1.22 s total -[ 2023-09-21 08:11:45 ] Completed eval batch 0 metrics update 0.568 ms, 1.22 s total -Saving checkpoint at epoch 72 val batch 0 -[ 2023-09-21 08:11:45 ] Completed saving temp checkpoint 396.244 ms, 1.62 s total -[ 2023-09-21 08:11:45 ] Completed replacing temp checkpoint with checkpoint 23.093 ms, 1.64 s total -[ 2023-09-21 08:11:46 ] Completed eval batch 1 to device 545.331 ms, 2.19 s total -[ 2023-09-21 08:11:46 ] Completed eval batch 1 forward 39.984 ms, 2.23 s total -[ 2023-09-21 08:11:46 ] Completed eval batch 1 recons_loss 0.437 ms, 2.23 s total -[ 2023-09-21 08:11:46 ] Completed eval batch 1 metrics update 67.760 ms, 2.29 s total -Saving checkpoint at epoch 72 val batch 1 -Epoch 72 val loss: 0.0794 -[ 2023-09-21 08:11:46 ] Completed saving temp checkpoint 412.818 ms, 2.71 s total -[ 2023-09-21 08:11:46 ] Completed replacing temp checkpoint with checkpoint 22.298 ms, 2.73 s total -[ 2023-09-21 08:11:46 ] Completed evaluating generator for epoch 72 50.466 ms, 2.78 s total - - -EPOCH :: 73 - - -[ 2023-09-21 08:11:46 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 73 from training step 0 - -[ 2023-09-21 08:11:50 ] Completed train batch 0 to device 3,181.154 ms, 3.18 s total -[ 2023-09-21 08:11:50 ] Completed train batch 0 generator forward 26.084 ms, 3.21 s total -[ 2023-09-21 08:11:50 ] Completed train batch 0 generator loss: 1.325 66.549 ms, 3.27 s total -[ 2023-09-21 08:11:50 ] Completed train batch 0 generator backward 483.546 ms, 3.76 s total -[ 2023-09-21 08:11:50 ] Completed train batch 0 discriminator loss 0.098 59.189 ms, 3.82 s total -[ 2023-09-21 08:11:50 ] Completed train batch 0 discriminator backward 96.352 ms, 3.91 s total -Epoch [73] Step [0/2], gen_loss: 0.443, disc_loss: 0.033 -[ 2023-09-21 08:11:50 ] Completed train batch 0 metrics update 5.581 ms, 3.92 s total -Saving checkpoint at epoch 73 train batch 0 -[ 2023-09-21 08:11:51 ] Completed saving temp checkpoint 460.586 ms, 4.38 s total -[ 2023-09-21 08:11:51 ] Completed replacing temp checkpoint with checkpoint 34.043 ms, 4.41 s total -[ 2023-09-21 08:11:53 ] Completed train batch 1 to device 2,106.484 ms, 6.52 s total -[ 2023-09-21 08:11:53 ] Completed train batch 1 generator forward 30.064 ms, 6.55 s total -[ 2023-09-21 08:11:53 ] Completed train batch 1 generator loss: 1.291 64.351 ms, 6.61 s total -[ 2023-09-21 08:11:54 ] Completed train batch 1 generator backward 661.737 ms, 7.28 s total -[ 2023-09-21 08:11:54 ] Completed train batch 1 discriminator loss 0.098 56.516 ms, 7.33 s total -[ 2023-09-21 08:11:54 ] Completed train batch 1 discriminator backward 88.479 ms, 7.42 s total -Epoch [73] Step [1/2], gen_loss: 0.442, disc_loss: 0.033 -[ 2023-09-21 08:11:54 ] Completed train batch 1 metrics update 5.269 ms, 7.43 s total -Saving checkpoint at epoch 73 train batch 1 -[ 2023-09-21 08:11:54 ] Completed saving temp checkpoint 455.297 ms, 7.88 s total -[ 2023-09-21 08:11:54 ] Completed replacing temp checkpoint with checkpoint 26.319 ms, 7.91 s total -Epoch [73] :: gen_loss: 0.442, disc_loss: 0.033 -[ 2023-09-21 08:11:54 ] Completed training generator for epoch 73 53.993 ms, 7.96 s total -[ 2023-09-21 08:11:54 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 73 from eval step 0 - -[ 2023-09-21 08:11:56 ] Completed eval batch 0 to device 1,113.735 ms, 1.11 s total -[ 2023-09-21 08:11:56 ] Completed eval batch 0 forward 24.830 ms, 1.14 s total -[ 2023-09-21 08:11:56 ] Completed eval batch 0 recons_loss 0.390 ms, 1.14 s total -[ 2023-09-21 08:11:56 ] Completed eval batch 0 metrics update 0.711 ms, 1.14 s total -Saving checkpoint at epoch 73 val batch 0 -[ 2023-09-21 08:11:56 ] Completed saving temp checkpoint 384.825 ms, 1.52 s total -[ 2023-09-21 08:11:56 ] Completed replacing temp checkpoint with checkpoint 30.683 ms, 1.56 s total -[ 2023-09-21 08:11:56 ] Completed eval batch 1 to device 460.177 ms, 2.02 s total -[ 2023-09-21 08:11:56 ] Completed eval batch 1 forward 21.869 ms, 2.04 s total -[ 2023-09-21 08:11:56 ] Completed eval batch 1 recons_loss 0.346 ms, 2.04 s total -[ 2023-09-21 08:11:57 ] Completed eval batch 1 metrics update 287.086 ms, 2.32 s total -Saving checkpoint at epoch 73 val batch 1 -Epoch 73 val loss: 0.0816 -[ 2023-09-21 08:11:57 ] Completed saving temp checkpoint 429.478 ms, 2.75 s total -[ 2023-09-21 08:11:57 ] Completed replacing temp checkpoint with checkpoint 20.118 ms, 2.77 s total -[ 2023-09-21 08:11:57 ] Completed evaluating generator for epoch 73 48.632 ms, 2.82 s total - - -EPOCH :: 74 - - -[ 2023-09-21 08:11:57 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 74 from training step 0 - -[ 2023-09-21 08:12:00 ] Completed train batch 0 to device 3,226.211 ms, 3.23 s total -[ 2023-09-21 08:12:01 ] Completed train batch 0 generator forward 32.484 ms, 3.26 s total -[ 2023-09-21 08:12:01 ] Completed train batch 0 generator loss: 1.296 66.538 ms, 3.33 s total -[ 2023-09-21 08:12:01 ] Completed train batch 0 generator backward 565.629 ms, 3.89 s total -[ 2023-09-21 08:12:01 ] Completed train batch 0 discriminator loss 0.092 57.858 ms, 3.95 s total -[ 2023-09-21 08:12:01 ] Completed train batch 0 discriminator backward 105.334 ms, 4.05 s total -Epoch [74] Step [0/2], gen_loss: 0.433, disc_loss: 0.031 -[ 2023-09-21 08:12:01 ] Completed train batch 0 metrics update 21.263 ms, 4.08 s total -Saving checkpoint at epoch 74 train batch 0 -[ 2023-09-21 08:12:02 ] Completed saving temp checkpoint 421.005 ms, 4.50 s total -[ 2023-09-21 08:12:02 ] Completed replacing temp checkpoint with checkpoint 21.021 ms, 4.52 s total -[ 2023-09-21 08:12:03 ] Completed train batch 1 to device 1,716.890 ms, 6.23 s total -[ 2023-09-21 08:12:04 ] Completed train batch 1 generator forward 22.992 ms, 6.26 s total -[ 2023-09-21 08:12:04 ] Completed train batch 1 generator loss: 1.431 64.256 ms, 6.32 s total -[ 2023-09-21 08:12:05 ] Completed train batch 1 generator backward 1,025.352 ms, 7.35 s total -[ 2023-09-21 08:12:05 ] Completed train batch 1 discriminator loss 0.088 68.942 ms, 7.42 s total -[ 2023-09-21 08:12:05 ] Completed train batch 1 discriminator backward 61.145 ms, 7.48 s total -Epoch [74] Step [1/2], gen_loss: 0.438, disc_loss: 0.030 -[ 2023-09-21 08:12:05 ] Completed train batch 1 metrics update 2.128 ms, 7.48 s total -Saving checkpoint at epoch 74 train batch 1 -[ 2023-09-21 08:12:05 ] Completed saving temp checkpoint 431.521 ms, 7.91 s total -[ 2023-09-21 08:12:05 ] Completed replacing temp checkpoint with checkpoint 22.538 ms, 7.93 s total -Epoch [74] :: gen_loss: 0.435, disc_loss: 0.030 -[ 2023-09-21 08:12:05 ] Completed training generator for epoch 74 54.696 ms, 7.99 s total -[ 2023-09-21 08:12:05 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 74 from eval step 0 - -[ 2023-09-21 08:12:06 ] Completed eval batch 0 to device 1,143.475 ms, 1.14 s total -[ 2023-09-21 08:12:06 ] Completed eval batch 0 forward 25.599 ms, 1.17 s total -[ 2023-09-21 08:12:06 ] Completed eval batch 0 recons_loss 0.346 ms, 1.17 s total -[ 2023-09-21 08:12:06 ] Completed eval batch 0 metrics update 0.693 ms, 1.17 s total -Saving checkpoint at epoch 74 val batch 0 -[ 2023-09-21 08:12:07 ] Completed saving temp checkpoint 434.375 ms, 1.60 s total -[ 2023-09-21 08:12:07 ] Completed replacing temp checkpoint with checkpoint 33.352 ms, 1.64 s total -[ 2023-09-21 08:12:07 ] Completed eval batch 1 to device 457.752 ms, 2.10 s total -[ 2023-09-21 08:12:07 ] Completed eval batch 1 forward 22.208 ms, 2.12 s total -[ 2023-09-21 08:12:07 ] Completed eval batch 1 recons_loss 0.350 ms, 2.12 s total -[ 2023-09-21 08:12:07 ] Completed eval batch 1 metrics update 100.968 ms, 2.22 s total -Saving checkpoint at epoch 74 val batch 1 -Epoch 74 val loss: 0.0658 -[ 2023-09-21 08:12:08 ] Completed saving temp checkpoint 361.060 ms, 2.58 s total -[ 2023-09-21 08:12:08 ] Completed replacing temp checkpoint with checkpoint 25.610 ms, 2.61 s total -[ 2023-09-21 08:12:08 ] Completed evaluating generator for epoch 74 55.727 ms, 2.66 s total - - -EPOCH :: 75 - - -[ 2023-09-21 08:12:08 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 75 from training step 0 - -[ 2023-09-21 08:12:11 ] Completed train batch 0 to device 2,934.395 ms, 2.93 s total -[ 2023-09-21 08:12:11 ] Completed train batch 0 generator forward 26.872 ms, 2.96 s total -[ 2023-09-21 08:12:11 ] Completed train batch 0 generator loss: 1.286 66.668 ms, 3.03 s total -[ 2023-09-21 08:12:12 ] Completed train batch 0 generator backward 947.278 ms, 3.98 s total -[ 2023-09-21 08:12:12 ] Completed train batch 0 discriminator loss 0.090 62.886 ms, 4.04 s total -[ 2023-09-21 08:12:12 ] Completed train batch 0 discriminator backward 90.037 ms, 4.13 s total -Epoch [75] Step [0/2], gen_loss: 0.436, disc_loss: 0.029 -[ 2023-09-21 08:12:12 ] Completed train batch 0 metrics update 4.243 ms, 4.13 s total -Saving checkpoint at epoch 75 train batch 0 -[ 2023-09-21 08:12:12 ] Completed saving temp checkpoint 428.024 ms, 4.56 s total -[ 2023-09-21 08:12:12 ] Completed replacing temp checkpoint with checkpoint 31.415 ms, 4.59 s total -[ 2023-09-21 08:12:14 ] Completed train batch 1 to device 1,023.425 ms, 5.62 s total -[ 2023-09-21 08:12:14 ] Completed train batch 1 generator forward 23.913 ms, 5.64 s total -[ 2023-09-21 08:12:14 ] Completed train batch 1 generator loss: 1.365 65.011 ms, 5.70 s total -[ 2023-09-21 08:12:16 ] Completed train batch 1 generator backward 1,945.294 ms, 7.65 s total -[ 2023-09-21 08:12:16 ] Completed train batch 1 discriminator loss 0.081 56.625 ms, 7.71 s total -[ 2023-09-21 08:12:16 ] Completed train batch 1 discriminator backward 74.133 ms, 7.78 s total -Epoch [75] Step [1/2], gen_loss: 0.447, disc_loss: 0.027 -[ 2023-09-21 08:12:16 ] Completed train batch 1 metrics update 1.367 ms, 7.78 s total -Saving checkpoint at epoch 75 train batch 1 -[ 2023-09-21 08:12:16 ] Completed saving temp checkpoint 452.463 ms, 8.23 s total -[ 2023-09-21 08:12:16 ] Completed replacing temp checkpoint with checkpoint 23.898 ms, 8.26 s total -Epoch [75] :: gen_loss: 0.441, disc_loss: 0.028 -[ 2023-09-21 08:12:16 ] Completed training generator for epoch 75 52.999 ms, 8.31 s total -[ 2023-09-21 08:12:16 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 75 from eval step 0 - -[ 2023-09-21 08:12:17 ] Completed eval batch 0 to device 1,163.130 ms, 1.16 s total -[ 2023-09-21 08:12:17 ] Completed eval batch 0 forward 41.380 ms, 1.20 s total -[ 2023-09-21 08:12:17 ] Completed eval batch 0 recons_loss 0.296 ms, 1.20 s total -[ 2023-09-21 08:12:17 ] Completed eval batch 0 metrics update 0.627 ms, 1.21 s total -Saving checkpoint at epoch 75 val batch 0 -[ 2023-09-21 08:12:18 ] Completed saving temp checkpoint 405.962 ms, 1.61 s total -[ 2023-09-21 08:12:18 ] Completed replacing temp checkpoint with checkpoint 33.534 ms, 1.64 s total -[ 2023-09-21 08:12:18 ] Completed eval batch 1 to device 375.302 ms, 2.02 s total -[ 2023-09-21 08:12:18 ] Completed eval batch 1 forward 21.859 ms, 2.04 s total -[ 2023-09-21 08:12:18 ] Completed eval batch 1 recons_loss 0.333 ms, 2.04 s total -[ 2023-09-21 08:12:18 ] Completed eval batch 1 metrics update 185.881 ms, 2.23 s total -Saving checkpoint at epoch 75 val batch 1 -Epoch 75 val loss: 0.0618 -[ 2023-09-21 08:12:19 ] Completed saving temp checkpoint 435.086 ms, 2.66 s total -[ 2023-09-21 08:12:19 ] Completed replacing temp checkpoint with checkpoint 33.545 ms, 2.70 s total -[ 2023-09-21 08:12:19 ] Completed evaluating generator for epoch 75 55.952 ms, 2.75 s total - - -EPOCH :: 76 - - -[ 2023-09-21 08:12:19 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 76 from training step 0 - -[ 2023-09-21 08:12:22 ] Completed train batch 0 to device 3,014.040 ms, 3.01 s total -[ 2023-09-21 08:12:22 ] Completed train batch 0 generator forward 26.714 ms, 3.04 s total -[ 2023-09-21 08:12:22 ] Completed train batch 0 generator loss: 1.285 66.636 ms, 3.11 s total -[ 2023-09-21 08:12:23 ] Completed train batch 0 generator backward 643.326 ms, 3.75 s total -[ 2023-09-21 08:12:23 ] Completed train batch 0 discriminator loss 0.081 56.926 ms, 3.81 s total -[ 2023-09-21 08:12:23 ] Completed train batch 0 discriminator backward 106.519 ms, 3.91 s total -Epoch [76] Step [0/2], gen_loss: 0.428, disc_loss: 0.027 -[ 2023-09-21 08:12:23 ] Completed train batch 0 metrics update 3.109 ms, 3.92 s total -Saving checkpoint at epoch 76 train batch 0 -[ 2023-09-21 08:12:23 ] Completed saving temp checkpoint 423.190 ms, 4.34 s total -[ 2023-09-21 08:12:23 ] Completed replacing temp checkpoint with checkpoint 29.092 ms, 4.37 s total -[ 2023-09-21 08:12:25 ] Completed train batch 1 to device 1,385.659 ms, 5.76 s total -[ 2023-09-21 08:12:25 ] Completed train batch 1 generator forward 39.103 ms, 5.79 s total -[ 2023-09-21 08:12:25 ] Completed train batch 1 generator loss: 1.448 63.654 ms, 5.86 s total -[ 2023-09-21 08:12:26 ] Completed train batch 1 generator backward 1,261.234 ms, 7.12 s total -[ 2023-09-21 08:12:26 ] Completed train batch 1 discriminator loss 0.080 56.463 ms, 7.18 s total -[ 2023-09-21 08:12:26 ] Completed train batch 1 discriminator backward 72.639 ms, 7.25 s total -Epoch [76] Step [1/2], gen_loss: 0.465, disc_loss: 0.027 -[ 2023-09-21 08:12:26 ] Completed train batch 1 metrics update 1.084 ms, 7.25 s total -Saving checkpoint at epoch 76 train batch 1 -[ 2023-09-21 08:12:27 ] Completed saving temp checkpoint 464.993 ms, 7.71 s total -[ 2023-09-21 08:12:27 ] Completed replacing temp checkpoint with checkpoint 24.421 ms, 7.74 s total -Epoch [76] :: gen_loss: 0.446, disc_loss: 0.027 -[ 2023-09-21 08:12:27 ] Completed training generator for epoch 76 57.006 ms, 7.80 s total -[ 2023-09-21 08:12:27 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 76 from eval step 0 - -[ 2023-09-21 08:12:28 ] Completed eval batch 0 to device 1,171.629 ms, 1.17 s total -[ 2023-09-21 08:12:28 ] Completed eval batch 0 forward 42.575 ms, 1.21 s total -[ 2023-09-21 08:12:28 ] Completed eval batch 0 recons_loss 0.318 ms, 1.21 s total -[ 2023-09-21 08:12:28 ] Completed eval batch 0 metrics update 0.554 ms, 1.22 s total -Saving checkpoint at epoch 76 val batch 0 -[ 2023-09-21 08:12:28 ] Completed saving temp checkpoint 457.609 ms, 1.67 s total -[ 2023-09-21 08:12:28 ] Completed replacing temp checkpoint with checkpoint 37.383 ms, 1.71 s total -[ 2023-09-21 08:12:29 ] Completed eval batch 1 to device 347.045 ms, 2.06 s total -[ 2023-09-21 08:12:29 ] Completed eval batch 1 forward 40.310 ms, 2.10 s total -[ 2023-09-21 08:12:29 ] Completed eval batch 1 recons_loss 0.519 ms, 2.10 s total -[ 2023-09-21 08:12:29 ] Completed eval batch 1 metrics update 131.453 ms, 2.23 s total -Saving checkpoint at epoch 76 val batch 1 -Epoch 76 val loss: 0.0715 -[ 2023-09-21 08:12:30 ] Completed saving temp checkpoint 539.074 ms, 2.77 s total -[ 2023-09-21 08:12:30 ] Completed replacing temp checkpoint with checkpoint 33.298 ms, 2.80 s total -[ 2023-09-21 08:12:30 ] Completed evaluating generator for epoch 76 72.310 ms, 2.87 s total - - -EPOCH :: 77 - - -[ 2023-09-21 08:12:30 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 77 from training step 0 - -[ 2023-09-21 08:12:33 ] Completed train batch 0 to device 3,064.954 ms, 3.06 s total -[ 2023-09-21 08:12:33 ] Completed train batch 0 generator forward 26.080 ms, 3.09 s total -[ 2023-09-21 08:12:33 ] Completed train batch 0 generator loss: 1.278 65.871 ms, 3.16 s total -[ 2023-09-21 08:12:33 ] Completed train batch 0 generator backward 491.118 ms, 3.65 s total -[ 2023-09-21 08:12:33 ] Completed train batch 0 discriminator loss 0.103 76.216 ms, 3.72 s total -[ 2023-09-21 08:12:33 ] Completed train batch 0 discriminator backward 69.385 ms, 3.79 s total -Epoch [77] Step [0/2], gen_loss: 0.431, disc_loss: 0.034 -[ 2023-09-21 08:12:33 ] Completed train batch 0 metrics update 7.152 ms, 3.80 s total -Saving checkpoint at epoch 77 train batch 0 -[ 2023-09-21 08:12:34 ] Completed saving temp checkpoint 603.802 ms, 4.40 s total -[ 2023-09-21 08:12:34 ] Completed replacing temp checkpoint with checkpoint 38.718 ms, 4.44 s total -[ 2023-09-21 08:12:36 ] Completed train batch 1 to device 1,508.849 ms, 5.95 s total -[ 2023-09-21 08:12:36 ] Completed train batch 1 generator forward 26.731 ms, 5.98 s total -[ 2023-09-21 08:12:36 ] Completed train batch 1 generator loss: 1.314 64.571 ms, 6.04 s total -[ 2023-09-21 08:12:37 ] Completed train batch 1 generator backward 1,001.168 ms, 7.04 s total -[ 2023-09-21 08:12:37 ] Completed train batch 1 discriminator loss 0.107 56.452 ms, 7.10 s total -[ 2023-09-21 08:12:37 ] Completed train batch 1 discriminator backward 72.795 ms, 7.17 s total -Epoch [77] Step [1/2], gen_loss: 0.442, disc_loss: 0.036 -[ 2023-09-21 08:12:37 ] Completed train batch 1 metrics update 0.880 ms, 7.17 s total -Saving checkpoint at epoch 77 train batch 1 -[ 2023-09-21 08:12:37 ] Completed saving temp checkpoint 458.492 ms, 7.63 s total -[ 2023-09-21 08:12:37 ] Completed replacing temp checkpoint with checkpoint 28.692 ms, 7.66 s total -Epoch [77] :: gen_loss: 0.436, disc_loss: 0.035 -[ 2023-09-21 08:12:37 ] Completed training generator for epoch 77 57.193 ms, 7.72 s total -[ 2023-09-21 08:12:37 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 77 from eval step 0 - -[ 2023-09-21 08:12:39 ] Completed eval batch 0 to device 1,339.295 ms, 1.34 s total -[ 2023-09-21 08:12:39 ] Completed eval batch 0 forward 43.233 ms, 1.38 s total -[ 2023-09-21 08:12:39 ] Completed eval batch 0 recons_loss 0.298 ms, 1.38 s total -[ 2023-09-21 08:12:39 ] Completed eval batch 0 metrics update 0.513 ms, 1.38 s total -Saving checkpoint at epoch 77 val batch 0 -[ 2023-09-21 08:12:39 ] Completed saving temp checkpoint 531.855 ms, 1.92 s total -[ 2023-09-21 08:12:39 ] Completed replacing temp checkpoint with checkpoint 32.410 ms, 1.95 s total -[ 2023-09-21 08:12:40 ] Completed eval batch 1 to device 240.498 ms, 2.19 s total -[ 2023-09-21 08:12:40 ] Completed eval batch 1 forward 40.450 ms, 2.23 s total -[ 2023-09-21 08:12:40 ] Completed eval batch 1 recons_loss 0.579 ms, 2.23 s total -[ 2023-09-21 08:12:40 ] Completed eval batch 1 metrics update 0.318 ms, 2.23 s total -Saving checkpoint at epoch 77 val batch 1 -Epoch 77 val loss: 0.0733 -[ 2023-09-21 08:12:40 ] Completed saving temp checkpoint 406.553 ms, 2.64 s total -[ 2023-09-21 08:12:40 ] Completed replacing temp checkpoint with checkpoint 22.236 ms, 2.66 s total -[ 2023-09-21 08:12:40 ] Completed evaluating generator for epoch 77 61.369 ms, 2.72 s total - - -EPOCH :: 78 - - -[ 2023-09-21 08:12:40 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 78 from training step 0 - -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 08:23:28 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 08:23:28 ] Completed importing Timer 0.034 ms, 0.00 s total -[ 2023-09-21 08:23:32 ] Completed importing everything else 4,605.591 ms, 4.61 s total -| distributed init (rank 0): env:// -| distributed init (rank 4): env:// -| distributed init (rank 5): env:// -| distributed init (rank 3): env:// -| distributed init (rank 2): env:// -| distributed init (rank 1): env:// -[ 2023-09-21 08:23:40 ] Completed preliminaries 7,903.409 ms, 12.51 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 08:23:40 ] Completed build datasets 10.761 ms, 12.52 s total -[ 2023-09-21 08:23:40 ] Completed build samplers 0.041 ms, 12.52 s total -[ 2023-09-21 08:23:40 ] Completed build dataloaders 0.131 ms, 12.52 s total -[ 2023-09-21 08:23:40 ] Completed generator to device 95.443 ms, 12.62 s total -[ 2023-09-21 08:23:41 ] Completed discriminator to device 7.251 ms, 12.62 s total -[ 2023-09-21 08:23:41 ] Completed loss functions 371.143 ms, 12.99 s total -[ 2023-09-21 08:23:41 ] Completed models prepped for distribution 90.552 ms, 13.08 s total -[ 2023-09-21 08:23:41 ] Completed optimizers 0.664 ms, 13.09 s total -[ 2023-09-21 08:23:41 ] Completed grad scalers 0.024 ms, 13.09 s total -[ 2023-09-21 08:23:41 ] Completed checkpoint retrieval 451.235 ms, 13.54 s total - - -EPOCH :: 77 - - -[ 2023-09-21 08:23:41 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 77 from training step 2 - -Epoch [77] :: gen_loss: 0.436, disc_loss: 0.035 -[ 2023-09-21 08:23:42 ] Completed training generator for epoch 77 133.966 ms, 0.13 s total -[ 2023-09-21 08:23:42 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 77 from eval step 2 - -[ 2023-09-21 08:23:42 ] Completed evaluating generator for epoch 77 134.040 ms, 0.13 s total - - -EPOCH :: 78 - - -[ 2023-09-21 08:23:42 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 78 from training step 0 - -[ 2023-09-21 08:23:46 ] Completed train batch 0 to device 3,827.840 ms, 3.83 s total -[ 2023-09-21 08:23:47 ] Completed train batch 0 generator forward 1,532.256 ms, 5.36 s total -[ 2023-09-21 08:23:47 ] Completed train batch 0 generator loss: 1.284 75.993 ms, 5.44 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 08:23:48 ] Completed train batch 0 generator backward 812.561 ms, 6.25 s total -[ 2023-09-21 08:23:48 ] Completed train batch 0 discriminator loss 0.087 58.532 ms, 6.31 s total -[ 2023-09-21 08:23:48 ] Completed train batch 0 discriminator backward 112.880 ms, 6.42 s total -Epoch [78] Step [0/2], gen_loss: 0.428, disc_loss: 0.030 -[ 2023-09-21 08:23:48 ] Completed train batch 0 metrics update 15.599 ms, 6.44 s total -Saving checkpoint at epoch 78 train batch 0 -[ 2023-09-21 08:23:49 ] Completed saving temp checkpoint 475.398 ms, 6.91 s total -[ 2023-09-21 08:23:49 ] Completed replacing temp checkpoint with checkpoint 132.559 ms, 7.04 s total -[ 2023-09-21 08:23:49 ] Completed train batch 1 to device 133.143 ms, 7.18 s total -[ 2023-09-21 08:23:49 ] Completed train batch 1 generator forward 40.847 ms, 7.22 s total -[ 2023-09-21 08:23:49 ] Completed train batch 1 generator loss: 1.290 63.243 ms, 7.28 s total -[ 2023-09-21 08:23:50 ] Completed train batch 1 generator backward 971.942 ms, 8.25 s total -[ 2023-09-21 08:23:50 ] Completed train batch 1 discriminator loss 0.080 56.534 ms, 8.31 s total -[ 2023-09-21 08:23:50 ] Completed train batch 1 discriminator backward 72.372 ms, 8.38 s total -Epoch [78] Step [1/2], gen_loss: 0.440, disc_loss: 0.028 -[ 2023-09-21 08:23:50 ] Completed train batch 1 metrics update 0.658 ms, 8.38 s total -Saving checkpoint at epoch 78 train batch 1 -[ 2023-09-21 08:23:51 ] Completed saving temp checkpoint 1,045.206 ms, 9.43 s total -[ 2023-09-21 08:23:51 ] Completed replacing temp checkpoint with checkpoint 25.958 ms, 9.45 s total -Epoch [78] :: gen_loss: 0.434, disc_loss: 0.029 -[ 2023-09-21 08:23:51 ] Completed training generator for epoch 78 40.824 ms, 9.49 s total -[ 2023-09-21 08:23:51 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 78 from eval step 0 - -[ 2023-09-21 08:23:52 ] Completed eval batch 0 to device 1,242.060 ms, 1.24 s total -[ 2023-09-21 08:23:52 ] Completed eval batch 0 forward 29.650 ms, 1.27 s total -[ 2023-09-21 08:23:52 ] Completed eval batch 0 recons_loss 0.289 ms, 1.27 s total -[ 2023-09-21 08:23:52 ] Completed eval batch 0 metrics update 0.590 ms, 1.27 s total -Saving checkpoint at epoch 78 val batch 0 -[ 2023-09-21 08:23:53 ] Completed saving temp checkpoint 922.263 ms, 2.19 s total -[ 2023-09-21 08:23:53 ] Completed replacing temp checkpoint with checkpoint 22.613 ms, 2.22 s total -[ 2023-09-21 08:23:53 ] Completed eval batch 1 to device 14.402 ms, 2.23 s total -[ 2023-09-21 08:23:53 ] Completed eval batch 1 forward 23.463 ms, 2.26 s total -[ 2023-09-21 08:23:53 ] Completed eval batch 1 recons_loss 0.337 ms, 2.26 s total -[ 2023-09-21 08:23:53 ] Completed eval batch 1 metrics update 0.300 ms, 2.26 s total -Saving checkpoint at epoch 78 val batch 1 -Epoch 78 val loss: 0.0694 -[ 2023-09-21 08:23:54 ] Completed saving temp checkpoint 979.155 ms, 3.24 s total -[ 2023-09-21 08:23:54 ] Completed replacing temp checkpoint with checkpoint 26.288 ms, 3.26 s total -[ 2023-09-21 08:23:55 ] Completed evaluating generator for epoch 78 56.873 ms, 3.32 s total - - -EPOCH :: 79 - - -[ 2023-09-21 08:23:55 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 79 from training step 0 - -[ 2023-09-21 08:23:57 ] Completed train batch 0 to device 2,884.299 ms, 2.88 s total -[ 2023-09-21 08:23:57 ] Completed train batch 0 generator forward 29.973 ms, 2.91 s total -[ 2023-09-21 08:23:57 ] Completed train batch 0 generator loss: 1.306 65.165 ms, 2.98 s total -[ 2023-09-21 08:23:58 ] Completed train batch 0 generator backward 269.149 ms, 3.25 s total -[ 2023-09-21 08:23:58 ] Completed train batch 0 discriminator loss 0.083 57.814 ms, 3.31 s total -[ 2023-09-21 08:23:58 ] Completed train batch 0 discriminator backward 81.217 ms, 3.39 s total -Epoch [79] Step [0/2], gen_loss: 0.441, disc_loss: 0.028 -[ 2023-09-21 08:23:58 ] Completed train batch 0 metrics update 2.051 ms, 3.39 s total -Saving checkpoint at epoch 79 train batch 0 -[ 2023-09-21 08:23:59 ] Completed saving temp checkpoint 966.520 ms, 4.36 s total -[ 2023-09-21 08:23:59 ] Completed replacing temp checkpoint with checkpoint 21.086 ms, 4.38 s total -[ 2023-09-21 08:24:00 ] Completed train batch 1 to device 1,233.560 ms, 5.61 s total -[ 2023-09-21 08:24:00 ] Completed train batch 1 generator forward 40.966 ms, 5.65 s total -[ 2023-09-21 08:24:00 ] Completed train batch 1 generator loss: 1.326 63.604 ms, 5.72 s total -[ 2023-09-21 08:24:01 ] Completed train batch 1 generator backward 922.803 ms, 6.64 s total -[ 2023-09-21 08:24:01 ] Completed train batch 1 discriminator loss 0.077 57.170 ms, 6.70 s total -[ 2023-09-21 08:24:01 ] Completed train batch 1 discriminator backward 83.720 ms, 6.78 s total -Epoch [79] Step [1/2], gen_loss: 0.455, disc_loss: 0.026 -[ 2023-09-21 08:24:01 ] Completed train batch 1 metrics update 0.530 ms, 6.78 s total -Saving checkpoint at epoch 79 train batch 1 -[ 2023-09-21 08:24:02 ] Completed saving temp checkpoint 989.977 ms, 7.77 s total -[ 2023-09-21 08:24:02 ] Completed replacing temp checkpoint with checkpoint 38.489 ms, 7.81 s total -Epoch [79] :: gen_loss: 0.448, disc_loss: 0.027 -[ 2023-09-21 08:24:02 ] Completed training generator for epoch 79 53.137 ms, 7.86 s total -[ 2023-09-21 08:24:02 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 79 from eval step 0 - -[ 2023-09-21 08:24:04 ] Completed eval batch 0 to device 1,336.194 ms, 1.34 s total -[ 2023-09-21 08:24:04 ] Completed eval batch 0 forward 39.059 ms, 1.38 s total -[ 2023-09-21 08:24:04 ] Completed eval batch 0 recons_loss 0.226 ms, 1.38 s total -[ 2023-09-21 08:24:04 ] Completed eval batch 0 metrics update 0.516 ms, 1.38 s total -Saving checkpoint at epoch 79 val batch 0 -[ 2023-09-21 08:24:05 ] Completed saving temp checkpoint 919.627 ms, 2.30 s total -[ 2023-09-21 08:24:05 ] Completed replacing temp checkpoint with checkpoint 24.222 ms, 2.32 s total -[ 2023-09-21 08:24:05 ] Completed eval batch 1 to device 14.630 ms, 2.33 s total -[ 2023-09-21 08:24:05 ] Completed eval batch 1 forward 21.765 ms, 2.36 s total -[ 2023-09-21 08:24:05 ] Completed eval batch 1 recons_loss 0.349 ms, 2.36 s total -[ 2023-09-21 08:24:05 ] Completed eval batch 1 metrics update 0.325 ms, 2.36 s total -Saving checkpoint at epoch 79 val batch 1 -Epoch 79 val loss: 0.0730 -[ 2023-09-21 08:24:06 ] Completed saving temp checkpoint 981.806 ms, 3.34 s total -[ 2023-09-21 08:24:06 ] Completed replacing temp checkpoint with checkpoint 29.803 ms, 3.37 s total -[ 2023-09-21 08:24:06 ] Completed evaluating generator for epoch 79 55.828 ms, 3.42 s total - - -EPOCH :: 80 - - -[ 2023-09-21 08:24:06 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 80 from training step 0 - -[ 2023-09-21 08:24:09 ] Completed train batch 0 to device 3,560.598 ms, 3.56 s total -[ 2023-09-21 08:24:09 ] Completed train batch 0 generator forward 30.196 ms, 3.59 s total -[ 2023-09-21 08:24:09 ] Completed train batch 0 generator loss: 1.329 65.396 ms, 3.66 s total -[ 2023-09-21 08:24:10 ] Completed train batch 0 generator backward 144.300 ms, 3.80 s total -[ 2023-09-21 08:24:10 ] Completed train batch 0 discriminator loss 0.074 56.959 ms, 3.86 s total -[ 2023-09-21 08:24:10 ] Completed train batch 0 discriminator backward 93.631 ms, 3.95 s total -Epoch [80] Step [0/2], gen_loss: 0.432, disc_loss: 0.025 -[ 2023-09-21 08:24:10 ] Completed train batch 0 metrics update 4.695 ms, 3.96 s total -Saving checkpoint at epoch 80 train batch 0 -[ 2023-09-21 08:24:10 ] Completed saving temp checkpoint 392.269 ms, 4.35 s total -[ 2023-09-21 08:24:10 ] Completed replacing temp checkpoint with checkpoint 29.280 ms, 4.38 s total -[ 2023-09-21 08:24:12 ] Completed train batch 1 to device 2,197.854 ms, 6.58 s total -[ 2023-09-21 08:24:12 ] Completed train batch 1 generator forward 40.978 ms, 6.62 s total -[ 2023-09-21 08:24:12 ] Completed train batch 1 generator loss: 1.300 64.089 ms, 6.68 s total -[ 2023-09-21 08:24:13 ] Completed train batch 1 generator backward 125.874 ms, 6.81 s total -[ 2023-09-21 08:24:13 ] Completed train batch 1 discriminator loss 0.064 58.102 ms, 6.86 s total -[ 2023-09-21 08:24:13 ] Completed train batch 1 discriminator backward 74.156 ms, 6.94 s total -Epoch [80] Step [1/2], gen_loss: 0.450, disc_loss: 0.023 -[ 2023-09-21 08:24:13 ] Completed train batch 1 metrics update 5.055 ms, 6.94 s total -Saving checkpoint at epoch 80 train batch 1 -[ 2023-09-21 08:24:13 ] Completed saving temp checkpoint 429.206 ms, 7.37 s total -[ 2023-09-21 08:24:13 ] Completed replacing temp checkpoint with checkpoint 25.480 ms, 7.40 s total -Epoch [80] :: gen_loss: 0.441, disc_loss: 0.024 -[ 2023-09-21 08:24:13 ] Completed training generator for epoch 80 53.877 ms, 7.45 s total -[ 2023-09-21 08:24:13 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 80 from eval step 0 - -[ 2023-09-21 08:24:14 ] Completed eval batch 0 to device 1,155.643 ms, 1.16 s total -[ 2023-09-21 08:24:14 ] Completed eval batch 0 forward 25.243 ms, 1.18 s total -[ 2023-09-21 08:24:14 ] Completed eval batch 0 recons_loss 0.353 ms, 1.18 s total -[ 2023-09-21 08:24:14 ] Completed eval batch 0 metrics update 0.641 ms, 1.18 s total -Saving checkpoint at epoch 80 val batch 0 -[ 2023-09-21 08:24:15 ] Completed saving temp checkpoint 326.440 ms, 1.51 s total -[ 2023-09-21 08:24:15 ] Completed replacing temp checkpoint with checkpoint 33.278 ms, 1.54 s total -[ 2023-09-21 08:24:15 ] Completed eval batch 1 to device 538.413 ms, 2.08 s total -[ 2023-09-21 08:24:15 ] Completed eval batch 1 forward 23.520 ms, 2.10 s total -[ 2023-09-21 08:24:15 ] Completed eval batch 1 recons_loss 0.357 ms, 2.10 s total -[ 2023-09-21 08:24:16 ] Completed eval batch 1 metrics update 251.075 ms, 2.35 s total -Saving checkpoint at epoch 80 val batch 1 -Epoch 80 val loss: 0.0816 -[ 2023-09-21 08:24:16 ] Completed saving temp checkpoint 415.822 ms, 2.77 s total -[ 2023-09-21 08:24:16 ] Completed replacing temp checkpoint with checkpoint 20.535 ms, 2.79 s total -[ 2023-09-21 08:24:16 ] Completed evaluating generator for epoch 80 51.350 ms, 2.84 s total - - -EPOCH :: 81 - - -[ 2023-09-21 08:24:16 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 81 from training step 0 - -[ 2023-09-21 08:24:19 ] Completed train batch 0 to device 2,861.946 ms, 2.86 s total -[ 2023-09-21 08:24:19 ] Completed train batch 0 generator forward 25.791 ms, 2.89 s total -[ 2023-09-21 08:24:19 ] Completed train batch 0 generator loss: 1.349 76.612 ms, 2.96 s total -[ 2023-09-21 08:24:20 ] Completed train batch 0 generator backward 817.780 ms, 3.78 s total -[ 2023-09-21 08:24:20 ] Completed train batch 0 discriminator loss 0.069 57.783 ms, 3.84 s total -[ 2023-09-21 08:24:20 ] Completed train batch 0 discriminator backward 107.669 ms, 3.95 s total -Epoch [81] Step [0/2], gen_loss: 0.438, disc_loss: 0.023 -[ 2023-09-21 08:24:20 ] Completed train batch 0 metrics update 7.724 ms, 3.96 s total -Saving checkpoint at epoch 81 train batch 0 -[ 2023-09-21 08:24:20 ] Completed saving temp checkpoint 377.141 ms, 4.33 s total -[ 2023-09-21 08:24:20 ] Completed replacing temp checkpoint with checkpoint 22.075 ms, 4.35 s total -[ 2023-09-21 08:24:22 ] Completed train batch 1 to device 1,294.767 ms, 5.65 s total -[ 2023-09-21 08:24:22 ] Completed train batch 1 generator forward 41.925 ms, 5.69 s total -[ 2023-09-21 08:24:22 ] Completed train batch 1 generator loss: 1.370 63.254 ms, 5.75 s total -[ 2023-09-21 08:24:23 ] Completed train batch 1 generator backward 1,463.059 ms, 7.22 s total -[ 2023-09-21 08:24:23 ] Completed train batch 1 discriminator loss 0.067 56.588 ms, 7.27 s total -[ 2023-09-21 08:24:23 ] Completed train batch 1 discriminator backward 76.091 ms, 7.35 s total -Epoch [81] Step [1/2], gen_loss: 0.449, disc_loss: 0.023 -[ 2023-09-21 08:24:23 ] Completed train batch 1 metrics update 5.704 ms, 7.36 s total -Saving checkpoint at epoch 81 train batch 1 -[ 2023-09-21 08:24:24 ] Completed saving temp checkpoint 365.515 ms, 7.72 s total -[ 2023-09-21 08:24:24 ] Completed replacing temp checkpoint with checkpoint 19.259 ms, 7.74 s total -Epoch [81] :: gen_loss: 0.443, disc_loss: 0.023 -[ 2023-09-21 08:24:24 ] Completed training generator for epoch 81 53.969 ms, 7.79 s total -[ 2023-09-21 08:24:24 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 81 from eval step 0 - -[ 2023-09-21 08:24:25 ] Completed eval batch 0 to device 1,209.739 ms, 1.21 s total -[ 2023-09-21 08:24:25 ] Completed eval batch 0 forward 24.842 ms, 1.23 s total -[ 2023-09-21 08:24:25 ] Completed eval batch 0 recons_loss 0.328 ms, 1.23 s total -[ 2023-09-21 08:24:25 ] Completed eval batch 0 metrics update 0.632 ms, 1.24 s total -Saving checkpoint at epoch 81 val batch 0 -[ 2023-09-21 08:24:25 ] Completed saving temp checkpoint 326.066 ms, 1.56 s total -[ 2023-09-21 08:24:25 ] Completed replacing temp checkpoint with checkpoint 19.708 ms, 1.58 s total -[ 2023-09-21 08:24:26 ] Completed eval batch 1 to device 650.834 ms, 2.23 s total -[ 2023-09-21 08:24:26 ] Completed eval batch 1 forward 22.894 ms, 2.26 s total -[ 2023-09-21 08:24:26 ] Completed eval batch 1 recons_loss 0.443 ms, 2.26 s total -[ 2023-09-21 08:24:26 ] Completed eval batch 1 metrics update 47.785 ms, 2.30 s total -Saving checkpoint at epoch 81 val batch 1 -Epoch 81 val loss: 0.0696 -[ 2023-09-21 08:24:27 ] Completed saving temp checkpoint 428.947 ms, 2.73 s total -[ 2023-09-21 08:24:27 ] Completed replacing temp checkpoint with checkpoint 27.182 ms, 2.76 s total -[ 2023-09-21 08:24:27 ] Completed evaluating generator for epoch 81 53.409 ms, 2.81 s total - - -EPOCH :: 82 - - -[ 2023-09-21 08:24:27 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 82 from training step 0 - -[ 2023-09-21 08:24:30 ] Completed train batch 0 to device 2,839.534 ms, 2.84 s total -[ 2023-09-21 08:24:30 ] Completed train batch 0 generator forward 44.247 ms, 2.88 s total -[ 2023-09-21 08:24:30 ] Completed train batch 0 generator loss: 1.280 66.084 ms, 2.95 s total -[ 2023-09-21 08:24:30 ] Completed train batch 0 generator backward 815.233 ms, 3.77 s total -[ 2023-09-21 08:24:31 ] Completed train batch 0 discriminator loss 0.067 57.360 ms, 3.82 s total -[ 2023-09-21 08:24:31 ] Completed train batch 0 discriminator backward 84.333 ms, 3.91 s total -Epoch [82] Step [0/2], gen_loss: 0.439, disc_loss: 0.023 -[ 2023-09-21 08:24:31 ] Completed train batch 0 metrics update 3.811 ms, 3.91 s total -Saving checkpoint at epoch 82 train batch 0 -[ 2023-09-21 08:24:31 ] Completed saving temp checkpoint 395.677 ms, 4.31 s total -[ 2023-09-21 08:24:31 ] Completed replacing temp checkpoint with checkpoint 24.729 ms, 4.33 s total -[ 2023-09-21 08:24:32 ] Completed train batch 1 to device 1,139.682 ms, 5.47 s total -[ 2023-09-21 08:24:32 ] Completed train batch 1 generator forward 41.393 ms, 5.51 s total -[ 2023-09-21 08:24:32 ] Completed train batch 1 generator loss: 1.361 62.215 ms, 5.57 s total -[ 2023-09-21 08:24:34 ] Completed train batch 1 generator backward 1,686.891 ms, 7.26 s total -[ 2023-09-21 08:24:34 ] Completed train batch 1 discriminator loss 0.073 56.631 ms, 7.32 s total -[ 2023-09-21 08:24:34 ] Completed train batch 1 discriminator backward 71.914 ms, 7.39 s total -Epoch [82] Step [1/2], gen_loss: 0.443, disc_loss: 0.022 -[ 2023-09-21 08:24:34 ] Completed train batch 1 metrics update 1.993 ms, 7.39 s total -Saving checkpoint at epoch 82 train batch 1 -[ 2023-09-21 08:24:35 ] Completed saving temp checkpoint 428.372 ms, 7.82 s total -[ 2023-09-21 08:24:35 ] Completed replacing temp checkpoint with checkpoint 30.879 ms, 7.85 s total -Epoch [82] :: gen_loss: 0.441, disc_loss: 0.023 -[ 2023-09-21 08:24:35 ] Completed training generator for epoch 82 53.851 ms, 7.90 s total -[ 2023-09-21 08:24:35 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 82 from eval step 0 - -[ 2023-09-21 08:24:36 ] Completed eval batch 0 to device 1,081.710 ms, 1.08 s total -[ 2023-09-21 08:24:36 ] Completed eval batch 0 forward 23.689 ms, 1.11 s total -[ 2023-09-21 08:24:36 ] Completed eval batch 0 recons_loss 0.272 ms, 1.11 s total -[ 2023-09-21 08:24:36 ] Completed eval batch 0 metrics update 0.548 ms, 1.11 s total -Saving checkpoint at epoch 82 val batch 0 -[ 2023-09-21 08:24:36 ] Completed saving temp checkpoint 380.521 ms, 1.49 s total -[ 2023-09-21 08:24:36 ] Completed replacing temp checkpoint with checkpoint 29.702 ms, 1.52 s total -[ 2023-09-21 08:24:37 ] Completed eval batch 1 to device 461.535 ms, 1.98 s total -[ 2023-09-21 08:24:37 ] Completed eval batch 1 forward 21.476 ms, 2.00 s total -[ 2023-09-21 08:24:37 ] Completed eval batch 1 recons_loss 0.365 ms, 2.00 s total -[ 2023-09-21 08:24:37 ] Completed eval batch 1 metrics update 221.963 ms, 2.22 s total -Saving checkpoint at epoch 82 val batch 1 -Epoch 82 val loss: 0.0634 -[ 2023-09-21 08:24:37 ] Completed saving temp checkpoint 425.685 ms, 2.65 s total -[ 2023-09-21 08:24:37 ] Completed replacing temp checkpoint with checkpoint 31.340 ms, 2.68 s total -[ 2023-09-21 08:24:37 ] Completed evaluating generator for epoch 82 47.623 ms, 2.73 s total - - -EPOCH :: 83 - - -[ 2023-09-21 08:24:37 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 83 from training step 0 - -[ 2023-09-21 08:24:40 ] Completed train batch 0 to device 3,012.115 ms, 3.01 s total -[ 2023-09-21 08:24:40 ] Completed train batch 0 generator forward 44.323 ms, 3.06 s total -[ 2023-09-21 08:24:40 ] Completed train batch 0 generator loss: 1.388 66.317 ms, 3.12 s total -[ 2023-09-21 08:24:41 ] Completed train batch 0 generator backward 613.751 ms, 3.74 s total -[ 2023-09-21 08:24:41 ] Completed train batch 0 discriminator loss 0.069 57.614 ms, 3.79 s total -[ 2023-09-21 08:24:41 ] Completed train batch 0 discriminator backward 104.452 ms, 3.90 s total -Epoch [83] Step [0/2], gen_loss: 0.453, disc_loss: 0.022 -[ 2023-09-21 08:24:41 ] Completed train batch 0 metrics update 16.740 ms, 3.92 s total -Saving checkpoint at epoch 83 train batch 0 -[ 2023-09-21 08:24:42 ] Completed saving temp checkpoint 438.755 ms, 4.35 s total -[ 2023-09-21 08:24:42 ] Completed replacing temp checkpoint with checkpoint 22.632 ms, 4.38 s total -[ 2023-09-21 08:24:43 ] Completed train batch 1 to device 1,595.004 ms, 5.97 s total -[ 2023-09-21 08:24:43 ] Completed train batch 1 generator forward 22.069 ms, 5.99 s total -[ 2023-09-21 08:24:43 ] Completed train batch 1 generator loss: 1.346 63.175 ms, 6.06 s total -[ 2023-09-21 08:24:44 ] Completed train batch 1 generator backward 943.995 ms, 7.00 s total -[ 2023-09-21 08:24:44 ] Completed train batch 1 discriminator loss 0.065 56.361 ms, 7.06 s total -[ 2023-09-21 08:24:44 ] Completed train batch 1 discriminator backward 88.012 ms, 7.15 s total -Epoch [83] Step [1/2], gen_loss: 0.430, disc_loss: 0.022 -[ 2023-09-21 08:24:44 ] Completed train batch 1 metrics update 5.284 ms, 7.15 s total -Saving checkpoint at epoch 83 train batch 1 -[ 2023-09-21 08:24:45 ] Completed saving temp checkpoint 491.947 ms, 7.64 s total -[ 2023-09-21 08:24:45 ] Completed replacing temp checkpoint with checkpoint 32.319 ms, 7.67 s total -Epoch [83] :: gen_loss: 0.442, disc_loss: 0.022 -[ 2023-09-21 08:24:45 ] Completed training generator for epoch 83 63.397 ms, 7.74 s total -[ 2023-09-21 08:24:45 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 83 from eval step 0 - -[ 2023-09-21 08:24:46 ] Completed eval batch 0 to device 1,097.869 ms, 1.10 s total -[ 2023-09-21 08:24:46 ] Completed eval batch 0 forward 24.549 ms, 1.12 s total -[ 2023-09-21 08:24:46 ] Completed eval batch 0 recons_loss 0.303 ms, 1.12 s total -[ 2023-09-21 08:24:46 ] Completed eval batch 0 metrics update 0.552 ms, 1.12 s total -Saving checkpoint at epoch 83 val batch 0 -[ 2023-09-21 08:24:47 ] Completed saving temp checkpoint 491.824 ms, 1.62 s total -[ 2023-09-21 08:24:47 ] Completed replacing temp checkpoint with checkpoint 28.285 ms, 1.64 s total -[ 2023-09-21 08:24:47 ] Completed eval batch 1 to device 315.130 ms, 1.96 s total -[ 2023-09-21 08:24:47 ] Completed eval batch 1 forward 39.689 ms, 2.00 s total -[ 2023-09-21 08:24:47 ] Completed eval batch 1 recons_loss 0.566 ms, 2.00 s total -[ 2023-09-21 08:24:47 ] Completed eval batch 1 metrics update 67.483 ms, 2.07 s total -Saving checkpoint at epoch 83 val batch 1 -Epoch 83 val loss: 0.0621 -[ 2023-09-21 08:24:48 ] Completed saving temp checkpoint 541.697 ms, 2.61 s total -[ 2023-09-21 08:24:48 ] Completed replacing temp checkpoint with checkpoint 30.848 ms, 2.64 s total -[ 2023-09-21 08:24:48 ] Completed evaluating generator for epoch 83 72.205 ms, 2.71 s total - - -EPOCH :: 84 - - -[ 2023-09-21 08:24:48 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 84 from training step 0 - -[ 2023-09-21 08:24:51 ] Completed train batch 0 to device 3,071.151 ms, 3.07 s total -[ 2023-09-21 08:24:51 ] Completed train batch 0 generator forward 26.905 ms, 3.10 s total -[ 2023-09-21 08:24:51 ] Completed train batch 0 generator loss: 1.305 66.369 ms, 3.16 s total -[ 2023-09-21 08:24:51 ] Completed train batch 0 generator backward 502.222 ms, 3.67 s total -[ 2023-09-21 08:24:51 ] Completed train batch 0 discriminator loss 0.064 59.061 ms, 3.73 s total -[ 2023-09-21 08:24:52 ] Completed train batch 0 discriminator backward 74.967 ms, 3.80 s total -Epoch [84] Step [0/2], gen_loss: 0.434, disc_loss: 0.022 -[ 2023-09-21 08:24:52 ] Completed train batch 0 metrics update 5.606 ms, 3.81 s total -Saving checkpoint at epoch 84 train batch 0 -[ 2023-09-21 08:24:52 ] Completed saving temp checkpoint 449.948 ms, 4.26 s total -[ 2023-09-21 08:24:52 ] Completed replacing temp checkpoint with checkpoint 29.964 ms, 4.29 s total -[ 2023-09-21 08:24:54 ] Completed train batch 1 to device 1,714.652 ms, 6.00 s total -[ 2023-09-21 08:24:54 ] Completed train batch 1 generator forward 41.117 ms, 6.04 s total -[ 2023-09-21 08:24:54 ] Completed train batch 1 generator loss: 1.284 64.494 ms, 6.11 s total -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 08:37:11 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 08:37:11 ] Completed importing Timer 0.023 ms, 0.00 s total -[ 2023-09-21 08:37:15 ] Completed importing everything else 4,560.760 ms, 4.56 s total -| distributed init (rank 1): env:// -| distributed init (rank 4): env:// -| distributed init (rank 2): env:// -| distributed init (rank 3): env:// -| distributed init (rank 5): env:// -| distributed init (rank 0): env:// -[ 2023-09-21 08:37:23 ] Completed preliminaries 8,000.092 ms, 12.56 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 08:37:23 ] Completed build datasets 12.248 ms, 12.57 s total -[ 2023-09-21 08:37:23 ] Completed build samplers 0.048 ms, 12.57 s total -[ 2023-09-21 08:37:23 ] Completed build dataloaders 0.128 ms, 12.57 s total -[ 2023-09-21 08:37:24 ] Completed generator to device 98.323 ms, 12.67 s total -[ 2023-09-21 08:37:24 ] Completed discriminator to device 7.038 ms, 12.68 s total -[ 2023-09-21 08:37:24 ] Completed loss functions 387.323 ms, 13.07 s total -[ 2023-09-21 08:37:24 ] Completed models prepped for distribution 79.678 ms, 13.15 s total -[ 2023-09-21 08:37:24 ] Completed optimizers 0.649 ms, 13.15 s total -[ 2023-09-21 08:37:24 ] Completed grad scalers 0.021 ms, 13.15 s total -[ 2023-09-21 08:37:25 ] Completed checkpoint retrieval 456.307 ms, 13.60 s total - - -EPOCH :: 84 - - -[ 2023-09-21 08:37:25 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 84 from training step 1 - -[ 2023-09-21 08:37:27 ] Completed train batch 1 to device 2,969.141 ms, 2.97 s total -[ 2023-09-21 08:37:29 ] Completed train batch 1 generator forward 1,053.736 ms, 4.02 s total -[ 2023-09-21 08:37:29 ] Completed train batch 1 generator loss: 1.283 72.949 ms, 4.10 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 08:37:30 ] Completed train batch 1 generator backward 1,528.393 ms, 5.62 s total -[ 2023-09-21 08:37:30 ] Completed train batch 1 discriminator loss 0.069 65.458 ms, 5.69 s total -[ 2023-09-21 08:37:30 ] Completed train batch 1 discriminator backward 60.379 ms, 5.75 s total -Epoch [84] Step [1/2], gen_loss: 0.434, disc_loss: 0.022 -[ 2023-09-21 08:37:30 ] Completed train batch 1 metrics update 5.680 ms, 5.76 s total -Saving checkpoint at epoch 84 train batch 1 -[ 2023-09-21 08:37:31 ] Completed saving temp checkpoint 456.080 ms, 6.21 s total -[ 2023-09-21 08:37:31 ] Completed replacing temp checkpoint with checkpoint 140.221 ms, 6.35 s total -Epoch [84] :: gen_loss: 0.434, disc_loss: 0.022 -[ 2023-09-21 08:37:31 ] Completed training generator for epoch 84 41.081 ms, 6.39 s total -[ 2023-09-21 08:37:31 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 84 from eval step 0 - -[ 2023-09-21 08:37:32 ] Completed eval batch 0 to device 1,084.780 ms, 1.08 s total -[ 2023-09-21 08:37:32 ] Completed eval batch 0 forward 32.321 ms, 1.12 s total -[ 2023-09-21 08:37:32 ] Completed eval batch 0 recons_loss 0.394 ms, 1.12 s total -[ 2023-09-21 08:37:32 ] Completed eval batch 0 metrics update 0.730 ms, 1.12 s total -Saving checkpoint at epoch 84 val batch 0 -[ 2023-09-21 08:37:32 ] Completed saving temp checkpoint 423.445 ms, 1.54 s total -[ 2023-09-21 08:37:32 ] Completed replacing temp checkpoint with checkpoint 26.338 ms, 1.57 s total -[ 2023-09-21 08:37:33 ] Completed eval batch 1 to device 445.819 ms, 2.01 s total -[ 2023-09-21 08:37:33 ] Completed eval batch 1 forward 39.346 ms, 2.05 s total -[ 2023-09-21 08:37:33 ] Completed eval batch 1 recons_loss 0.342 ms, 2.05 s total -[ 2023-09-21 08:37:33 ] Completed eval batch 1 metrics update 0.310 ms, 2.05 s total -Saving checkpoint at epoch 84 val batch 1 -Epoch 84 val loss: 0.0690 -[ 2023-09-21 08:37:33 ] Completed saving temp checkpoint 445.363 ms, 2.50 s total -[ 2023-09-21 08:37:33 ] Completed replacing temp checkpoint with checkpoint 18.873 ms, 2.52 s total -[ 2023-09-21 08:37:33 ] Completed evaluating generator for epoch 84 56.829 ms, 2.57 s total - - -EPOCH :: 85 - - -[ 2023-09-21 08:37:33 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 85 from training step 0 - -[ 2023-09-21 08:37:36 ] Completed train batch 0 to device 3,002.618 ms, 3.00 s total -[ 2023-09-21 08:37:37 ] Completed train batch 0 generator forward 32.380 ms, 3.03 s total -[ 2023-09-21 08:37:37 ] Completed train batch 0 generator loss: 1.246 75.447 ms, 3.11 s total -[ 2023-09-21 08:37:37 ] Completed train batch 0 generator backward 593.391 ms, 3.70 s total -[ 2023-09-21 08:37:37 ] Completed train batch 0 discriminator loss 0.064 56.687 ms, 3.76 s total -[ 2023-09-21 08:37:37 ] Completed train batch 0 discriminator backward 95.863 ms, 3.86 s total -Epoch [85] Step [0/2], gen_loss: 0.434, disc_loss: 0.022 -[ 2023-09-21 08:37:37 ] Completed train batch 0 metrics update 3.381 ms, 3.86 s total -Saving checkpoint at epoch 85 train batch 0 -[ 2023-09-21 08:37:38 ] Completed saving temp checkpoint 395.102 ms, 4.25 s total -[ 2023-09-21 08:37:38 ] Completed replacing temp checkpoint with checkpoint 21.464 ms, 4.28 s total -[ 2023-09-21 08:37:39 ] Completed train batch 1 to device 1,571.264 ms, 5.85 s total -[ 2023-09-21 08:37:39 ] Completed train batch 1 generator forward 22.235 ms, 5.87 s total -[ 2023-09-21 08:37:39 ] Completed train batch 1 generator loss: 1.313 62.679 ms, 5.93 s total -[ 2023-09-21 08:37:41 ] Completed train batch 1 generator backward 1,238.011 ms, 7.17 s total -[ 2023-09-21 08:37:41 ] Completed train batch 1 discriminator loss 0.065 56.200 ms, 7.23 s total -[ 2023-09-21 08:37:41 ] Completed train batch 1 discriminator backward 72.238 ms, 7.30 s total -Epoch [85] Step [1/2], gen_loss: 0.432, disc_loss: 0.023 -[ 2023-09-21 08:37:41 ] Completed train batch 1 metrics update 4.912 ms, 7.30 s total -Saving checkpoint at epoch 85 train batch 1 -[ 2023-09-21 08:37:41 ] Completed saving temp checkpoint 435.985 ms, 7.74 s total -[ 2023-09-21 08:37:41 ] Completed replacing temp checkpoint with checkpoint 29.507 ms, 7.77 s total -Epoch [85] :: gen_loss: 0.433, disc_loss: 0.023 -[ 2023-09-21 08:37:41 ] Completed training generator for epoch 85 73.936 ms, 7.84 s total -[ 2023-09-21 08:37:41 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 85 from eval step 0 - -[ 2023-09-21 08:37:42 ] Completed eval batch 0 to device 1,108.904 ms, 1.11 s total -[ 2023-09-21 08:37:42 ] Completed eval batch 0 forward 24.962 ms, 1.13 s total -[ 2023-09-21 08:37:42 ] Completed eval batch 0 recons_loss 0.306 ms, 1.13 s total -[ 2023-09-21 08:37:42 ] Completed eval batch 0 metrics update 0.587 ms, 1.13 s total -Saving checkpoint at epoch 85 val batch 0 -[ 2023-09-21 08:37:43 ] Completed saving temp checkpoint 426.435 ms, 1.56 s total -[ 2023-09-21 08:37:43 ] Completed replacing temp checkpoint with checkpoint 27.424 ms, 1.59 s total -[ 2023-09-21 08:37:43 ] Completed eval batch 1 to device 376.246 ms, 1.96 s total -[ 2023-09-21 08:37:43 ] Completed eval batch 1 forward 39.468 ms, 2.00 s total -[ 2023-09-21 08:37:43 ] Completed eval batch 1 recons_loss 0.567 ms, 2.00 s total -[ 2023-09-21 08:37:43 ] Completed eval batch 1 metrics update 114.613 ms, 2.12 s total -Saving checkpoint at epoch 85 val batch 1 -Epoch 85 val loss: 0.0624 -[ 2023-09-21 08:37:44 ] Completed saving temp checkpoint 429.945 ms, 2.55 s total -[ 2023-09-21 08:37:44 ] Completed replacing temp checkpoint with checkpoint 31.621 ms, 2.58 s total -[ 2023-09-21 08:37:44 ] Completed evaluating generator for epoch 85 52.873 ms, 2.63 s total - - -EPOCH :: 86 - - -[ 2023-09-21 08:37:44 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 86 from training step 0 - -[ 2023-09-21 08:37:47 ] Completed train batch 0 to device 3,002.154 ms, 3.00 s total -[ 2023-09-21 08:37:47 ] Completed train batch 0 generator forward 44.928 ms, 3.05 s total -[ 2023-09-21 08:37:47 ] Completed train batch 0 generator loss: 1.232 65.435 ms, 3.11 s total -[ 2023-09-21 08:37:48 ] Completed train batch 0 generator backward 608.948 ms, 3.72 s total -[ 2023-09-21 08:37:48 ] Completed train batch 0 discriminator loss 0.067 56.595 ms, 3.78 s total -[ 2023-09-21 08:37:48 ] Completed train batch 0 discriminator backward 100.503 ms, 3.88 s total -Epoch [86] Step [0/2], gen_loss: 0.427, disc_loss: 0.022 -[ 2023-09-21 08:37:48 ] Completed train batch 0 metrics update 5.914 ms, 3.88 s total -Saving checkpoint at epoch 86 train batch 0 -[ 2023-09-21 08:37:48 ] Completed saving temp checkpoint 461.155 ms, 4.35 s total -[ 2023-09-21 08:37:48 ] Completed replacing temp checkpoint with checkpoint 32.349 ms, 4.38 s total -[ 2023-09-21 08:37:50 ] Completed train batch 1 to device 1,369.635 ms, 5.75 s total -[ 2023-09-21 08:37:50 ] Completed train batch 1 generator forward 40.617 ms, 5.79 s total -[ 2023-09-21 08:37:50 ] Completed train batch 1 generator loss: 1.168 62.415 ms, 5.85 s total -[ 2023-09-21 08:37:51 ] Completed train batch 1 generator backward 1,379.927 ms, 7.23 s total -[ 2023-09-21 08:37:51 ] Completed train batch 1 discriminator loss 0.064 55.839 ms, 7.29 s total -[ 2023-09-21 08:37:51 ] Completed train batch 1 discriminator backward 79.100 ms, 7.37 s total -Epoch [86] Step [1/2], gen_loss: 0.421, disc_loss: 0.022 -[ 2023-09-21 08:37:51 ] Completed train batch 1 metrics update 5.673 ms, 7.37 s total -Saving checkpoint at epoch 86 train batch 1 -[ 2023-09-21 08:37:52 ] Completed saving temp checkpoint 420.687 ms, 7.79 s total -[ 2023-09-21 08:37:52 ] Completed replacing temp checkpoint with checkpoint 20.134 ms, 7.81 s total -Epoch [86] :: gen_loss: 0.424, disc_loss: 0.022 -[ 2023-09-21 08:37:52 ] Completed training generator for epoch 86 67.529 ms, 7.88 s total -[ 2023-09-21 08:37:52 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 86 from eval step 0 - -[ 2023-09-21 08:37:53 ] Completed eval batch 0 to device 1,112.662 ms, 1.11 s total -[ 2023-09-21 08:37:53 ] Completed eval batch 0 forward 35.351 ms, 1.15 s total -[ 2023-09-21 08:37:53 ] Completed eval batch 0 recons_loss 0.360 ms, 1.15 s total -[ 2023-09-21 08:37:53 ] Completed eval batch 0 metrics update 0.692 ms, 1.15 s total -Saving checkpoint at epoch 86 val batch 0 -[ 2023-09-21 08:37:53 ] Completed saving temp checkpoint 346.958 ms, 1.50 s total -[ 2023-09-21 08:37:53 ] Completed replacing temp checkpoint with checkpoint 26.376 ms, 1.52 s total -[ 2023-09-21 08:37:54 ] Completed eval batch 1 to device 532.656 ms, 2.06 s total -[ 2023-09-21 08:37:54 ] Completed eval batch 1 forward 40.455 ms, 2.10 s total -[ 2023-09-21 08:37:54 ] Completed eval batch 1 recons_loss 0.627 ms, 2.10 s total -[ 2023-09-21 08:37:54 ] Completed eval batch 1 metrics update 56.625 ms, 2.15 s total -Saving checkpoint at epoch 86 val batch 1 -Epoch 86 val loss: 0.0551 -[ 2023-09-21 08:37:54 ] Completed saving temp checkpoint 385.306 ms, 2.54 s total -[ 2023-09-21 08:37:54 ] Completed replacing temp checkpoint with checkpoint 18.097 ms, 2.56 s total -[ 2023-09-21 08:37:54 ] Completed evaluating generator for epoch 86 65.376 ms, 2.62 s total - - -EPOCH :: 87 - - -[ 2023-09-21 08:37:54 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 87 from training step 0 - -[ 2023-09-21 08:37:58 ] Completed train batch 0 to device 3,057.718 ms, 3.06 s total -[ 2023-09-21 08:37:58 ] Completed train batch 0 generator forward 40.051 ms, 3.10 s total -[ 2023-09-21 08:37:58 ] Completed train batch 0 generator loss: 1.276 68.036 ms, 3.17 s total -[ 2023-09-21 08:37:58 ] Completed train batch 0 generator backward 518.593 ms, 3.68 s total -[ 2023-09-21 08:37:58 ] Completed train batch 0 discriminator loss 0.069 61.354 ms, 3.75 s total -[ 2023-09-21 08:37:58 ] Completed train batch 0 discriminator backward 79.619 ms, 3.83 s total -Epoch [87] Step [0/2], gen_loss: 0.428, disc_loss: 0.023 -[ 2023-09-21 08:37:58 ] Completed train batch 0 metrics update 3.643 ms, 3.83 s total -Saving checkpoint at epoch 87 train batch 0 -[ 2023-09-21 08:37:59 ] Completed saving temp checkpoint 437.393 ms, 4.27 s total -[ 2023-09-21 08:37:59 ] Completed replacing temp checkpoint with checkpoint 23.168 ms, 4.29 s total -[ 2023-09-21 08:38:00 ] Completed train batch 1 to device 1,476.512 ms, 5.77 s total -[ 2023-09-21 08:38:00 ] Completed train batch 1 generator forward 41.139 ms, 5.81 s total -[ 2023-09-21 08:38:00 ] Completed train batch 1 generator loss: 1.257 63.657 ms, 5.87 s total -[ 2023-09-21 08:38:02 ] Completed train batch 1 generator backward 1,227.927 ms, 7.10 s total -[ 2023-09-21 08:38:02 ] Completed train batch 1 discriminator loss 0.071 55.779 ms, 7.15 s total -[ 2023-09-21 08:38:02 ] Completed train batch 1 discriminator backward 69.511 ms, 7.22 s total -Epoch [87] Step [1/2], gen_loss: 0.414, disc_loss: 0.025 -[ 2023-09-21 08:38:02 ] Completed train batch 1 metrics update 2.022 ms, 7.23 s total -Saving checkpoint at epoch 87 train batch 1 -[ 2023-09-21 08:38:02 ] Completed saving temp checkpoint 434.943 ms, 7.66 s total -[ 2023-09-21 08:38:02 ] Completed replacing temp checkpoint with checkpoint 22.395 ms, 7.68 s total -Epoch [87] :: gen_loss: 0.421, disc_loss: 0.024 -[ 2023-09-21 08:38:02 ] Completed training generator for epoch 87 63.161 ms, 7.75 s total -[ 2023-09-21 08:38:02 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 87 from eval step 0 - -[ 2023-09-21 08:38:03 ] Completed eval batch 0 to device 1,128.317 ms, 1.13 s total -[ 2023-09-21 08:38:03 ] Completed eval batch 0 forward 31.858 ms, 1.16 s total -[ 2023-09-21 08:38:03 ] Completed eval batch 0 recons_loss 0.340 ms, 1.16 s total -[ 2023-09-21 08:38:03 ] Completed eval batch 0 metrics update 0.624 ms, 1.16 s total -Saving checkpoint at epoch 87 val batch 0 -[ 2023-09-21 08:38:04 ] Completed saving temp checkpoint 410.998 ms, 1.57 s total -[ 2023-09-21 08:38:04 ] Completed replacing temp checkpoint with checkpoint 25.324 ms, 1.60 s total -[ 2023-09-21 08:38:04 ] Completed eval batch 1 to device 484.901 ms, 2.08 s total -[ 2023-09-21 08:38:04 ] Completed eval batch 1 forward 21.453 ms, 2.10 s total -[ 2023-09-21 08:38:04 ] Completed eval batch 1 recons_loss 0.341 ms, 2.10 s total -[ 2023-09-21 08:38:04 ] Completed eval batch 1 metrics update 90.751 ms, 2.19 s total -Saving checkpoint at epoch 87 val batch 1 -Epoch 87 val loss: 0.0602 -[ 2023-09-21 08:38:05 ] Completed saving temp checkpoint 404.916 ms, 2.60 s total -[ 2023-09-21 08:38:05 ] Completed replacing temp checkpoint with checkpoint 20.038 ms, 2.62 s total -[ 2023-09-21 08:38:05 ] Completed evaluating generator for epoch 87 49.208 ms, 2.67 s total - - -EPOCH :: 88 - - -[ 2023-09-21 08:38:05 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 88 from training step 0 - -[ 2023-09-21 08:38:08 ] Completed train batch 0 to device 2,889.790 ms, 2.89 s total -[ 2023-09-21 08:38:08 ] Completed train batch 0 generator forward 25.395 ms, 2.92 s total -[ 2023-09-21 08:38:08 ] Completed train batch 0 generator loss: 1.266 63.676 ms, 2.98 s total -[ 2023-09-21 08:38:08 ] Completed train batch 0 generator backward 608.918 ms, 3.59 s total -[ 2023-09-21 08:38:09 ] Completed train batch 0 discriminator loss 0.072 79.180 ms, 3.67 s total -[ 2023-09-21 08:38:09 ] Completed train batch 0 discriminator backward 78.404 ms, 3.75 s total -Epoch [88] Step [0/2], gen_loss: 0.420, disc_loss: 0.024 -[ 2023-09-21 08:38:09 ] Completed train batch 0 metrics update 24.650 ms, 3.77 s total -Saving checkpoint at epoch 88 train batch 0 -[ 2023-09-21 08:38:09 ] Completed saving temp checkpoint 405.534 ms, 4.18 s total -[ 2023-09-21 08:38:09 ] Completed replacing temp checkpoint with checkpoint 35.664 ms, 4.21 s total -[ 2023-09-21 08:38:10 ] Completed train batch 1 to device 1,297.992 ms, 5.51 s total -[ 2023-09-21 08:38:10 ] Completed train batch 1 generator forward 42.154 ms, 5.55 s total -[ 2023-09-21 08:38:10 ] Completed train batch 1 generator loss: 1.224 62.226 ms, 5.61 s total -[ 2023-09-21 08:38:12 ] Completed train batch 1 generator backward 1,368.258 ms, 6.98 s total -[ 2023-09-21 08:38:12 ] Completed train batch 1 discriminator loss 0.069 56.244 ms, 7.04 s total -[ 2023-09-21 08:38:12 ] Completed train batch 1 discriminator backward 91.421 ms, 7.13 s total -Epoch [88] Step [1/2], gen_loss: 0.426, disc_loss: 0.023 -[ 2023-09-21 08:38:12 ] Completed train batch 1 metrics update 0.410 ms, 7.13 s total -Saving checkpoint at epoch 88 train batch 1 -[ 2023-09-21 08:38:12 ] Completed saving temp checkpoint 480.104 ms, 7.61 s total -[ 2023-09-21 08:38:13 ] Completed replacing temp checkpoint with checkpoint 28.960 ms, 7.64 s total -Epoch [88] :: gen_loss: 0.423, disc_loss: 0.023 -[ 2023-09-21 08:38:13 ] Completed training generator for epoch 88 79.437 ms, 7.72 s total -[ 2023-09-21 08:38:13 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 88 from eval step 0 - -[ 2023-09-21 08:38:14 ] Completed eval batch 0 to device 1,038.056 ms, 1.04 s total -[ 2023-09-21 08:38:14 ] Completed eval batch 0 forward 25.152 ms, 1.06 s total -[ 2023-09-21 08:38:14 ] Completed eval batch 0 recons_loss 0.307 ms, 1.06 s total -[ 2023-09-21 08:38:14 ] Completed eval batch 0 metrics update 0.585 ms, 1.06 s total -Saving checkpoint at epoch 88 val batch 0 -[ 2023-09-21 08:38:14 ] Completed saving temp checkpoint 424.186 ms, 1.49 s total -[ 2023-09-21 08:38:14 ] Completed replacing temp checkpoint with checkpoint 34.263 ms, 1.52 s total -[ 2023-09-21 08:38:14 ] Completed eval batch 1 to device 327.003 ms, 1.85 s total -[ 2023-09-21 08:38:14 ] Completed eval batch 1 forward 21.470 ms, 1.87 s total -[ 2023-09-21 08:38:14 ] Completed eval batch 1 recons_loss 0.337 ms, 1.87 s total -[ 2023-09-21 08:38:15 ] Completed eval batch 1 metrics update 260.805 ms, 2.13 s total -Saving checkpoint at epoch 88 val batch 1 -Epoch 88 val loss: 0.0691 -[ 2023-09-21 08:38:15 ] Completed saving temp checkpoint 439.432 ms, 2.57 s total -[ 2023-09-21 08:38:15 ] Completed replacing temp checkpoint with checkpoint 24.629 ms, 2.60 s total -[ 2023-09-21 08:38:15 ] Completed evaluating generator for epoch 88 51.939 ms, 2.65 s total - - -EPOCH :: 89 - - -[ 2023-09-21 08:38:15 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 89 from training step 0 - -[ 2023-09-21 08:38:18 ] Completed train batch 0 to device 2,942.213 ms, 2.94 s total -[ 2023-09-21 08:38:18 ] Completed train batch 0 generator forward 31.825 ms, 2.97 s total -[ 2023-09-21 08:38:18 ] Completed train batch 0 generator loss: 1.289 65.331 ms, 3.04 s total -[ 2023-09-21 08:38:19 ] Completed train batch 0 generator backward 603.243 ms, 3.64 s total -[ 2023-09-21 08:38:19 ] Completed train batch 0 discriminator loss 0.073 64.733 ms, 3.71 s total -[ 2023-09-21 08:38:19 ] Completed train batch 0 discriminator backward 95.130 ms, 3.80 s total -Epoch [89] Step [0/2], gen_loss: 0.419, disc_loss: 0.024 -[ 2023-09-21 08:38:19 ] Completed train batch 0 metrics update 16.866 ms, 3.82 s total -Saving checkpoint at epoch 89 train batch 0 -[ 2023-09-21 08:38:19 ] Completed saving temp checkpoint 430.833 ms, 4.25 s total -[ 2023-09-21 08:38:20 ] Completed replacing temp checkpoint with checkpoint 31.681 ms, 4.28 s total -[ 2023-09-21 08:38:21 ] Completed train batch 1 to device 1,592.870 ms, 5.87 s total -[ 2023-09-21 08:38:21 ] Completed train batch 1 generator forward 26.398 ms, 5.90 s total -[ 2023-09-21 08:38:21 ] Completed train batch 1 generator loss: 1.280 62.429 ms, 5.96 s total -[ 2023-09-21 08:38:22 ] Completed train batch 1 generator backward 1,119.422 ms, 7.08 s total -[ 2023-09-21 08:38:22 ] Completed train batch 1 discriminator loss 0.075 55.964 ms, 7.14 s total -[ 2023-09-21 08:38:22 ] Completed train batch 1 discriminator backward 90.185 ms, 7.23 s total -Epoch [89] Step [1/2], gen_loss: 0.430, disc_loss: 0.026 -[ 2023-09-21 08:38:22 ] Completed train batch 1 metrics update 5.049 ms, 7.23 s total -Saving checkpoint at epoch 89 train batch 1 -[ 2023-09-21 08:38:23 ] Completed saving temp checkpoint 450.621 ms, 7.68 s total -[ 2023-09-21 08:38:23 ] Completed replacing temp checkpoint with checkpoint 20.642 ms, 7.71 s total -Epoch [89] :: gen_loss: 0.424, disc_loss: 0.025 -[ 2023-09-21 08:38:23 ] Completed training generator for epoch 89 70.188 ms, 7.78 s total -[ 2023-09-21 08:38:23 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 89 from eval step 0 - -[ 2023-09-21 08:38:24 ] Completed eval batch 0 to device 1,093.326 ms, 1.09 s total -[ 2023-09-21 08:38:24 ] Completed eval batch 0 forward 30.590 ms, 1.12 s total -[ 2023-09-21 08:38:24 ] Completed eval batch 0 recons_loss 0.324 ms, 1.12 s total -[ 2023-09-21 08:38:24 ] Completed eval batch 0 metrics update 0.580 ms, 1.12 s total -Saving checkpoint at epoch 89 val batch 0 -[ 2023-09-21 08:38:25 ] Completed saving temp checkpoint 390.913 ms, 1.52 s total -[ 2023-09-21 08:38:25 ] Completed replacing temp checkpoint with checkpoint 33.642 ms, 1.55 s total -[ 2023-09-21 08:38:25 ] Completed eval batch 1 to device 420.094 ms, 1.97 s total -[ 2023-09-21 08:38:25 ] Completed eval batch 1 forward 39.962 ms, 2.01 s total -[ 2023-09-21 08:38:25 ] Completed eval batch 1 recons_loss 0.554 ms, 2.01 s total -[ 2023-09-21 08:38:25 ] Completed eval batch 1 metrics update 136.875 ms, 2.15 s total -Saving checkpoint at epoch 89 val batch 1 -Epoch 89 val loss: 0.0649 -[ 2023-09-21 08:38:26 ] Completed saving temp checkpoint 345.337 ms, 2.49 s total -[ 2023-09-21 08:38:26 ] Completed replacing temp checkpoint with checkpoint 17.623 ms, 2.51 s total -[ 2023-09-21 08:38:26 ] Completed evaluating generator for epoch 89 59.337 ms, 2.57 s total - - -EPOCH :: 90 - - -[ 2023-09-21 08:38:26 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 90 from training step 0 - -[ 2023-09-21 08:38:29 ] Completed train batch 0 to device 2,966.603 ms, 2.97 s total -[ 2023-09-21 08:38:29 ] Completed train batch 0 generator forward 42.630 ms, 3.01 s total -[ 2023-09-21 08:38:29 ] Completed train batch 0 generator loss: 1.292 65.599 ms, 3.07 s total -[ 2023-09-21 08:38:29 ] Completed train batch 0 generator backward 678.595 ms, 3.75 s total -[ 2023-09-21 08:38:29 ] Completed train batch 0 discriminator loss 0.084 58.003 ms, 3.81 s total -[ 2023-09-21 08:38:29 ] Completed train batch 0 discriminator backward 92.737 ms, 3.90 s total -Epoch [90] Step [0/2], gen_loss: 0.415, disc_loss: 0.028 -[ 2023-09-21 08:38:29 ] Completed train batch 0 metrics update 6.498 ms, 3.91 s total -Saving checkpoint at epoch 90 train batch 0 -[ 2023-09-21 08:38:30 ] Completed saving temp checkpoint 377.957 ms, 4.29 s total -[ 2023-09-21 08:38:30 ] Completed replacing temp checkpoint with checkpoint 23.757 ms, 4.31 s total -[ 2023-09-21 08:38:31 ] Completed train batch 1 to device 1,278.076 ms, 5.59 s total -[ 2023-09-21 08:38:31 ] Completed train batch 1 generator forward 38.504 ms, 5.63 s total -[ 2023-09-21 08:38:31 ] Completed train batch 1 generator loss: 1.277 61.971 ms, 5.69 s total -[ 2023-09-21 08:38:33 ] Completed train batch 1 generator backward 1,517.513 ms, 7.21 s total -[ 2023-09-21 08:38:33 ] Completed train batch 1 discriminator loss 0.076 56.034 ms, 7.26 s total -[ 2023-09-21 08:38:33 ] Completed train batch 1 discriminator backward 72.188 ms, 7.34 s total -Epoch [90] Step [1/2], gen_loss: 0.427, disc_loss: 0.026 -[ 2023-09-21 08:38:33 ] Completed train batch 1 metrics update 1.332 ms, 7.34 s total -Saving checkpoint at epoch 90 train batch 1 -[ 2023-09-21 08:38:33 ] Completed saving temp checkpoint 387.382 ms, 7.73 s total -[ 2023-09-21 08:38:33 ] Completed replacing temp checkpoint with checkpoint 21.270 ms, 7.75 s total -Epoch [90] :: gen_loss: 0.421, disc_loss: 0.027 -[ 2023-09-21 08:38:33 ] Completed training generator for epoch 90 76.188 ms, 7.82 s total -[ 2023-09-21 08:38:33 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 90 from eval step 0 - -[ 2023-09-21 08:38:35 ] Completed eval batch 0 to device 1,121.755 ms, 1.12 s total -[ 2023-09-21 08:38:35 ] Completed eval batch 0 forward 40.190 ms, 1.16 s total -[ 2023-09-21 08:38:35 ] Completed eval batch 0 recons_loss 0.310 ms, 1.16 s total -[ 2023-09-21 08:38:35 ] Completed eval batch 0 metrics update 0.602 ms, 1.16 s total -Saving checkpoint at epoch 90 val batch 0 -[ 2023-09-21 08:38:35 ] Completed saving temp checkpoint 564.244 ms, 1.73 s total -[ 2023-09-21 08:38:35 ] Completed replacing temp checkpoint with checkpoint 28.342 ms, 1.76 s total -[ 2023-09-21 08:38:36 ] Completed eval batch 1 to device 382.637 ms, 2.14 s total -[ 2023-09-21 08:38:36 ] Completed eval batch 1 forward 38.893 ms, 2.18 s total -[ 2023-09-21 08:38:36 ] Completed eval batch 1 recons_loss 0.342 ms, 2.18 s total -[ 2023-09-21 08:38:36 ] Completed eval batch 1 metrics update 0.315 ms, 2.18 s total -Saving checkpoint at epoch 90 val batch 1 -Epoch 90 val loss: 0.0634 -[ 2023-09-21 08:38:36 ] Completed saving temp checkpoint 420.783 ms, 2.60 s total -[ 2023-09-21 08:38:36 ] Completed replacing temp checkpoint with checkpoint 30.398 ms, 2.63 s total -[ 2023-09-21 08:38:36 ] Completed evaluating generator for epoch 90 78.627 ms, 2.71 s total - - -EPOCH :: 91 - - -[ 2023-09-21 08:38:36 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 91 from training step 0 - -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 08:49:25 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 08:49:25 ] Completed importing Timer 0.023 ms, 0.00 s total -[ 2023-09-21 08:49:30 ] Completed importing everything else 4,769.650 ms, 4.77 s total -| distributed init (rank 2): env:// -| distributed init (rank 1): env:// -| distributed init (rank 3): env:// -| distributed init (rank 4): env:// -| distributed init (rank 5): env:// -| distributed init (rank 0): env:// -[ 2023-09-21 08:49:38 ] Completed preliminaries 7,886.886 ms, 12.66 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 08:49:38 ] Completed build datasets 11.201 ms, 12.67 s total -[ 2023-09-21 08:49:38 ] Completed build samplers 0.047 ms, 12.67 s total -[ 2023-09-21 08:49:38 ] Completed build dataloaders 0.135 ms, 12.67 s total -[ 2023-09-21 08:49:38 ] Completed generator to device 78.187 ms, 12.75 s total -[ 2023-09-21 08:49:38 ] Completed discriminator to device 7.165 ms, 12.75 s total -[ 2023-09-21 08:49:38 ] Completed loss functions 408.468 ms, 13.16 s total -[ 2023-09-21 08:49:38 ] Completed models prepped for distribution 79.521 ms, 13.24 s total -[ 2023-09-21 08:49:38 ] Completed optimizers 0.678 ms, 13.24 s total -[ 2023-09-21 08:49:38 ] Completed grad scalers 0.021 ms, 13.24 s total -[ 2023-09-21 08:49:39 ] Completed checkpoint retrieval 479.484 ms, 13.72 s total - - -EPOCH :: 90 - - -[ 2023-09-21 08:49:39 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 90 from training step 2 - -Epoch [90] :: gen_loss: 0.421, disc_loss: 0.027 -[ 2023-09-21 08:49:39 ] Completed training generator for epoch 90 142.817 ms, 0.14 s total -[ 2023-09-21 08:49:39 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 90 from eval step 2 - -[ 2023-09-21 08:49:39 ] Completed evaluating generator for epoch 90 114.368 ms, 0.11 s total - - -EPOCH :: 91 - - -[ 2023-09-21 08:49:39 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 91 from training step 0 - -[ 2023-09-21 08:49:43 ] Completed train batch 0 to device 4,072.147 ms, 4.07 s total -[ 2023-09-21 08:49:45 ] Completed train batch 0 generator forward 1,452.056 ms, 5.52 s total -[ 2023-09-21 08:49:45 ] Completed train batch 0 generator loss: 1.250 89.324 ms, 5.61 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 08:49:45 ] Completed train batch 0 generator backward 669.703 ms, 6.28 s total -[ 2023-09-21 08:49:46 ] Completed train batch 0 discriminator loss 0.071 57.166 ms, 6.34 s total -[ 2023-09-21 08:49:46 ] Completed train batch 0 discriminator backward 90.714 ms, 6.43 s total -Epoch [91] Step [0/2], gen_loss: 0.421, disc_loss: 0.024 -[ 2023-09-21 08:49:46 ] Completed train batch 0 metrics update 3.618 ms, 6.43 s total -Saving checkpoint at epoch 91 train batch 0 -[ 2023-09-21 08:49:46 ] Completed saving temp checkpoint 419.153 ms, 6.85 s total -[ 2023-09-21 08:49:46 ] Completed replacing temp checkpoint with checkpoint 132.930 ms, 6.99 s total -[ 2023-09-21 08:49:47 ] Completed train batch 1 to device 403.669 ms, 7.39 s total -[ 2023-09-21 08:49:47 ] Completed train batch 1 generator forward 22.212 ms, 7.41 s total -[ 2023-09-21 08:49:47 ] Completed train batch 1 generator loss: 1.298 61.855 ms, 7.47 s total -[ 2023-09-21 08:49:47 ] Completed train batch 1 generator backward 770.879 ms, 8.25 s total -[ 2023-09-21 08:49:47 ] Completed train batch 1 discriminator loss 0.067 56.664 ms, 8.30 s total -[ 2023-09-21 08:49:48 ] Completed train batch 1 discriminator backward 71.514 ms, 8.37 s total -Epoch [91] Step [1/2], gen_loss: 0.425, disc_loss: 0.022 -[ 2023-09-21 08:49:48 ] Completed train batch 1 metrics update 0.824 ms, 8.37 s total -Saving checkpoint at epoch 91 train batch 1 -[ 2023-09-21 08:49:48 ] Completed saving temp checkpoint 502.739 ms, 8.88 s total -[ 2023-09-21 08:49:48 ] Completed replacing temp checkpoint with checkpoint 35.246 ms, 8.91 s total -Epoch [91] :: gen_loss: 0.423, disc_loss: 0.023 -[ 2023-09-21 08:49:48 ] Completed training generator for epoch 91 53.671 ms, 8.97 s total -[ 2023-09-21 08:49:48 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 91 from eval step 0 - -[ 2023-09-21 08:49:49 ] Completed eval batch 0 to device 1,053.857 ms, 1.05 s total -[ 2023-09-21 08:49:49 ] Completed eval batch 0 forward 41.864 ms, 1.10 s total -[ 2023-09-21 08:49:49 ] Completed eval batch 0 recons_loss 0.379 ms, 1.10 s total -[ 2023-09-21 08:49:49 ] Completed eval batch 0 metrics update 0.699 ms, 1.10 s total -Saving checkpoint at epoch 91 val batch 0 -[ 2023-09-21 08:49:50 ] Completed saving temp checkpoint 594.697 ms, 1.69 s total -[ 2023-09-21 08:49:50 ] Completed replacing temp checkpoint with checkpoint 37.013 ms, 1.73 s total -[ 2023-09-21 08:49:50 ] Completed eval batch 1 to device 176.701 ms, 1.91 s total -[ 2023-09-21 08:49:50 ] Completed eval batch 1 forward 39.966 ms, 1.95 s total -[ 2023-09-21 08:49:50 ] Completed eval batch 1 recons_loss 0.571 ms, 1.95 s total -[ 2023-09-21 08:49:50 ] Completed eval batch 1 metrics update 85.173 ms, 2.03 s total -Saving checkpoint at epoch 91 val batch 1 -Epoch 91 val loss: 0.0741 -[ 2023-09-21 08:49:51 ] Completed saving temp checkpoint 621.084 ms, 2.65 s total -[ 2023-09-21 08:49:51 ] Completed replacing temp checkpoint with checkpoint 32.583 ms, 2.68 s total -[ 2023-09-21 08:49:51 ] Completed evaluating generator for epoch 91 53.094 ms, 2.74 s total - - -EPOCH :: 92 - - -[ 2023-09-21 08:49:51 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 92 from training step 0 - -[ 2023-09-21 08:49:54 ] Completed train batch 0 to device 2,883.975 ms, 2.88 s total -[ 2023-09-21 08:49:54 ] Completed train batch 0 generator forward 30.731 ms, 2.91 s total -[ 2023-09-21 08:49:54 ] Completed train batch 0 generator loss: 1.214 66.723 ms, 2.98 s total -[ 2023-09-21 08:49:54 ] Completed train batch 0 generator backward 500.278 ms, 3.48 s total -[ 2023-09-21 08:49:54 ] Completed train batch 0 discriminator loss 0.065 56.526 ms, 3.54 s total -[ 2023-09-21 08:49:55 ] Completed train batch 0 discriminator backward 104.204 ms, 3.64 s total -Epoch [92] Step [0/2], gen_loss: 0.422, disc_loss: 0.022 -[ 2023-09-21 08:49:55 ] Completed train batch 0 metrics update 5.716 ms, 3.65 s total -Saving checkpoint at epoch 92 train batch 0 -[ 2023-09-21 08:49:55 ] Completed saving temp checkpoint 435.245 ms, 4.08 s total -[ 2023-09-21 08:49:55 ] Completed replacing temp checkpoint with checkpoint 25.642 ms, 4.11 s total -[ 2023-09-21 08:49:57 ] Completed train batch 1 to device 1,529.921 ms, 5.64 s total -[ 2023-09-21 08:49:57 ] Completed train batch 1 generator forward 39.712 ms, 5.68 s total -[ 2023-09-21 08:49:57 ] Completed train batch 1 generator loss: 1.289 63.459 ms, 5.74 s total -[ 2023-09-21 08:49:58 ] Completed train batch 1 generator backward 1,129.666 ms, 6.87 s total -[ 2023-09-21 08:49:58 ] Completed train batch 1 discriminator loss 0.064 55.793 ms, 6.93 s total -[ 2023-09-21 08:49:58 ] Completed train batch 1 discriminator backward 74.619 ms, 7.00 s total -Epoch [92] Step [1/2], gen_loss: 0.426, disc_loss: 0.022 -[ 2023-09-21 08:49:58 ] Completed train batch 1 metrics update 1.875 ms, 7.00 s total -Saving checkpoint at epoch 92 train batch 1 -[ 2023-09-21 08:49:58 ] Completed saving temp checkpoint 483.508 ms, 7.49 s total -[ 2023-09-21 08:49:58 ] Completed replacing temp checkpoint with checkpoint 22.755 ms, 7.51 s total -Epoch [92] :: gen_loss: 0.424, disc_loss: 0.022 -[ 2023-09-21 08:49:58 ] Completed training generator for epoch 92 55.788 ms, 7.57 s total -[ 2023-09-21 08:49:58 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 92 from eval step 0 - -[ 2023-09-21 08:50:00 ] Completed eval batch 0 to device 1,067.960 ms, 1.07 s total -[ 2023-09-21 08:50:00 ] Completed eval batch 0 forward 25.414 ms, 1.09 s total -[ 2023-09-21 08:50:00 ] Completed eval batch 0 recons_loss 0.304 ms, 1.09 s total -[ 2023-09-21 08:50:00 ] Completed eval batch 0 metrics update 0.586 ms, 1.09 s total -Saving checkpoint at epoch 92 val batch 0 -[ 2023-09-21 08:50:00 ] Completed saving temp checkpoint 400.325 ms, 1.49 s total -[ 2023-09-21 08:50:00 ] Completed replacing temp checkpoint with checkpoint 26.536 ms, 1.52 s total -[ 2023-09-21 08:50:00 ] Completed eval batch 1 to device 396.832 ms, 1.92 s total -[ 2023-09-21 08:50:00 ] Completed eval batch 1 forward 21.551 ms, 1.94 s total -[ 2023-09-21 08:50:00 ] Completed eval batch 1 recons_loss 0.340 ms, 1.94 s total -[ 2023-09-21 08:50:01 ] Completed eval batch 1 metrics update 285.019 ms, 2.22 s total -Saving checkpoint at epoch 92 val batch 1 -Epoch 92 val loss: 0.0771 -[ 2023-09-21 08:50:01 ] Completed saving temp checkpoint 418.460 ms, 2.64 s total -[ 2023-09-21 08:50:01 ] Completed replacing temp checkpoint with checkpoint 22.137 ms, 2.67 s total -[ 2023-09-21 08:50:01 ] Completed evaluating generator for epoch 92 50.349 ms, 2.72 s total - - -EPOCH :: 93 - - -[ 2023-09-21 08:50:01 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 93 from training step 0 - -[ 2023-09-21 08:50:04 ] Completed train batch 0 to device 2,965.656 ms, 2.97 s total -[ 2023-09-21 08:50:04 ] Completed train batch 0 generator forward 33.314 ms, 3.00 s total -[ 2023-09-21 08:50:04 ] Completed train batch 0 generator loss: 1.285 66.512 ms, 3.07 s total -[ 2023-09-21 08:50:05 ] Completed train batch 0 generator backward 681.252 ms, 3.75 s total -[ 2023-09-21 08:50:05 ] Completed train batch 0 discriminator loss 0.064 56.422 ms, 3.80 s total -[ 2023-09-21 08:50:05 ] Completed train batch 0 discriminator backward 105.179 ms, 3.91 s total -Epoch [93] Step [0/2], gen_loss: 0.422, disc_loss: 0.021 -[ 2023-09-21 08:50:05 ] Completed train batch 0 metrics update 15.974 ms, 3.92 s total -Saving checkpoint at epoch 93 train batch 0 -[ 2023-09-21 08:50:06 ] Completed saving temp checkpoint 422.942 ms, 4.35 s total -[ 2023-09-21 08:50:06 ] Completed replacing temp checkpoint with checkpoint 32.483 ms, 4.38 s total -[ 2023-09-21 08:50:07 ] Completed train batch 1 to device 1,308.132 ms, 5.69 s total -[ 2023-09-21 08:50:07 ] Completed train batch 1 generator forward 23.112 ms, 5.71 s total -[ 2023-09-21 08:50:07 ] Completed train batch 1 generator loss: 1.289 64.847 ms, 5.78 s total -[ 2023-09-21 08:50:08 ] Completed train batch 1 generator backward 1,460.318 ms, 7.24 s total -[ 2023-09-21 08:50:08 ] Completed train batch 1 discriminator loss 0.061 56.436 ms, 7.29 s total -[ 2023-09-21 08:50:09 ] Completed train batch 1 discriminator backward 86.575 ms, 7.38 s total -Epoch [93] Step [1/2], gen_loss: 0.426, disc_loss: 0.020 -[ 2023-09-21 08:50:09 ] Completed train batch 1 metrics update 5.310 ms, 7.38 s total -Saving checkpoint at epoch 93 train batch 1 -[ 2023-09-21 08:50:09 ] Completed saving temp checkpoint 423.224 ms, 7.81 s total -[ 2023-09-21 08:50:09 ] Completed replacing temp checkpoint with checkpoint 21.512 ms, 7.83 s total -Epoch [93] :: gen_loss: 0.424, disc_loss: 0.021 -[ 2023-09-21 08:50:09 ] Completed training generator for epoch 93 51.936 ms, 7.88 s total -[ 2023-09-21 08:50:09 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 93 from eval step 0 - -[ 2023-09-21 08:50:10 ] Completed eval batch 0 to device 1,125.783 ms, 1.13 s total -[ 2023-09-21 08:50:10 ] Completed eval batch 0 forward 31.697 ms, 1.16 s total -[ 2023-09-21 08:50:10 ] Completed eval batch 0 recons_loss 0.321 ms, 1.16 s total -[ 2023-09-21 08:50:10 ] Completed eval batch 0 metrics update 0.579 ms, 1.16 s total -Saving checkpoint at epoch 93 val batch 0 -[ 2023-09-21 08:50:11 ] Completed saving temp checkpoint 392.276 ms, 1.55 s total -[ 2023-09-21 08:50:11 ] Completed replacing temp checkpoint with checkpoint 22.299 ms, 1.57 s total -[ 2023-09-21 08:50:11 ] Completed eval batch 1 to device 457.164 ms, 2.03 s total -[ 2023-09-21 08:50:11 ] Completed eval batch 1 forward 21.511 ms, 2.05 s total -[ 2023-09-21 08:50:11 ] Completed eval batch 1 recons_loss 0.344 ms, 2.05 s total -[ 2023-09-21 08:50:11 ] Completed eval batch 1 metrics update 146.902 ms, 2.20 s total -Saving checkpoint at epoch 93 val batch 1 -Epoch 93 val loss: 0.0753 -[ 2023-09-21 08:50:12 ] Completed saving temp checkpoint 452.113 ms, 2.65 s total -[ 2023-09-21 08:50:12 ] Completed replacing temp checkpoint with checkpoint 22.818 ms, 2.67 s total -[ 2023-09-21 08:50:12 ] Completed evaluating generator for epoch 93 51.088 ms, 2.72 s total - - -EPOCH :: 94 - - -[ 2023-09-21 08:50:12 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 94 from training step 0 - -[ 2023-09-21 08:50:15 ] Completed train batch 0 to device 3,063.700 ms, 3.06 s total -[ 2023-09-21 08:50:15 ] Completed train batch 0 generator forward 28.348 ms, 3.09 s total -[ 2023-09-21 08:50:15 ] Completed train batch 0 generator loss: 1.236 66.632 ms, 3.16 s total -[ 2023-09-21 08:50:15 ] Completed train batch 0 generator backward 450.609 ms, 3.61 s total -[ 2023-09-21 08:50:15 ] Completed train batch 0 discriminator loss 0.059 75.608 ms, 3.68 s total -[ 2023-09-21 08:50:16 ] Completed train batch 0 discriminator backward 82.230 ms, 3.77 s total -Epoch [94] Step [0/2], gen_loss: 0.420, disc_loss: 0.020 -[ 2023-09-21 08:50:16 ] Completed train batch 0 metrics update 8.298 ms, 3.78 s total -Saving checkpoint at epoch 94 train batch 0 -[ 2023-09-21 08:50:16 ] Completed saving temp checkpoint 418.287 ms, 4.19 s total -[ 2023-09-21 08:50:16 ] Completed replacing temp checkpoint with checkpoint 25.319 ms, 4.22 s total -[ 2023-09-21 08:50:18 ] Completed train batch 1 to device 1,614.854 ms, 5.83 s total -[ 2023-09-21 08:50:18 ] Completed train batch 1 generator forward 23.832 ms, 5.86 s total -[ 2023-09-21 08:50:18 ] Completed train batch 1 generator loss: 1.274 65.963 ms, 5.92 s total -[ 2023-09-21 08:50:19 ] Completed train batch 1 generator backward 1,104.187 ms, 7.03 s total -[ 2023-09-21 08:50:19 ] Completed train batch 1 discriminator loss 0.059 59.019 ms, 7.09 s total -[ 2023-09-21 08:50:19 ] Completed train batch 1 discriminator backward 58.181 ms, 7.15 s total -Epoch [94] Step [1/2], gen_loss: 0.428, disc_loss: 0.019 -[ 2023-09-21 08:50:19 ] Completed train batch 1 metrics update 5.017 ms, 7.15 s total -Saving checkpoint at epoch 94 train batch 1 -[ 2023-09-21 08:50:19 ] Completed saving temp checkpoint 466.656 ms, 7.62 s total -[ 2023-09-21 08:50:19 ] Completed replacing temp checkpoint with checkpoint 21.747 ms, 7.64 s total -Epoch [94] :: gen_loss: 0.424, disc_loss: 0.019 -[ 2023-09-21 08:50:19 ] Completed training generator for epoch 94 57.259 ms, 7.70 s total -[ 2023-09-21 08:50:19 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 94 from eval step 0 - -[ 2023-09-21 08:50:21 ] Completed eval batch 0 to device 1,128.398 ms, 1.13 s total -[ 2023-09-21 08:50:21 ] Completed eval batch 0 forward 25.459 ms, 1.15 s total -[ 2023-09-21 08:50:21 ] Completed eval batch 0 recons_loss 0.320 ms, 1.15 s total -[ 2023-09-21 08:50:21 ] Completed eval batch 0 metrics update 0.606 ms, 1.15 s total -Saving checkpoint at epoch 94 val batch 0 -[ 2023-09-21 08:50:21 ] Completed saving temp checkpoint 351.894 ms, 1.51 s total -[ 2023-09-21 08:50:21 ] Completed replacing temp checkpoint with checkpoint 30.848 ms, 1.54 s total -[ 2023-09-21 08:50:21 ] Completed eval batch 1 to device 468.216 ms, 2.01 s total -[ 2023-09-21 08:50:21 ] Completed eval batch 1 forward 21.547 ms, 2.03 s total -[ 2023-09-21 08:50:21 ] Completed eval batch 1 recons_loss 0.338 ms, 2.03 s total -[ 2023-09-21 08:50:22 ] Completed eval batch 1 metrics update 91.798 ms, 2.12 s total -Saving checkpoint at epoch 94 val batch 1 -Epoch 94 val loss: 0.0711 -[ 2023-09-21 08:50:22 ] Completed saving temp checkpoint 414.122 ms, 2.53 s total -[ 2023-09-21 08:50:22 ] Completed replacing temp checkpoint with checkpoint 20.486 ms, 2.55 s total -[ 2023-09-21 08:50:22 ] Completed evaluating generator for epoch 94 49.853 ms, 2.60 s total - - -EPOCH :: 95 - - -[ 2023-09-21 08:50:22 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 95 from training step 0 - -[ 2023-09-21 08:50:25 ] Completed train batch 0 to device 3,136.137 ms, 3.14 s total -[ 2023-09-21 08:50:25 ] Completed train batch 0 generator forward 41.057 ms, 3.18 s total -[ 2023-09-21 08:50:25 ] Completed train batch 0 generator loss: 1.203 65.557 ms, 3.24 s total -[ 2023-09-21 08:50:26 ] Completed train batch 0 generator backward 545.893 ms, 3.79 s total -[ 2023-09-21 08:50:26 ] Completed train batch 0 discriminator loss 0.058 56.472 ms, 3.85 s total -[ 2023-09-21 08:50:26 ] Completed train batch 0 discriminator backward 95.229 ms, 3.94 s total -Epoch [95] Step [0/2], gen_loss: 0.418, disc_loss: 0.019 -[ 2023-09-21 08:50:26 ] Completed train batch 0 metrics update 5.165 ms, 3.95 s total -Saving checkpoint at epoch 95 train batch 0 -[ 2023-09-21 08:50:26 ] Completed saving temp checkpoint 358.983 ms, 4.30 s total -[ 2023-09-21 08:50:26 ] Completed replacing temp checkpoint with checkpoint 18.578 ms, 4.32 s total -[ 2023-09-21 08:50:28 ] Completed train batch 1 to device 1,568.036 ms, 5.89 s total -[ 2023-09-21 08:50:28 ] Completed train batch 1 generator forward 37.903 ms, 5.93 s total -[ 2023-09-21 08:50:28 ] Completed train batch 1 generator loss: 1.223 63.637 ms, 5.99 s total -[ 2023-09-21 08:50:29 ] Completed train batch 1 generator backward 1,224.784 ms, 7.22 s total -[ 2023-09-21 08:50:29 ] Completed train batch 1 discriminator loss 0.054 55.883 ms, 7.27 s total -[ 2023-09-21 08:50:29 ] Completed train batch 1 discriminator backward 80.620 ms, 7.35 s total -Epoch [95] Step [1/2], gen_loss: 0.421, disc_loss: 0.018 -[ 2023-09-21 08:50:29 ] Completed train batch 1 metrics update 2.873 ms, 7.36 s total -Saving checkpoint at epoch 95 train batch 1 -[ 2023-09-21 08:50:30 ] Completed saving temp checkpoint 470.625 ms, 7.83 s total -[ 2023-09-21 08:50:30 ] Completed replacing temp checkpoint with checkpoint 22.603 ms, 7.85 s total -Epoch [95] :: gen_loss: 0.420, disc_loss: 0.019 -[ 2023-09-21 08:50:30 ] Completed training generator for epoch 95 49.735 ms, 7.90 s total -[ 2023-09-21 08:50:30 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 95 from eval step 0 - -[ 2023-09-21 08:50:31 ] Completed eval batch 0 to device 1,082.916 ms, 1.08 s total -[ 2023-09-21 08:50:31 ] Completed eval batch 0 forward 41.575 ms, 1.12 s total -[ 2023-09-21 08:50:31 ] Completed eval batch 0 recons_loss 0.250 ms, 1.12 s total -[ 2023-09-21 08:50:31 ] Completed eval batch 0 metrics update 0.542 ms, 1.13 s total -Saving checkpoint at epoch 95 val batch 0 -[ 2023-09-21 08:50:31 ] Completed saving temp checkpoint 399.327 ms, 1.52 s total -[ 2023-09-21 08:50:32 ] Completed replacing temp checkpoint with checkpoint 26.569 ms, 1.55 s total -[ 2023-09-21 08:50:32 ] Completed eval batch 1 to device 365.562 ms, 1.92 s total -[ 2023-09-21 08:50:32 ] Completed eval batch 1 forward 40.143 ms, 1.96 s total -[ 2023-09-21 08:50:32 ] Completed eval batch 1 recons_loss 0.574 ms, 1.96 s total -[ 2023-09-21 08:50:32 ] Completed eval batch 1 metrics update 163.984 ms, 2.12 s total -Saving checkpoint at epoch 95 val batch 1 -Epoch 95 val loss: 0.0623 -[ 2023-09-21 08:50:33 ] Completed saving temp checkpoint 451.803 ms, 2.57 s total -[ 2023-09-21 08:50:33 ] Completed replacing temp checkpoint with checkpoint 21.917 ms, 2.60 s total -[ 2023-09-21 08:50:33 ] Completed evaluating generator for epoch 95 53.738 ms, 2.65 s total - - -EPOCH :: 96 - - -[ 2023-09-21 08:50:33 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 96 from training step 0 - -[ 2023-09-21 08:50:36 ] Completed train batch 0 to device 2,992.449 ms, 2.99 s total -[ 2023-09-21 08:50:36 ] Completed train batch 0 generator forward 26.961 ms, 3.02 s total -[ 2023-09-21 08:50:36 ] Completed train batch 0 generator loss: 1.224 66.396 ms, 3.09 s total -[ 2023-09-21 08:50:36 ] Completed train batch 0 generator backward 503.869 ms, 3.59 s total -[ 2023-09-21 08:50:36 ] Completed train batch 0 discriminator loss 0.051 56.577 ms, 3.65 s total -[ 2023-09-21 08:50:36 ] Completed train batch 0 discriminator backward 87.840 ms, 3.73 s total -Epoch [96] Step [0/2], gen_loss: 0.421, disc_loss: 0.017 -[ 2023-09-21 08:50:36 ] Completed train batch 0 metrics update 12.389 ms, 3.75 s total -Saving checkpoint at epoch 96 train batch 0 -[ 2023-09-21 08:50:37 ] Completed saving temp checkpoint 367.978 ms, 4.11 s total -[ 2023-09-21 08:50:37 ] Completed replacing temp checkpoint with checkpoint 21.711 ms, 4.14 s total -[ 2023-09-21 08:50:38 ] Completed train batch 1 to device 1,629.278 ms, 5.77 s total -[ 2023-09-21 08:50:38 ] Completed train batch 1 generator forward 22.273 ms, 5.79 s total -[ 2023-09-21 08:50:38 ] Completed train batch 1 generator loss: 1.276 63.434 ms, 5.85 s total -[ 2023-09-21 08:50:40 ] Completed train batch 1 generator backward 1,133.581 ms, 6.98 s total -[ 2023-09-21 08:50:40 ] Completed train batch 1 discriminator loss 0.053 55.728 ms, 7.04 s total -[ 2023-09-21 08:50:40 ] Completed train batch 1 discriminator backward 69.563 ms, 7.11 s total -Epoch [96] Step [1/2], gen_loss: 0.423, disc_loss: 0.018 -[ 2023-09-21 08:50:40 ] Completed train batch 1 metrics update 1.950 ms, 7.11 s total -Saving checkpoint at epoch 96 train batch 1 -[ 2023-09-21 08:50:40 ] Completed saving temp checkpoint 464.833 ms, 7.58 s total -[ 2023-09-21 08:50:40 ] Completed replacing temp checkpoint with checkpoint 26.742 ms, 7.60 s total -Epoch [96] :: gen_loss: 0.422, disc_loss: 0.018 -[ 2023-09-21 08:50:40 ] Completed training generator for epoch 96 63.028 ms, 7.67 s total -[ 2023-09-21 08:50:40 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 96 from eval step 0 - -[ 2023-09-21 08:50:41 ] Completed eval batch 0 to device 1,077.419 ms, 1.08 s total -[ 2023-09-21 08:50:41 ] Completed eval batch 0 forward 43.355 ms, 1.12 s total -[ 2023-09-21 08:50:41 ] Completed eval batch 0 recons_loss 0.377 ms, 1.12 s total -[ 2023-09-21 08:50:41 ] Completed eval batch 0 metrics update 0.578 ms, 1.12 s total -Saving checkpoint at epoch 96 val batch 0 -[ 2023-09-21 08:50:42 ] Completed saving temp checkpoint 383.158 ms, 1.50 s total -[ 2023-09-21 08:50:42 ] Completed replacing temp checkpoint with checkpoint 22.101 ms, 1.53 s total -[ 2023-09-21 08:50:42 ] Completed eval batch 1 to device 400.853 ms, 1.93 s total -[ 2023-09-21 08:50:42 ] Completed eval batch 1 forward 33.008 ms, 1.96 s total -[ 2023-09-21 08:50:42 ] Completed eval batch 1 recons_loss 0.390 ms, 1.96 s total -[ 2023-09-21 08:50:42 ] Completed eval batch 1 metrics update 165.486 ms, 2.13 s total -Saving checkpoint at epoch 96 val batch 1 -Epoch 96 val loss: 0.0653 -[ 2023-09-21 08:50:43 ] Completed saving temp checkpoint 429.637 ms, 2.56 s total -[ 2023-09-21 08:50:43 ] Completed replacing temp checkpoint with checkpoint 30.061 ms, 2.59 s total -[ 2023-09-21 08:50:43 ] Completed evaluating generator for epoch 96 53.936 ms, 2.64 s total - - -EPOCH :: 97 - - -[ 2023-09-21 08:50:43 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 97 from training step 0 - -[ 2023-09-21 08:50:46 ] Completed train batch 0 to device 3,024.185 ms, 3.02 s total -[ 2023-09-21 08:50:46 ] Completed train batch 0 generator forward 44.007 ms, 3.07 s total -[ 2023-09-21 08:50:46 ] Completed train batch 0 generator loss: 1.244 66.023 ms, 3.13 s total -[ 2023-09-21 08:50:47 ] Completed train batch 0 generator backward 599.942 ms, 3.73 s total -[ 2023-09-21 08:50:47 ] Completed train batch 0 discriminator loss 0.054 75.593 ms, 3.81 s total -[ 2023-09-21 08:50:47 ] Completed train batch 0 discriminator backward 71.806 ms, 3.88 s total -Epoch [97] Step [0/2], gen_loss: 0.420, disc_loss: 0.018 -[ 2023-09-21 08:50:47 ] Completed train batch 0 metrics update 0.684 ms, 3.88 s total -Saving checkpoint at epoch 97 train batch 0 -[ 2023-09-21 08:50:47 ] Completed saving temp checkpoint 396.388 ms, 4.28 s total -[ 2023-09-21 08:50:47 ] Completed replacing temp checkpoint with checkpoint 30.673 ms, 4.31 s total -[ 2023-09-21 08:50:49 ] Completed train batch 1 to device 1,530.837 ms, 5.84 s total -[ 2023-09-21 08:50:49 ] Completed train batch 1 generator forward 41.378 ms, 5.88 s total -[ 2023-09-21 08:50:49 ] Completed train batch 1 generator loss: 1.248 63.925 ms, 5.95 s total -[ 2023-09-21 08:50:50 ] Completed train batch 1 generator backward 1,215.917 ms, 7.16 s total -[ 2023-09-21 08:50:50 ] Completed train batch 1 discriminator loss 0.052 55.929 ms, 7.22 s total -[ 2023-09-21 08:50:50 ] Completed train batch 1 discriminator backward 73.024 ms, 7.29 s total -Epoch [97] Step [1/2], gen_loss: 0.427, disc_loss: 0.018 -[ 2023-09-21 08:50:50 ] Completed train batch 1 metrics update 0.400 ms, 7.29 s total -Saving checkpoint at epoch 97 train batch 1 -[ 2023-09-21 08:50:51 ] Completed saving temp checkpoint 425.599 ms, 7.72 s total -[ 2023-09-21 08:50:51 ] Completed replacing temp checkpoint with checkpoint 25.642 ms, 7.74 s total -Epoch [97] :: gen_loss: 0.423, disc_loss: 0.018 -[ 2023-09-21 08:50:51 ] Completed training generator for epoch 97 67.135 ms, 7.81 s total -[ 2023-09-21 08:50:51 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 97 from eval step 0 - -[ 2023-09-21 08:50:52 ] Completed eval batch 0 to device 1,081.840 ms, 1.08 s total -[ 2023-09-21 08:50:52 ] Completed eval batch 0 forward 44.007 ms, 1.13 s total -[ 2023-09-21 08:50:52 ] Completed eval batch 0 recons_loss 0.313 ms, 1.13 s total -[ 2023-09-21 08:50:52 ] Completed eval batch 0 metrics update 0.573 ms, 1.13 s total -Saving checkpoint at epoch 97 val batch 0 -[ 2023-09-21 08:50:52 ] Completed saving temp checkpoint 416.440 ms, 1.54 s total -[ 2023-09-21 08:50:52 ] Completed replacing temp checkpoint with checkpoint 25.529 ms, 1.57 s total -[ 2023-09-21 08:50:53 ] Completed eval batch 1 to device 355.590 ms, 1.92 s total -[ 2023-09-21 08:50:53 ] Completed eval batch 1 forward 39.866 ms, 1.96 s total -[ 2023-09-21 08:50:53 ] Completed eval batch 1 recons_loss 0.341 ms, 1.96 s total -[ 2023-09-21 08:50:53 ] Completed eval batch 1 metrics update 111.569 ms, 2.08 s total -Saving checkpoint at epoch 97 val batch 1 -Epoch 97 val loss: 0.0613 -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 09:04:39 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 09:04:39 ] Completed importing Timer 0.020 ms, 0.00 s total -[ 2023-09-21 09:04:44 ] Completed importing everything else 4,945.590 ms, 4.95 s total -| distributed init (rank 2): env:// -| distributed init (rank 1): env:// -| distributed init (rank 5): env:// -| distributed init (rank 4): env:// -| distributed init (rank 0): env:// -| distributed init (rank 3): env:// -[ 2023-09-21 09:04:52 ] Completed preliminaries 7,978.812 ms, 12.92 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 09:04:52 ] Completed build datasets 11.549 ms, 12.94 s total -[ 2023-09-21 09:04:52 ] Completed build samplers 0.049 ms, 12.94 s total -[ 2023-09-21 09:04:52 ] Completed build dataloaders 0.136 ms, 12.94 s total -[ 2023-09-21 09:04:52 ] Completed generator to device 80.354 ms, 13.02 s total -[ 2023-09-21 09:04:52 ] Completed discriminator to device 7.366 ms, 13.02 s total -[ 2023-09-21 09:04:52 ] Completed loss functions 383.310 ms, 13.41 s total -[ 2023-09-21 09:04:52 ] Completed models prepped for distribution 99.896 ms, 13.51 s total -[ 2023-09-21 09:04:52 ] Completed optimizers 0.655 ms, 13.51 s total -[ 2023-09-21 09:04:52 ] Completed grad scalers 0.029 ms, 13.51 s total -[ 2023-09-21 09:04:53 ] Completed checkpoint retrieval 474.925 ms, 13.98 s total - - -EPOCH :: 97 - - -[ 2023-09-21 09:04:53 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 97 from training step 2 - -Epoch [97] :: gen_loss: 0.423, disc_loss: 0.018 -[ 2023-09-21 09:04:53 ] Completed training generator for epoch 97 128.915 ms, 0.13 s total -[ 2023-09-21 09:04:53 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 97 from eval step 1 - -[ 2023-09-21 09:04:54 ] Completed eval batch 1 to device 1,547.702 ms, 1.55 s total -[ 2023-09-21 09:04:56 ] Completed eval batch 1 forward 1,111.353 ms, 2.66 s total -[ 2023-09-21 09:04:56 ] Completed eval batch 1 recons_loss 0.262 ms, 2.66 s total -[ 2023-09-21 09:04:56 ] Completed eval batch 1 metrics update 453.418 ms, 3.11 s total -Saving checkpoint at epoch 97 val batch 1 -Epoch 97 val loss: 0.0597 -[ 2023-09-21 09:04:56 ] Completed saving temp checkpoint 350.483 ms, 3.46 s total -[ 2023-09-21 09:04:56 ] Completed replacing temp checkpoint with checkpoint 121.903 ms, 3.59 s total -[ 2023-09-21 09:04:56 ] Completed evaluating generator for epoch 97 44.084 ms, 3.63 s total - - -EPOCH :: 98 - - -[ 2023-09-21 09:04:56 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 98 from training step 0 - -[ 2023-09-21 09:05:00 ] Completed train batch 0 to device 3,103.570 ms, 3.10 s total -[ 2023-09-21 09:05:00 ] Completed train batch 0 generator forward 33.859 ms, 3.14 s total -[ 2023-09-21 09:05:00 ] Completed train batch 0 generator loss: 1.289 70.038 ms, 3.21 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 09:05:00 ] Completed train batch 0 generator backward 686.476 ms, 3.89 s total -[ 2023-09-21 09:05:00 ] Completed train batch 0 discriminator loss 0.054 55.546 ms, 3.95 s total -[ 2023-09-21 09:05:01 ] Completed train batch 0 discriminator backward 102.629 ms, 4.05 s total -Epoch [98] Step [0/2], gen_loss: 0.431, disc_loss: 0.018 -[ 2023-09-21 09:05:01 ] Completed train batch 0 metrics update 6.263 ms, 4.06 s total -Saving checkpoint at epoch 98 train batch 0 -[ 2023-09-21 09:05:01 ] Completed saving temp checkpoint 443.161 ms, 4.50 s total -[ 2023-09-21 09:05:01 ] Completed replacing temp checkpoint with checkpoint 21.252 ms, 4.52 s total -[ 2023-09-21 09:05:03 ] Completed train batch 1 to device 1,627.861 ms, 6.15 s total -[ 2023-09-21 09:05:03 ] Completed train batch 1 generator forward 25.533 ms, 6.18 s total -[ 2023-09-21 09:05:03 ] Completed train batch 1 generator loss: 1.283 62.189 ms, 6.24 s total -[ 2023-09-21 09:05:04 ] Completed train batch 1 generator backward 1,041.873 ms, 7.28 s total -[ 2023-09-21 09:05:04 ] Completed train batch 1 discriminator loss 0.056 55.876 ms, 7.34 s total -[ 2023-09-21 09:05:04 ] Completed train batch 1 discriminator backward 66.381 ms, 7.40 s total -Epoch [98] Step [1/2], gen_loss: 0.420, disc_loss: 0.019 -[ 2023-09-21 09:05:04 ] Completed train batch 1 metrics update 2.823 ms, 7.41 s total -Saving checkpoint at epoch 98 train batch 1 -[ 2023-09-21 09:05:04 ] Completed saving temp checkpoint 438.400 ms, 7.84 s total -[ 2023-09-21 09:05:04 ] Completed replacing temp checkpoint with checkpoint 23.734 ms, 7.87 s total -Epoch [98] :: gen_loss: 0.426, disc_loss: 0.018 -[ 2023-09-21 09:05:04 ] Completed training generator for epoch 98 55.206 ms, 7.92 s total -[ 2023-09-21 09:05:04 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 98 from eval step 0 - -[ 2023-09-21 09:05:06 ] Completed eval batch 0 to device 1,096.301 ms, 1.10 s total -[ 2023-09-21 09:05:06 ] Completed eval batch 0 forward 24.919 ms, 1.12 s total -[ 2023-09-21 09:05:06 ] Completed eval batch 0 recons_loss 0.295 ms, 1.12 s total -[ 2023-09-21 09:05:06 ] Completed eval batch 0 metrics update 0.593 ms, 1.12 s total -Saving checkpoint at epoch 98 val batch 0 -[ 2023-09-21 09:05:06 ] Completed saving temp checkpoint 381.706 ms, 1.50 s total -[ 2023-09-21 09:05:06 ] Completed replacing temp checkpoint with checkpoint 22.987 ms, 1.53 s total -[ 2023-09-21 09:05:06 ] Completed eval batch 1 to device 452.805 ms, 1.98 s total -[ 2023-09-21 09:05:06 ] Completed eval batch 1 forward 39.107 ms, 2.02 s total -[ 2023-09-21 09:05:06 ] Completed eval batch 1 recons_loss 0.553 ms, 2.02 s total -[ 2023-09-21 09:05:07 ] Completed eval batch 1 metrics update 253.018 ms, 2.27 s total -Saving checkpoint at epoch 98 val batch 1 -Epoch 98 val loss: 0.0687 -[ 2023-09-21 09:05:07 ] Completed saving temp checkpoint 411.712 ms, 2.68 s total -[ 2023-09-21 09:05:07 ] Completed replacing temp checkpoint with checkpoint 27.734 ms, 2.71 s total -[ 2023-09-21 09:05:07 ] Completed evaluating generator for epoch 98 51.729 ms, 2.76 s total - - -EPOCH :: 99 - - -[ 2023-09-21 09:05:07 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 99 from training step 0 - -[ 2023-09-21 09:05:10 ] Completed train batch 0 to device 2,980.073 ms, 2.98 s total -[ 2023-09-21 09:05:10 ] Completed train batch 0 generator forward 24.550 ms, 3.00 s total -[ 2023-09-21 09:05:10 ] Completed train batch 0 generator loss: 1.302 63.461 ms, 3.07 s total -[ 2023-09-21 09:05:11 ] Completed train batch 0 generator backward 741.768 ms, 3.81 s total -[ 2023-09-21 09:05:11 ] Completed train batch 0 discriminator loss 0.058 56.786 ms, 3.87 s total -[ 2023-09-21 09:05:11 ] Completed train batch 0 discriminator backward 96.073 ms, 3.96 s total -Epoch [99] Step [0/2], gen_loss: 0.426, disc_loss: 0.019 -[ 2023-09-21 09:05:11 ] Completed train batch 0 metrics update 5.610 ms, 3.97 s total -Saving checkpoint at epoch 99 train batch 0 -[ 2023-09-21 09:05:12 ] Completed saving temp checkpoint 384.608 ms, 4.35 s total -[ 2023-09-21 09:05:12 ] Completed replacing temp checkpoint with checkpoint 18.394 ms, 4.37 s total -[ 2023-09-21 09:05:13 ] Completed train batch 1 to device 1,181.832 ms, 5.55 s total -[ 2023-09-21 09:05:13 ] Completed train batch 1 generator forward 41.383 ms, 5.59 s total -[ 2023-09-21 09:05:13 ] Completed train batch 1 generator loss: 1.314 61.957 ms, 5.66 s total -[ 2023-09-21 09:05:14 ] Completed train batch 1 generator backward 1,448.958 ms, 7.11 s total -[ 2023-09-21 09:05:14 ] Completed train batch 1 discriminator loss 0.069 56.044 ms, 7.16 s total -[ 2023-09-21 09:05:14 ] Completed train batch 1 discriminator backward 69.443 ms, 7.23 s total -Epoch [99] Step [1/2], gen_loss: 0.430, disc_loss: 0.019 -[ 2023-09-21 09:05:14 ] Completed train batch 1 metrics update 2.796 ms, 7.23 s total -Saving checkpoint at epoch 99 train batch 1 -[ 2023-09-21 09:05:15 ] Completed saving temp checkpoint 426.307 ms, 7.66 s total -[ 2023-09-21 09:05:15 ] Completed replacing temp checkpoint with checkpoint 21.965 ms, 7.68 s total -Epoch [99] :: gen_loss: 0.428, disc_loss: 0.019 -[ 2023-09-21 09:05:15 ] Completed training generator for epoch 99 55.724 ms, 7.74 s total -[ 2023-09-21 09:05:15 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 99 from eval step 0 - -[ 2023-09-21 09:05:16 ] Completed eval batch 0 to device 1,094.851 ms, 1.09 s total -[ 2023-09-21 09:05:16 ] Completed eval batch 0 forward 31.004 ms, 1.13 s total -[ 2023-09-21 09:05:16 ] Completed eval batch 0 recons_loss 0.345 ms, 1.13 s total -[ 2023-09-21 09:05:16 ] Completed eval batch 0 metrics update 0.640 ms, 1.13 s total -Saving checkpoint at epoch 99 val batch 0 -[ 2023-09-21 09:05:16 ] Completed saving temp checkpoint 447.265 ms, 1.57 s total -[ 2023-09-21 09:05:17 ] Completed replacing temp checkpoint with checkpoint 22.580 ms, 1.60 s total -[ 2023-09-21 09:05:17 ] Completed eval batch 1 to device 397.169 ms, 1.99 s total -[ 2023-09-21 09:05:17 ] Completed eval batch 1 forward 27.069 ms, 2.02 s total -[ 2023-09-21 09:05:17 ] Completed eval batch 1 recons_loss 0.395 ms, 2.02 s total -[ 2023-09-21 09:05:17 ] Completed eval batch 1 metrics update 274.357 ms, 2.30 s total -Saving checkpoint at epoch 99 val batch 1 -Epoch 99 val loss: 0.0747 -[ 2023-09-21 09:05:18 ] Completed saving temp checkpoint 492.923 ms, 2.79 s total -[ 2023-09-21 09:05:18 ] Completed replacing temp checkpoint with checkpoint 19.645 ms, 2.81 s total -[ 2023-09-21 09:05:18 ] Completed evaluating generator for epoch 99 52.333 ms, 2.86 s total - - -EPOCH :: 100 - - -[ 2023-09-21 09:05:18 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 100 from training step 0 - -[ 2023-09-21 09:05:21 ] Completed train batch 0 to device 2,891.001 ms, 2.89 s total -[ 2023-09-21 09:05:21 ] Completed train batch 0 generator forward 24.771 ms, 2.92 s total -[ 2023-09-21 09:05:21 ] Completed train batch 0 generator loss: 1.276 64.803 ms, 2.98 s total -[ 2023-09-21 09:05:21 ] Completed train batch 0 generator backward 751.054 ms, 3.73 s total -[ 2023-09-21 09:05:22 ] Completed train batch 0 discriminator loss 0.062 75.276 ms, 3.81 s total -[ 2023-09-21 09:05:22 ] Completed train batch 0 discriminator backward 82.037 ms, 3.89 s total -Epoch [100] Step [0/2], gen_loss: 0.427, disc_loss: 0.021 -[ 2023-09-21 09:05:22 ] Completed train batch 0 metrics update 12.213 ms, 3.90 s total -Saving checkpoint at epoch 100 train batch 0 -[ 2023-09-21 09:05:22 ] Completed saving temp checkpoint 437.107 ms, 4.34 s total -[ 2023-09-21 09:05:22 ] Completed replacing temp checkpoint with checkpoint 22.387 ms, 4.36 s total -[ 2023-09-21 09:05:23 ] Completed train batch 1 to device 1,290.804 ms, 5.65 s total -[ 2023-09-21 09:05:23 ] Completed train batch 1 generator forward 38.132 ms, 5.69 s total -[ 2023-09-21 09:05:24 ] Completed train batch 1 generator loss: 1.293 64.538 ms, 5.75 s total -[ 2023-09-21 09:05:25 ] Completed train batch 1 generator backward 1,542.999 ms, 7.30 s total -[ 2023-09-21 09:05:25 ] Completed train batch 1 discriminator loss 0.067 55.804 ms, 7.35 s total -[ 2023-09-21 09:05:25 ] Completed train batch 1 discriminator backward 71.175 ms, 7.42 s total -Epoch [100] Step [1/2], gen_loss: 0.446, disc_loss: 0.021 -[ 2023-09-21 09:05:25 ] Completed train batch 1 metrics update 2.032 ms, 7.43 s total -Saving checkpoint at epoch 100 train batch 1 -[ 2023-09-21 09:05:26 ] Completed saving temp checkpoint 449.042 ms, 7.88 s total -[ 2023-09-21 09:05:26 ] Completed replacing temp checkpoint with checkpoint 23.054 ms, 7.90 s total -Epoch [100] :: gen_loss: 0.437, disc_loss: 0.021 -[ 2023-09-21 09:05:26 ] Completed training generator for epoch 100 49.971 ms, 7.95 s total -[ 2023-09-21 09:05:26 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 100 from eval step 0 - -[ 2023-09-21 09:05:27 ] Completed eval batch 0 to device 1,097.957 ms, 1.10 s total -[ 2023-09-21 09:05:27 ] Completed eval batch 0 forward 24.411 ms, 1.12 s total -[ 2023-09-21 09:05:27 ] Completed eval batch 0 recons_loss 0.388 ms, 1.12 s total -[ 2023-09-21 09:05:27 ] Completed eval batch 0 metrics update 0.671 ms, 1.12 s total -Saving checkpoint at epoch 100 val batch 0 -[ 2023-09-21 09:05:27 ] Completed saving temp checkpoint 378.759 ms, 1.50 s total -[ 2023-09-21 09:05:27 ] Completed replacing temp checkpoint with checkpoint 21.271 ms, 1.52 s total -[ 2023-09-21 09:05:28 ] Completed eval batch 1 to device 490.156 ms, 2.01 s total -[ 2023-09-21 09:05:28 ] Completed eval batch 1 forward 21.177 ms, 2.03 s total -[ 2023-09-21 09:05:28 ] Completed eval batch 1 recons_loss 0.342 ms, 2.04 s total -[ 2023-09-21 09:05:28 ] Completed eval batch 1 metrics update 216.298 ms, 2.25 s total -Saving checkpoint at epoch 100 val batch 1 -Epoch 100 val loss: 0.0564 -[ 2023-09-21 09:05:28 ] Completed saving temp checkpoint 497.737 ms, 2.75 s total -[ 2023-09-21 09:05:28 ] Completed replacing temp checkpoint with checkpoint 22.628 ms, 2.77 s total -[ 2023-09-21 09:05:29 ] Completed evaluating generator for epoch 100 61.719 ms, 2.83 s total - - -EPOCH :: 101 - - -[ 2023-09-21 09:05:29 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 101 from training step 0 - -[ 2023-09-21 09:05:32 ] Completed train batch 0 to device 3,122.790 ms, 3.12 s total -[ 2023-09-21 09:05:32 ] Completed train batch 0 generator forward 37.788 ms, 3.16 s total -[ 2023-09-21 09:05:32 ] Completed train batch 0 generator loss: 1.324 81.898 ms, 3.24 s total -[ 2023-09-21 09:05:32 ] Completed train batch 0 generator backward 520.331 ms, 3.76 s total -[ 2023-09-21 09:05:32 ] Completed train batch 0 discriminator loss 0.061 56.570 ms, 3.82 s total -[ 2023-09-21 09:05:32 ] Completed train batch 0 discriminator backward 99.804 ms, 3.92 s total -Epoch [101] Step [0/2], gen_loss: 0.423, disc_loss: 0.020 -[ 2023-09-21 09:05:32 ] Completed train batch 0 metrics update 26.518 ms, 3.95 s total -Saving checkpoint at epoch 101 train batch 0 -[ 2023-09-21 09:05:33 ] Completed saving temp checkpoint 439.259 ms, 4.38 s total -[ 2023-09-21 09:05:33 ] Completed replacing temp checkpoint with checkpoint 21.629 ms, 4.41 s total -[ 2023-09-21 09:05:35 ] Completed train batch 1 to device 1,649.682 ms, 6.06 s total -[ 2023-09-21 09:05:35 ] Completed train batch 1 generator forward 41.247 ms, 6.10 s total -[ 2023-09-21 09:05:35 ] Completed train batch 1 generator loss: 1.236 64.109 ms, 6.16 s total -[ 2023-09-21 09:05:36 ] Completed train batch 1 generator backward 1,175.436 ms, 7.34 s total -[ 2023-09-21 09:05:36 ] Completed train batch 1 discriminator loss 0.061 56.161 ms, 7.39 s total -[ 2023-09-21 09:05:36 ] Completed train batch 1 discriminator backward 87.835 ms, 7.48 s total -Epoch [101] Step [1/2], gen_loss: 0.414, disc_loss: 0.020 -[ 2023-09-21 09:05:36 ] Completed train batch 1 metrics update 5.363 ms, 7.49 s total -Saving checkpoint at epoch 101 train batch 1 -[ 2023-09-21 09:05:37 ] Completed saving temp checkpoint 493.990 ms, 7.98 s total -[ 2023-09-21 09:05:37 ] Completed replacing temp checkpoint with checkpoint 18.647 ms, 8.00 s total -Epoch [101] :: gen_loss: 0.419, disc_loss: 0.020 -[ 2023-09-21 09:05:37 ] Completed training generator for epoch 101 55.961 ms, 8.06 s total -[ 2023-09-21 09:05:37 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 101 from eval step 0 - -[ 2023-09-21 09:05:38 ] Completed eval batch 0 to device 1,063.648 ms, 1.06 s total -[ 2023-09-21 09:05:38 ] Completed eval batch 0 forward 31.081 ms, 1.09 s total -[ 2023-09-21 09:05:38 ] Completed eval batch 0 recons_loss 0.479 ms, 1.10 s total -[ 2023-09-21 09:05:38 ] Completed eval batch 0 metrics update 0.606 ms, 1.10 s total -Saving checkpoint at epoch 101 val batch 0 -[ 2023-09-21 09:05:38 ] Completed saving temp checkpoint 393.530 ms, 1.49 s total -[ 2023-09-21 09:05:38 ] Completed replacing temp checkpoint with checkpoint 23.116 ms, 1.51 s total -[ 2023-09-21 09:05:39 ] Completed eval batch 1 to device 427.068 ms, 1.94 s total -[ 2023-09-21 09:05:39 ] Completed eval batch 1 forward 21.093 ms, 1.96 s total -[ 2023-09-21 09:05:39 ] Completed eval batch 1 recons_loss 0.332 ms, 1.96 s total -[ 2023-09-21 09:05:39 ] Completed eval batch 1 metrics update 290.689 ms, 2.25 s total -Saving checkpoint at epoch 101 val batch 1 -Epoch 101 val loss: 0.0689 -[ 2023-09-21 09:05:39 ] Completed saving temp checkpoint 438.752 ms, 2.69 s total -[ 2023-09-21 09:05:39 ] Completed replacing temp checkpoint with checkpoint 23.247 ms, 2.71 s total -[ 2023-09-21 09:05:39 ] Completed evaluating generator for epoch 101 48.632 ms, 2.76 s total - - -EPOCH :: 102 - - -[ 2023-09-21 09:05:39 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 102 from training step 0 - -[ 2023-09-21 09:05:42 ] Completed train batch 0 to device 2,988.940 ms, 2.99 s total -[ 2023-09-21 09:05:42 ] Completed train batch 0 generator forward 32.924 ms, 3.02 s total -[ 2023-09-21 09:05:42 ] Completed train batch 0 generator loss: 1.303 70.749 ms, 3.09 s total -[ 2023-09-21 09:05:43 ] Completed train batch 0 generator backward 840.396 ms, 3.93 s total -[ 2023-09-21 09:05:43 ] Completed train batch 0 discriminator loss 0.063 56.352 ms, 3.99 s total -[ 2023-09-21 09:05:43 ] Completed train batch 0 discriminator backward 89.530 ms, 4.08 s total -Epoch [102] Step [0/2], gen_loss: 0.420, disc_loss: 0.021 -[ 2023-09-21 09:05:43 ] Completed train batch 0 metrics update 5.531 ms, 4.08 s total -Saving checkpoint at epoch 102 train batch 0 -[ 2023-09-21 09:05:44 ] Completed saving temp checkpoint 398.136 ms, 4.48 s total -[ 2023-09-21 09:05:44 ] Completed replacing temp checkpoint with checkpoint 29.949 ms, 4.51 s total -[ 2023-09-21 09:05:45 ] Completed train batch 1 to device 1,309.919 ms, 5.82 s total -[ 2023-09-21 09:05:45 ] Completed train batch 1 generator forward 23.978 ms, 5.85 s total -[ 2023-09-21 09:05:45 ] Completed train batch 1 generator loss: 1.316 62.174 ms, 5.91 s total -[ 2023-09-21 09:05:47 ] Completed train batch 1 generator backward 1,569.532 ms, 7.48 s total -[ 2023-09-21 09:05:47 ] Completed train batch 1 discriminator loss 0.061 57.146 ms, 7.54 s total -[ 2023-09-21 09:05:47 ] Completed train batch 1 discriminator backward 67.695 ms, 7.60 s total -Epoch [102] Step [1/2], gen_loss: 0.435, disc_loss: 0.020 -[ 2023-09-21 09:05:47 ] Completed train batch 1 metrics update 0.346 ms, 7.60 s total -Saving checkpoint at epoch 102 train batch 1 -[ 2023-09-21 09:05:47 ] Completed saving temp checkpoint 431.587 ms, 8.03 s total -[ 2023-09-21 09:05:47 ] Completed replacing temp checkpoint with checkpoint 15.982 ms, 8.05 s total -Epoch [102] :: gen_loss: 0.428, disc_loss: 0.021 -[ 2023-09-21 09:05:47 ] Completed training generator for epoch 102 60.162 ms, 8.11 s total -[ 2023-09-21 09:05:47 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 102 from eval step 0 - -[ 2023-09-21 09:05:49 ] Completed eval batch 0 to device 1,088.676 ms, 1.09 s total -[ 2023-09-21 09:05:49 ] Completed eval batch 0 forward 35.919 ms, 1.12 s total -[ 2023-09-21 09:05:49 ] Completed eval batch 0 recons_loss 0.301 ms, 1.12 s total -[ 2023-09-21 09:05:49 ] Completed eval batch 0 metrics update 0.555 ms, 1.13 s total -Saving checkpoint at epoch 102 val batch 0 -[ 2023-09-21 09:05:49 ] Completed saving temp checkpoint 407.814 ms, 1.53 s total -[ 2023-09-21 09:05:49 ] Completed replacing temp checkpoint with checkpoint 27.393 ms, 1.56 s total -[ 2023-09-21 09:05:49 ] Completed eval batch 1 to device 373.202 ms, 1.93 s total -[ 2023-09-21 09:05:49 ] Completed eval batch 1 forward 20.983 ms, 1.95 s total -[ 2023-09-21 09:05:49 ] Completed eval batch 1 recons_loss 0.327 ms, 1.96 s total -[ 2023-09-21 09:05:50 ] Completed eval batch 1 metrics update 325.189 ms, 2.28 s total -Saving checkpoint at epoch 102 val batch 1 -Epoch 102 val loss: 0.0679 -[ 2023-09-21 09:05:50 ] Completed saving temp checkpoint 379.258 ms, 2.66 s total -[ 2023-09-21 09:05:50 ] Completed replacing temp checkpoint with checkpoint 28.764 ms, 2.69 s total -[ 2023-09-21 09:05:50 ] Completed evaluating generator for epoch 102 55.988 ms, 2.74 s total - - -EPOCH :: 103 - - -[ 2023-09-21 09:05:50 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 103 from training step 0 - -[ 2023-09-21 09:05:53 ] Completed train batch 0 to device 3,189.901 ms, 3.19 s total -[ 2023-09-21 09:05:53 ] Completed train batch 0 generator forward 29.903 ms, 3.22 s total -[ 2023-09-21 09:05:54 ] Completed train batch 0 generator loss: 1.297 65.242 ms, 3.29 s total -[ 2023-09-21 09:05:54 ] Completed train batch 0 generator backward 587.118 ms, 3.87 s total -[ 2023-09-21 09:05:54 ] Completed train batch 0 discriminator loss 0.063 59.645 ms, 3.93 s total -[ 2023-09-21 09:05:54 ] Completed train batch 0 discriminator backward 98.898 ms, 4.03 s total -Epoch [103] Step [0/2], gen_loss: 0.413, disc_loss: 0.021 -[ 2023-09-21 09:05:54 ] Completed train batch 0 metrics update 4.318 ms, 4.04 s total -Saving checkpoint at epoch 103 train batch 0 -[ 2023-09-21 09:05:55 ] Completed saving temp checkpoint 369.789 ms, 4.40 s total -[ 2023-09-21 09:05:55 ] Completed replacing temp checkpoint with checkpoint 26.961 ms, 4.43 s total -[ 2023-09-21 09:05:56 ] Completed train batch 1 to device 1,704.911 ms, 6.14 s total -[ 2023-09-21 09:05:56 ] Completed train batch 1 generator forward 41.647 ms, 6.18 s total -[ 2023-09-21 09:05:56 ] Completed train batch 1 generator loss: 1.334 62.267 ms, 6.24 s total -[ 2023-09-21 09:05:58 ] Completed train batch 1 generator backward 1,280.121 ms, 7.52 s total -[ 2023-09-21 09:05:58 ] Completed train batch 1 discriminator loss 0.079 55.960 ms, 7.58 s total -[ 2023-09-21 09:05:58 ] Completed train batch 1 discriminator backward 75.147 ms, 7.65 s total -Epoch [103] Step [1/2], gen_loss: 0.420, disc_loss: 0.026 -[ 2023-09-21 09:05:58 ] Completed train batch 1 metrics update 1.933 ms, 7.65 s total -Saving checkpoint at epoch 103 train batch 1 -[ 2023-09-21 09:05:58 ] Completed saving temp checkpoint 431.677 ms, 8.09 s total -[ 2023-09-21 09:05:58 ] Completed replacing temp checkpoint with checkpoint 22.061 ms, 8.11 s total -Epoch [103] :: gen_loss: 0.417, disc_loss: 0.024 -[ 2023-09-21 09:05:58 ] Completed training generator for epoch 103 55.741 ms, 8.16 s total -[ 2023-09-21 09:05:58 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 103 from eval step 0 - -[ 2023-09-21 09:05:59 ] Completed eval batch 0 to device 1,046.044 ms, 1.05 s total -[ 2023-09-21 09:05:59 ] Completed eval batch 0 forward 23.653 ms, 1.07 s total -[ 2023-09-21 09:05:59 ] Completed eval batch 0 recons_loss 0.255 ms, 1.07 s total -[ 2023-09-21 09:05:59 ] Completed eval batch 0 metrics update 3.817 ms, 1.07 s total -Saving checkpoint at epoch 103 val batch 0 -[ 2023-09-21 09:06:00 ] Completed saving temp checkpoint 386.366 ms, 1.46 s total -[ 2023-09-21 09:06:00 ] Completed replacing temp checkpoint with checkpoint 30.874 ms, 1.49 s total -[ 2023-09-21 09:06:00 ] Completed eval batch 1 to device 499.912 ms, 1.99 s total -[ 2023-09-21 09:06:00 ] Completed eval batch 1 forward 21.123 ms, 2.01 s total -[ 2023-09-21 09:06:00 ] Completed eval batch 1 recons_loss 0.327 ms, 2.01 s total -[ 2023-09-21 09:06:01 ] Completed eval batch 1 metrics update 283.606 ms, 2.30 s total -Saving checkpoint at epoch 103 val batch 1 -Epoch 103 val loss: 0.0602 -[ 2023-09-21 09:06:01 ] Completed saving temp checkpoint 625.393 ms, 2.92 s total -[ 2023-09-21 09:06:01 ] Completed replacing temp checkpoint with checkpoint 28.276 ms, 2.95 s total -[ 2023-09-21 09:06:01 ] Completed evaluating generator for epoch 103 59.947 ms, 3.01 s total - - -EPOCH :: 104 - - -[ 2023-09-21 09:06:01 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 104 from training step 0 - -[ 2023-09-21 09:06:04 ] Completed train batch 0 to device 2,968.447 ms, 2.97 s total -[ 2023-09-21 09:06:04 ] Completed train batch 0 generator forward 27.982 ms, 3.00 s total -[ 2023-09-21 09:06:04 ] Completed train batch 0 generator loss: 1.256 64.689 ms, 3.06 s total -[ 2023-09-21 09:06:05 ] Completed train batch 0 generator backward 603.054 ms, 3.66 s total -[ 2023-09-21 09:06:05 ] Completed train batch 0 discriminator loss 0.098 58.169 ms, 3.72 s total -[ 2023-09-21 09:06:05 ] Completed train batch 0 discriminator backward 86.747 ms, 3.81 s total -Epoch [104] Step [0/2], gen_loss: 0.427, disc_loss: 0.032 -[ 2023-09-21 09:06:05 ] Completed train batch 0 metrics update 5.896 ms, 3.81 s total -Saving checkpoint at epoch 104 train batch 0 -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 09:16:53 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 09:16:53 ] Completed importing Timer 0.027 ms, 0.00 s total -[ 2023-09-21 09:16:58 ] Completed importing everything else 4,703.070 ms, 4.70 s total -| distributed init (rank 4): env:// -| distributed init (rank 3): env:// -| distributed init (rank 5): env:// -| distributed init (rank 1): env:// -| distributed init (rank 0): env:// -| distributed init (rank 2): env:// -[ 2023-09-21 09:17:05 ] Completed preliminaries 7,577.703 ms, 12.28 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 09:17:05 ] Completed build datasets 10.365 ms, 12.29 s total -[ 2023-09-21 09:17:05 ] Completed build samplers 0.046 ms, 12.29 s total -[ 2023-09-21 09:17:05 ] Completed build dataloaders 0.130 ms, 12.29 s total -[ 2023-09-21 09:17:05 ] Completed generator to device 92.476 ms, 12.38 s total -[ 2023-09-21 09:17:05 ] Completed discriminator to device 7.022 ms, 12.39 s total -[ 2023-09-21 09:17:06 ] Completed loss functions 389.794 ms, 12.78 s total -[ 2023-09-21 09:17:06 ] Completed models prepped for distribution 81.892 ms, 12.86 s total -[ 2023-09-21 09:17:06 ] Completed optimizers 0.657 ms, 12.86 s total -[ 2023-09-21 09:17:06 ] Completed grad scalers 0.024 ms, 12.86 s total -[ 2023-09-21 09:17:06 ] Completed checkpoint retrieval 489.597 ms, 13.35 s total - - -EPOCH :: 103 - - -[ 2023-09-21 09:17:06 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 103 from training step 2 - -Epoch [103] :: gen_loss: 0.417, disc_loss: 0.024 -[ 2023-09-21 09:17:07 ] Completed training generator for epoch 103 133.774 ms, 0.13 s total -[ 2023-09-21 09:17:07 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 103 from eval step 2 - -[ 2023-09-21 09:17:07 ] Completed evaluating generator for epoch 103 131.391 ms, 0.13 s total - - -EPOCH :: 104 - - -[ 2023-09-21 09:17:07 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 104 from training step 0 - -[ 2023-09-21 09:17:10 ] Completed train batch 0 to device 3,491.082 ms, 3.49 s total -[ 2023-09-21 09:17:12 ] Completed train batch 0 generator forward 1,432.966 ms, 4.92 s total -[ 2023-09-21 09:17:12 ] Completed train batch 0 generator loss: 1.272 102.698 ms, 5.03 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 09:17:13 ] Completed train batch 0 generator backward 1,136.538 ms, 6.16 s total -[ 2023-09-21 09:17:13 ] Completed train batch 0 discriminator loss 0.095 56.365 ms, 6.22 s total -[ 2023-09-21 09:17:13 ] Completed train batch 0 discriminator backward 94.419 ms, 6.31 s total -Epoch [104] Step [0/2], gen_loss: 0.432, disc_loss: 0.032 -[ 2023-09-21 09:17:13 ] Completed train batch 0 metrics update 5.468 ms, 6.32 s total -Saving checkpoint at epoch 104 train batch 0 -[ 2023-09-21 09:17:13 ] Completed saving temp checkpoint 437.389 ms, 6.76 s total -[ 2023-09-21 09:17:14 ] Completed replacing temp checkpoint with checkpoint 134.987 ms, 6.89 s total -[ 2023-09-21 09:17:14 ] Completed train batch 1 to device 16.722 ms, 6.91 s total -[ 2023-09-21 09:17:14 ] Completed train batch 1 generator forward 23.071 ms, 6.93 s total -[ 2023-09-21 09:17:14 ] Completed train batch 1 generator loss: 1.265 64.497 ms, 7.00 s total -[ 2023-09-21 09:17:15 ] Completed train batch 1 generator backward 1,395.457 ms, 8.39 s total -[ 2023-09-21 09:17:15 ] Completed train batch 1 discriminator loss 0.107 74.888 ms, 8.47 s total -[ 2023-09-21 09:17:15 ] Completed train batch 1 discriminator backward 55.972 ms, 8.52 s total -Epoch [104] Step [1/2], gen_loss: 0.445, disc_loss: 0.033 -[ 2023-09-21 09:17:15 ] Completed train batch 1 metrics update 0.759 ms, 8.52 s total -Saving checkpoint at epoch 104 train batch 1 -[ 2023-09-21 09:17:16 ] Completed saving temp checkpoint 427.909 ms, 8.95 s total -[ 2023-09-21 09:17:16 ] Completed replacing temp checkpoint with checkpoint 17.704 ms, 8.97 s total -Epoch [104] :: gen_loss: 0.439, disc_loss: 0.032 -[ 2023-09-21 09:17:16 ] Completed training generator for epoch 104 46.471 ms, 9.02 s total -[ 2023-09-21 09:17:16 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 104 from eval step 0 - -[ 2023-09-21 09:17:17 ] Completed eval batch 0 to device 1,126.812 ms, 1.13 s total -[ 2023-09-21 09:17:17 ] Completed eval batch 0 forward 25.378 ms, 1.15 s total -[ 2023-09-21 09:17:17 ] Completed eval batch 0 recons_loss 0.313 ms, 1.15 s total -[ 2023-09-21 09:17:17 ] Completed eval batch 0 metrics update 0.570 ms, 1.15 s total -Saving checkpoint at epoch 104 val batch 0 -[ 2023-09-21 09:17:17 ] Completed saving temp checkpoint 427.872 ms, 1.58 s total -[ 2023-09-21 09:17:17 ] Completed replacing temp checkpoint with checkpoint 19.680 ms, 1.60 s total -[ 2023-09-21 09:17:18 ] Completed eval batch 1 to device 415.800 ms, 2.02 s total -[ 2023-09-21 09:17:18 ] Completed eval batch 1 forward 39.407 ms, 2.06 s total -[ 2023-09-21 09:17:18 ] Completed eval batch 1 recons_loss 0.350 ms, 2.06 s total -[ 2023-09-21 09:17:18 ] Completed eval batch 1 metrics update 130.654 ms, 2.19 s total -Saving checkpoint at epoch 104 val batch 1 -Epoch 104 val loss: 0.0713 -[ 2023-09-21 09:17:18 ] Completed saving temp checkpoint 447.219 ms, 2.63 s total -[ 2023-09-21 09:17:18 ] Completed replacing temp checkpoint with checkpoint 17.953 ms, 2.65 s total -[ 2023-09-21 09:17:18 ] Completed evaluating generator for epoch 104 71.275 ms, 2.72 s total - - -EPOCH :: 105 - - -[ 2023-09-21 09:17:18 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 105 from training step 0 - -[ 2023-09-21 09:17:21 ] Completed train batch 0 to device 2,925.706 ms, 2.93 s total -[ 2023-09-21 09:17:21 ] Completed train batch 0 generator forward 25.216 ms, 2.95 s total -[ 2023-09-21 09:17:21 ] Completed train batch 0 generator loss: 1.252 64.793 ms, 3.02 s total -[ 2023-09-21 09:17:22 ] Completed train batch 0 generator backward 693.121 ms, 3.71 s total -[ 2023-09-21 09:17:22 ] Completed train batch 0 discriminator loss 0.074 56.663 ms, 3.77 s total -[ 2023-09-21 09:17:22 ] Completed train batch 0 discriminator backward 102.815 ms, 3.87 s total -Epoch [105] Step [0/2], gen_loss: 0.429, disc_loss: 0.024 -[ 2023-09-21 09:17:22 ] Completed train batch 0 metrics update 5.957 ms, 3.87 s total -Saving checkpoint at epoch 105 train batch 0 -[ 2023-09-21 09:17:23 ] Completed saving temp checkpoint 411.038 ms, 4.29 s total -[ 2023-09-21 09:17:23 ] Completed replacing temp checkpoint with checkpoint 27.329 ms, 4.31 s total -[ 2023-09-21 09:17:24 ] Completed train batch 1 to device 1,210.763 ms, 5.52 s total -[ 2023-09-21 09:17:24 ] Completed train batch 1 generator forward 40.550 ms, 5.56 s total -[ 2023-09-21 09:17:24 ] Completed train batch 1 generator loss: 1.351 61.963 ms, 5.63 s total -[ 2023-09-21 09:17:26 ] Completed train batch 1 generator backward 1,573.357 ms, 7.20 s total -[ 2023-09-21 09:17:26 ] Completed train batch 1 discriminator loss 0.067 55.998 ms, 7.26 s total -[ 2023-09-21 09:17:26 ] Completed train batch 1 discriminator backward 68.353 ms, 7.32 s total -Epoch [105] Step [1/2], gen_loss: 0.440, disc_loss: 0.022 -[ 2023-09-21 09:17:26 ] Completed train batch 1 metrics update 5.151 ms, 7.33 s total -Saving checkpoint at epoch 105 train batch 1 -[ 2023-09-21 09:17:26 ] Completed saving temp checkpoint 447.619 ms, 7.78 s total -[ 2023-09-21 09:17:26 ] Completed replacing temp checkpoint with checkpoint 21.579 ms, 7.80 s total -Epoch [105] :: gen_loss: 0.435, disc_loss: 0.023 -[ 2023-09-21 09:17:26 ] Completed training generator for epoch 105 56.655 ms, 7.85 s total -[ 2023-09-21 09:17:26 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 105 from eval step 0 - -[ 2023-09-21 09:17:27 ] Completed eval batch 0 to device 1,040.964 ms, 1.04 s total -[ 2023-09-21 09:17:27 ] Completed eval batch 0 forward 40.816 ms, 1.08 s total -[ 2023-09-21 09:17:27 ] Completed eval batch 0 recons_loss 0.361 ms, 1.08 s total -[ 2023-09-21 09:17:27 ] Completed eval batch 0 metrics update 0.614 ms, 1.08 s total -Saving checkpoint at epoch 105 val batch 0 -[ 2023-09-21 09:17:28 ] Completed saving temp checkpoint 435.042 ms, 1.52 s total -[ 2023-09-21 09:17:28 ] Completed replacing temp checkpoint with checkpoint 27.927 ms, 1.55 s total -[ 2023-09-21 09:17:28 ] Completed eval batch 1 to device 320.966 ms, 1.87 s total -[ 2023-09-21 09:17:28 ] Completed eval batch 1 forward 39.139 ms, 1.91 s total -[ 2023-09-21 09:17:28 ] Completed eval batch 1 recons_loss 0.365 ms, 1.91 s total -[ 2023-09-21 09:17:29 ] Completed eval batch 1 metrics update 351.710 ms, 2.26 s total -Saving checkpoint at epoch 105 val batch 1 -Epoch 105 val loss: 0.0811 -[ 2023-09-21 09:17:29 ] Completed saving temp checkpoint 470.187 ms, 2.73 s total -[ 2023-09-21 09:17:29 ] Completed replacing temp checkpoint with checkpoint 30.349 ms, 2.76 s total -[ 2023-09-21 09:17:29 ] Completed evaluating generator for epoch 105 55.149 ms, 2.81 s total - - -EPOCH :: 106 - - -[ 2023-09-21 09:17:29 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 106 from training step 0 - -[ 2023-09-21 09:17:32 ] Completed train batch 0 to device 2,825.473 ms, 2.83 s total -[ 2023-09-21 09:17:32 ] Completed train batch 0 generator forward 26.387 ms, 2.85 s total -[ 2023-09-21 09:17:32 ] Completed train batch 0 generator loss: 1.353 66.170 ms, 2.92 s total -[ 2023-09-21 09:17:33 ] Completed train batch 0 generator backward 974.867 ms, 3.89 s total -[ 2023-09-21 09:17:33 ] Completed train batch 0 discriminator loss 0.067 56.847 ms, 3.95 s total -[ 2023-09-21 09:17:33 ] Completed train batch 0 discriminator backward 99.101 ms, 4.05 s total -Epoch [106] Step [0/2], gen_loss: 0.448, disc_loss: 0.022 -[ 2023-09-21 09:17:33 ] Completed train batch 0 metrics update 6.585 ms, 4.06 s total -Saving checkpoint at epoch 106 train batch 0 -[ 2023-09-21 09:17:34 ] Completed saving temp checkpoint 383.525 ms, 4.44 s total -[ 2023-09-21 09:17:34 ] Completed replacing temp checkpoint with checkpoint 26.062 ms, 4.47 s total -[ 2023-09-21 09:17:35 ] Completed train batch 1 to device 1,060.469 ms, 5.53 s total -[ 2023-09-21 09:17:35 ] Completed train batch 1 generator forward 40.411 ms, 5.57 s total -[ 2023-09-21 09:17:35 ] Completed train batch 1 generator loss: 1.328 64.310 ms, 5.63 s total -[ 2023-09-21 09:17:37 ] Completed train batch 1 generator backward 1,980.092 ms, 7.61 s total -[ 2023-09-21 09:17:37 ] Completed train batch 1 discriminator loss 0.063 56.045 ms, 7.67 s total -[ 2023-09-21 09:17:37 ] Completed train batch 1 discriminator backward 87.934 ms, 7.75 s total -Epoch [106] Step [1/2], gen_loss: 0.438, disc_loss: 0.021 -[ 2023-09-21 09:17:37 ] Completed train batch 1 metrics update 5.733 ms, 7.76 s total -Saving checkpoint at epoch 106 train batch 1 -[ 2023-09-21 09:17:37 ] Completed saving temp checkpoint 397.696 ms, 8.16 s total -[ 2023-09-21 09:17:37 ] Completed replacing temp checkpoint with checkpoint 18.307 ms, 8.18 s total -Epoch [106] :: gen_loss: 0.443, disc_loss: 0.022 -[ 2023-09-21 09:17:37 ] Completed training generator for epoch 106 64.150 ms, 8.24 s total -[ 2023-09-21 09:17:37 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 106 from eval step 0 - -[ 2023-09-21 09:17:38 ] Completed eval batch 0 to device 1,080.504 ms, 1.08 s total -[ 2023-09-21 09:17:38 ] Completed eval batch 0 forward 30.507 ms, 1.11 s total -[ 2023-09-21 09:17:38 ] Completed eval batch 0 recons_loss 0.355 ms, 1.11 s total -[ 2023-09-21 09:17:38 ] Completed eval batch 0 metrics update 0.592 ms, 1.11 s total -Saving checkpoint at epoch 106 val batch 0 -[ 2023-09-21 09:17:39 ] Completed saving temp checkpoint 420.331 ms, 1.53 s total -[ 2023-09-21 09:17:39 ] Completed replacing temp checkpoint with checkpoint 28.374 ms, 1.56 s total -[ 2023-09-21 09:17:39 ] Completed eval batch 1 to device 406.590 ms, 1.97 s total -[ 2023-09-21 09:17:39 ] Completed eval batch 1 forward 39.469 ms, 2.01 s total -[ 2023-09-21 09:17:39 ] Completed eval batch 1 recons_loss 0.350 ms, 2.01 s total -[ 2023-09-21 09:17:40 ] Completed eval batch 1 metrics update 237.137 ms, 2.24 s total -Saving checkpoint at epoch 106 val batch 1 -Epoch 106 val loss: 0.0601 -[ 2023-09-21 09:17:40 ] Completed saving temp checkpoint 482.221 ms, 2.73 s total -[ 2023-09-21 09:17:40 ] Completed replacing temp checkpoint with checkpoint 27.473 ms, 2.75 s total -[ 2023-09-21 09:17:40 ] Completed evaluating generator for epoch 106 56.351 ms, 2.81 s total - - -EPOCH :: 107 - - -[ 2023-09-21 09:17:40 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 107 from training step 0 - -[ 2023-09-21 09:17:43 ] Completed train batch 0 to device 2,931.009 ms, 2.93 s total -[ 2023-09-21 09:17:43 ] Completed train batch 0 generator forward 26.728 ms, 2.96 s total -[ 2023-09-21 09:17:43 ] Completed train batch 0 generator loss: 1.273 66.058 ms, 3.02 s total -[ 2023-09-21 09:17:44 ] Completed train batch 0 generator backward 707.040 ms, 3.73 s total -[ 2023-09-21 09:17:44 ] Completed train batch 0 discriminator loss 0.065 57.848 ms, 3.79 s total -[ 2023-09-21 09:17:44 ] Completed train batch 0 discriminator backward 101.132 ms, 3.89 s total -Epoch [107] Step [0/2], gen_loss: 0.425, disc_loss: 0.022 -[ 2023-09-21 09:17:44 ] Completed train batch 0 metrics update 20.268 ms, 3.91 s total -Saving checkpoint at epoch 107 train batch 0 -[ 2023-09-21 09:17:44 ] Completed saving temp checkpoint 435.768 ms, 4.35 s total -[ 2023-09-21 09:17:45 ] Completed replacing temp checkpoint with checkpoint 24.070 ms, 4.37 s total -[ 2023-09-21 09:17:46 ] Completed train batch 1 to device 1,162.653 ms, 5.53 s total -[ 2023-09-21 09:17:46 ] Completed train batch 1 generator forward 40.851 ms, 5.57 s total -[ 2023-09-21 09:17:46 ] Completed train batch 1 generator loss: 1.350 63.652 ms, 5.64 s total -[ 2023-09-21 09:17:47 ] Completed train batch 1 generator backward 1,476.316 ms, 7.11 s total -[ 2023-09-21 09:17:47 ] Completed train batch 1 discriminator loss 0.069 55.974 ms, 7.17 s total -[ 2023-09-21 09:17:47 ] Completed train batch 1 discriminator backward 79.339 ms, 7.25 s total -Epoch [107] Step [1/2], gen_loss: 0.434, disc_loss: 0.023 -[ 2023-09-21 09:17:47 ] Completed train batch 1 metrics update 2.570 ms, 7.25 s total -Saving checkpoint at epoch 107 train batch 1 -[ 2023-09-21 09:17:48 ] Completed saving temp checkpoint 452.107 ms, 7.70 s total -[ 2023-09-21 09:17:48 ] Completed replacing temp checkpoint with checkpoint 20.848 ms, 7.72 s total -Epoch [107] :: gen_loss: 0.430, disc_loss: 0.022 -[ 2023-09-21 09:17:48 ] Completed training generator for epoch 107 58.452 ms, 7.78 s total -[ 2023-09-21 09:17:48 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 107 from eval step 0 - -[ 2023-09-21 09:17:49 ] Completed eval batch 0 to device 1,120.561 ms, 1.12 s total -[ 2023-09-21 09:17:49 ] Completed eval batch 0 forward 28.837 ms, 1.15 s total -[ 2023-09-21 09:17:49 ] Completed eval batch 0 recons_loss 0.276 ms, 1.15 s total -[ 2023-09-21 09:17:49 ] Completed eval batch 0 metrics update 0.730 ms, 1.15 s total -Saving checkpoint at epoch 107 val batch 0 -[ 2023-09-21 09:17:49 ] Completed saving temp checkpoint 424.929 ms, 1.58 s total -[ 2023-09-21 09:17:50 ] Completed replacing temp checkpoint with checkpoint 25.087 ms, 1.60 s total -[ 2023-09-21 09:17:50 ] Completed eval batch 1 to device 462.461 ms, 2.06 s total -[ 2023-09-21 09:17:50 ] Completed eval batch 1 forward 22.138 ms, 2.09 s total -[ 2023-09-21 09:17:50 ] Completed eval batch 1 recons_loss 0.412 ms, 2.09 s total -[ 2023-09-21 09:17:50 ] Completed eval batch 1 metrics update 190.967 ms, 2.28 s total -Saving checkpoint at epoch 107 val batch 1 -Epoch 107 val loss: 0.0555 -[ 2023-09-21 09:17:51 ] Completed saving temp checkpoint 475.392 ms, 2.75 s total -[ 2023-09-21 09:17:51 ] Completed replacing temp checkpoint with checkpoint 29.985 ms, 2.78 s total -[ 2023-09-21 09:17:51 ] Completed evaluating generator for epoch 107 60.941 ms, 2.84 s total - - -EPOCH :: 108 - - -[ 2023-09-21 09:17:51 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 108 from training step 0 - -[ 2023-09-21 09:17:54 ] Completed train batch 0 to device 2,966.358 ms, 2.97 s total -[ 2023-09-21 09:17:54 ] Completed train batch 0 generator forward 42.785 ms, 3.01 s total -[ 2023-09-21 09:17:54 ] Completed train batch 0 generator loss: 1.317 67.067 ms, 3.08 s total -[ 2023-09-21 09:17:55 ] Completed train batch 0 generator backward 669.834 ms, 3.75 s total -[ 2023-09-21 09:17:55 ] Completed train batch 0 discriminator loss 0.062 78.572 ms, 3.82 s total -[ 2023-09-21 09:17:55 ] Completed train batch 0 discriminator backward 69.244 ms, 3.89 s total -Epoch [108] Step [0/2], gen_loss: 0.427, disc_loss: 0.021 -[ 2023-09-21 09:17:55 ] Completed train batch 0 metrics update 7.249 ms, 3.90 s total -Saving checkpoint at epoch 108 train batch 0 -[ 2023-09-21 09:17:55 ] Completed saving temp checkpoint 443.965 ms, 4.35 s total -[ 2023-09-21 09:17:55 ] Completed replacing temp checkpoint with checkpoint 30.461 ms, 4.38 s total -[ 2023-09-21 09:17:57 ] Completed train batch 1 to device 1,416.422 ms, 5.79 s total -[ 2023-09-21 09:17:57 ] Completed train batch 1 generator forward 22.202 ms, 5.81 s total -[ 2023-09-21 09:17:57 ] Completed train batch 1 generator loss: 1.171 63.529 ms, 5.88 s total -[ 2023-09-21 09:17:58 ] Completed train batch 1 generator backward 1,395.786 ms, 7.27 s total -[ 2023-09-21 09:17:58 ] Completed train batch 1 discriminator loss 0.059 75.385 ms, 7.35 s total -[ 2023-09-21 09:17:58 ] Completed train batch 1 discriminator backward 59.752 ms, 7.41 s total -Epoch [108] Step [1/2], gen_loss: 0.424, disc_loss: 0.020 -[ 2023-09-21 09:17:58 ] Completed train batch 1 metrics update 0.764 ms, 7.41 s total -Saving checkpoint at epoch 108 train batch 1 -[ 2023-09-21 09:17:59 ] Completed saving temp checkpoint 462.027 ms, 7.87 s total -[ 2023-09-21 09:17:59 ] Completed replacing temp checkpoint with checkpoint 33.831 ms, 7.91 s total -Epoch [108] :: gen_loss: 0.426, disc_loss: 0.020 -[ 2023-09-21 09:17:59 ] Completed training generator for epoch 108 54.650 ms, 7.96 s total -[ 2023-09-21 09:17:59 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 108 from eval step 0 - -[ 2023-09-21 09:18:00 ] Completed eval batch 0 to device 1,059.971 ms, 1.06 s total -[ 2023-09-21 09:18:00 ] Completed eval batch 0 forward 25.360 ms, 1.09 s total -[ 2023-09-21 09:18:00 ] Completed eval batch 0 recons_loss 0.308 ms, 1.09 s total -[ 2023-09-21 09:18:00 ] Completed eval batch 0 metrics update 0.639 ms, 1.09 s total -Saving checkpoint at epoch 108 val batch 0 -[ 2023-09-21 09:18:00 ] Completed saving temp checkpoint 615.016 ms, 1.70 s total -[ 2023-09-21 09:18:00 ] Completed replacing temp checkpoint with checkpoint 24.679 ms, 1.73 s total -[ 2023-09-21 09:18:01 ] Completed eval batch 1 to device 178.253 ms, 1.90 s total -[ 2023-09-21 09:18:01 ] Completed eval batch 1 forward 39.397 ms, 1.94 s total -[ 2023-09-21 09:18:01 ] Completed eval batch 1 recons_loss 0.366 ms, 1.94 s total -[ 2023-09-21 09:18:01 ] Completed eval batch 1 metrics update 279.392 ms, 2.22 s total -Saving checkpoint at epoch 108 val batch 1 -Epoch 108 val loss: 0.0677 -[ 2023-09-21 09:18:02 ] Completed saving temp checkpoint 665.188 ms, 2.89 s total -[ 2023-09-21 09:18:02 ] Completed replacing temp checkpoint with checkpoint 56.300 ms, 2.94 s total -[ 2023-09-21 09:18:02 ] Completed evaluating generator for epoch 108 59.365 ms, 3.00 s total - - -EPOCH :: 109 - - -[ 2023-09-21 09:18:02 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 109 from training step 0 - -[ 2023-09-21 09:18:05 ] Completed train batch 0 to device 2,793.331 ms, 2.79 s total -[ 2023-09-21 09:18:05 ] Completed train batch 0 generator forward 31.728 ms, 2.83 s total -[ 2023-09-21 09:18:05 ] Completed train batch 0 generator loss: 1.203 66.579 ms, 2.89 s total -[ 2023-09-21 09:18:05 ] Completed train batch 0 generator backward 737.781 ms, 3.63 s total -[ 2023-09-21 09:18:05 ] Completed train batch 0 discriminator loss 0.055 57.727 ms, 3.69 s total -[ 2023-09-21 09:18:05 ] Completed train batch 0 discriminator backward 81.582 ms, 3.77 s total -Epoch [109] Step [0/2], gen_loss: 0.417, disc_loss: 0.019 -[ 2023-09-21 09:18:06 ] Completed train batch 0 metrics update 5.569 ms, 3.77 s total -Saving checkpoint at epoch 109 train batch 0 -[ 2023-09-21 09:18:06 ] Completed saving temp checkpoint 445.431 ms, 4.22 s total -[ 2023-09-21 09:18:06 ] Completed replacing temp checkpoint with checkpoint 29.725 ms, 4.25 s total -[ 2023-09-21 09:18:07 ] Completed train batch 1 to device 1,344.046 ms, 5.59 s total -[ 2023-09-21 09:18:07 ] Completed train batch 1 generator forward 22.302 ms, 5.62 s total -[ 2023-09-21 09:18:07 ] Completed train batch 1 generator loss: 1.284 63.993 ms, 5.68 s total -[ 2023-09-21 09:18:09 ] Completed train batch 1 generator backward 1,572.896 ms, 7.25 s total -[ 2023-09-21 09:18:09 ] Completed train batch 1 discriminator loss 0.060 55.978 ms, 7.31 s total -[ 2023-09-21 09:18:09 ] Completed train batch 1 discriminator backward 68.997 ms, 7.38 s total -Epoch [109] Step [1/2], gen_loss: 0.433, disc_loss: 0.019 -[ 2023-09-21 09:18:09 ] Completed train batch 1 metrics update 1.063 ms, 7.38 s total -Saving checkpoint at epoch 109 train batch 1 -[ 2023-09-21 09:18:10 ] Completed saving temp checkpoint 473.140 ms, 7.85 s total -[ 2023-09-21 09:18:10 ] Completed replacing temp checkpoint with checkpoint 22.917 ms, 7.87 s total -Epoch [109] :: gen_loss: 0.425, disc_loss: 0.019 -[ 2023-09-21 09:18:10 ] Completed training generator for epoch 109 57.350 ms, 7.93 s total -[ 2023-09-21 09:18:10 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 109 from eval step 0 - -[ 2023-09-21 09:18:11 ] Completed eval batch 0 to device 1,120.096 ms, 1.12 s total -[ 2023-09-21 09:18:11 ] Completed eval batch 0 forward 42.745 ms, 1.16 s total -[ 2023-09-21 09:18:11 ] Completed eval batch 0 recons_loss 0.314 ms, 1.16 s total -[ 2023-09-21 09:18:11 ] Completed eval batch 0 metrics update 0.615 ms, 1.16 s total -Saving checkpoint at epoch 109 val batch 0 -[ 2023-09-21 09:18:11 ] Completed saving temp checkpoint 435.866 ms, 1.60 s total -[ 2023-09-21 09:18:11 ] Completed replacing temp checkpoint with checkpoint 26.851 ms, 1.63 s total -[ 2023-09-21 09:18:12 ] Completed eval batch 1 to device 360.741 ms, 1.99 s total -[ 2023-09-21 09:18:12 ] Completed eval batch 1 forward 21.529 ms, 2.01 s total -[ 2023-09-21 09:18:12 ] Completed eval batch 1 recons_loss 0.340 ms, 2.01 s total -[ 2023-09-21 09:18:12 ] Completed eval batch 1 metrics update 168.435 ms, 2.18 s total -Saving checkpoint at epoch 109 val batch 1 -Epoch 109 val loss: 0.0657 -[ 2023-09-21 09:18:12 ] Completed saving temp checkpoint 443.287 ms, 2.62 s total -[ 2023-09-21 09:18:12 ] Completed replacing temp checkpoint with checkpoint 24.200 ms, 2.65 s total -[ 2023-09-21 09:18:12 ] Completed evaluating generator for epoch 109 56.589 ms, 2.70 s total - - -EPOCH :: 110 - - -[ 2023-09-21 09:18:12 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 110 from training step 0 - -[ 2023-09-21 09:18:15 ] Completed train batch 0 to device 3,016.631 ms, 3.02 s total -[ 2023-09-21 09:18:15 ] Completed train batch 0 generator forward 33.138 ms, 3.05 s total -[ 2023-09-21 09:18:15 ] Completed train batch 0 generator loss: 1.266 72.191 ms, 3.12 s total -[ 2023-09-21 09:18:16 ] Completed train batch 0 generator backward 590.816 ms, 3.71 s total -[ 2023-09-21 09:18:16 ] Completed train batch 0 discriminator loss 0.059 57.539 ms, 3.77 s total -[ 2023-09-21 09:18:16 ] Completed train batch 0 discriminator backward 94.061 ms, 3.86 s total -Epoch [110] Step [0/2], gen_loss: 0.421, disc_loss: 0.020 -[ 2023-09-21 09:18:16 ] Completed train batch 0 metrics update 3.869 ms, 3.87 s total -Saving checkpoint at epoch 110 train batch 0 -[ 2023-09-21 09:18:17 ] Completed saving temp checkpoint 456.660 ms, 4.32 s total -[ 2023-09-21 09:18:17 ] Completed replacing temp checkpoint with checkpoint 24.222 ms, 4.35 s total -[ 2023-09-21 09:18:18 ] Completed train batch 1 to device 1,381.370 ms, 5.73 s total -[ 2023-09-21 09:18:18 ] Completed train batch 1 generator forward 22.380 ms, 5.75 s total -[ 2023-09-21 09:18:18 ] Completed train batch 1 generator loss: 1.307 63.467 ms, 5.82 s total -[ 2023-09-21 09:18:19 ] Completed train batch 1 generator backward 1,310.842 ms, 7.13 s total -[ 2023-09-21 09:18:20 ] Completed train batch 1 discriminator loss 0.056 56.198 ms, 7.18 s total -[ 2023-09-21 09:18:20 ] Completed train batch 1 discriminator backward 66.427 ms, 7.25 s total -Epoch [110] Step [1/2], gen_loss: 0.428, disc_loss: 0.019 -[ 2023-09-21 09:18:20 ] Completed train batch 1 metrics update 5.529 ms, 7.26 s total -Saving checkpoint at epoch 110 train batch 1 -[ 2023-09-21 09:18:20 ] Completed saving temp checkpoint 483.696 ms, 7.74 s total -[ 2023-09-21 09:18:20 ] Completed replacing temp checkpoint with checkpoint 31.935 ms, 7.77 s total -Epoch [110] :: gen_loss: 0.425, disc_loss: 0.019 -[ 2023-09-21 09:18:20 ] Completed training generator for epoch 110 57.236 ms, 7.83 s total -[ 2023-09-21 09:18:20 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 110 from eval step 0 - -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 09:29:10 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 09:29:10 ] Completed importing Timer 0.026 ms, 0.00 s total -[ 2023-09-21 09:29:15 ] Completed importing everything else 4,696.381 ms, 4.70 s total -| distributed init (rank 1): env:// -| distributed init (rank 4): env:// -| distributed init (rank 0): env:// -| distributed init (rank 2): env:// -| distributed init (rank 3): env:// -| distributed init (rank 5): env:// -[ 2023-09-21 09:29:23 ] Completed preliminaries 8,133.692 ms, 12.83 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 09:29:23 ] Completed build datasets 11.905 ms, 12.84 s total -[ 2023-09-21 09:29:23 ] Completed build samplers 0.044 ms, 12.84 s total -[ 2023-09-21 09:29:23 ] Completed build dataloaders 0.163 ms, 12.84 s total -[ 2023-09-21 09:29:23 ] Completed generator to device 76.030 ms, 12.92 s total -[ 2023-09-21 09:29:23 ] Completed discriminator to device 7.243 ms, 12.93 s total -[ 2023-09-21 09:29:23 ] Completed loss functions 375.028 ms, 13.30 s total -[ 2023-09-21 09:29:23 ] Completed models prepped for distribution 100.542 ms, 13.40 s total -[ 2023-09-21 09:29:23 ] Completed optimizers 0.689 ms, 13.40 s total -[ 2023-09-21 09:29:23 ] Completed grad scalers 0.024 ms, 13.40 s total -[ 2023-09-21 09:29:24 ] Completed checkpoint retrieval 500.781 ms, 13.90 s total - - -EPOCH :: 110 - - -[ 2023-09-21 09:29:24 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 110 from training step 2 - -Epoch [110] :: gen_loss: 0.425, disc_loss: 0.019 -[ 2023-09-21 09:29:24 ] Completed training generator for epoch 110 116.103 ms, 0.12 s total -[ 2023-09-21 09:29:24 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 110 from eval step 0 - -[ 2023-09-21 09:29:25 ] Completed eval batch 0 to device 1,159.115 ms, 1.16 s total -[ 2023-09-21 09:29:27 ] Completed eval batch 0 forward 1,478.355 ms, 2.64 s total -[ 2023-09-21 09:29:27 ] Completed eval batch 0 recons_loss 0.559 ms, 2.64 s total -[ 2023-09-21 09:29:28 ] Completed eval batch 0 metrics update 995.067 ms, 3.63 s total -Saving checkpoint at epoch 110 val batch 0 -[ 2023-09-21 09:29:28 ] Completed saving temp checkpoint 311.898 ms, 3.94 s total -[ 2023-09-21 09:29:28 ] Completed replacing temp checkpoint with checkpoint 135.004 ms, 4.08 s total -[ 2023-09-21 09:29:28 ] Completed eval batch 1 to device 16.147 ms, 4.10 s total -[ 2023-09-21 09:29:28 ] Completed eval batch 1 forward 27.049 ms, 4.12 s total -[ 2023-09-21 09:29:28 ] Completed eval batch 1 recons_loss 0.403 ms, 4.12 s total -[ 2023-09-21 09:29:28 ] Completed eval batch 1 metrics update 0.334 ms, 4.12 s total -Saving checkpoint at epoch 110 val batch 1 -Epoch 110 val loss: 0.0549 -[ 2023-09-21 09:29:28 ] Completed saving temp checkpoint 409.982 ms, 4.53 s total -[ 2023-09-21 09:29:28 ] Completed replacing temp checkpoint with checkpoint 26.415 ms, 4.56 s total -[ 2023-09-21 09:29:29 ] Completed evaluating generator for epoch 110 37.694 ms, 4.60 s total - - -EPOCH :: 111 - - -[ 2023-09-21 09:29:29 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 111 from training step 0 - -[ 2023-09-21 09:29:32 ] Completed train batch 0 to device 3,067.241 ms, 3.07 s total -[ 2023-09-21 09:29:32 ] Completed train batch 0 generator forward 43.509 ms, 3.11 s total -[ 2023-09-21 09:29:32 ] Completed train batch 0 generator loss: 1.311 80.400 ms, 3.19 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 09:29:33 ] Completed train batch 0 generator backward 827.987 ms, 4.02 s total -[ 2023-09-21 09:29:33 ] Completed train batch 0 discriminator loss 0.054 58.016 ms, 4.08 s total -[ 2023-09-21 09:29:33 ] Completed train batch 0 discriminator backward 95.212 ms, 4.17 s total -Epoch [111] Step [0/2], gen_loss: 0.435, disc_loss: 0.018 -[ 2023-09-21 09:29:33 ] Completed train batch 0 metrics update 6.253 ms, 4.18 s total -Saving checkpoint at epoch 111 train batch 0 -[ 2023-09-21 09:29:33 ] Completed saving temp checkpoint 404.708 ms, 4.58 s total -[ 2023-09-21 09:29:33 ] Completed replacing temp checkpoint with checkpoint 30.282 ms, 4.61 s total -[ 2023-09-21 09:29:34 ] Completed train batch 1 to device 1,298.923 ms, 5.91 s total -[ 2023-09-21 09:29:34 ] Completed train batch 1 generator forward 23.327 ms, 5.94 s total -[ 2023-09-21 09:29:35 ] Completed train batch 1 generator loss: 1.289 63.757 ms, 6.00 s total -[ 2023-09-21 09:29:36 ] Completed train batch 1 generator backward 1,454.202 ms, 7.45 s total -[ 2023-09-21 09:29:36 ] Completed train batch 1 discriminator loss 0.051 56.641 ms, 7.51 s total -[ 2023-09-21 09:29:36 ] Completed train batch 1 discriminator backward 76.873 ms, 7.59 s total -Epoch [111] Step [1/2], gen_loss: 0.437, disc_loss: 0.017 -[ 2023-09-21 09:29:36 ] Completed train batch 1 metrics update 5.185 ms, 7.59 s total -Saving checkpoint at epoch 111 train batch 1 -[ 2023-09-21 09:29:37 ] Completed saving temp checkpoint 477.145 ms, 8.07 s total -[ 2023-09-21 09:29:37 ] Completed replacing temp checkpoint with checkpoint 24.147 ms, 8.09 s total -Epoch [111] :: gen_loss: 0.436, disc_loss: 0.018 -[ 2023-09-21 09:29:37 ] Completed training generator for epoch 111 55.629 ms, 8.15 s total -[ 2023-09-21 09:29:37 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 111 from eval step 0 - -[ 2023-09-21 09:29:38 ] Completed eval batch 0 to device 1,143.203 ms, 1.14 s total -[ 2023-09-21 09:29:38 ] Completed eval batch 0 forward 42.052 ms, 1.19 s total -[ 2023-09-21 09:29:38 ] Completed eval batch 0 recons_loss 0.343 ms, 1.19 s total -[ 2023-09-21 09:29:38 ] Completed eval batch 0 metrics update 0.600 ms, 1.19 s total -Saving checkpoint at epoch 111 val batch 0 -[ 2023-09-21 09:29:38 ] Completed saving temp checkpoint 388.731 ms, 1.57 s total -[ 2023-09-21 09:29:38 ] Completed replacing temp checkpoint with checkpoint 27.608 ms, 1.60 s total -[ 2023-09-21 09:29:39 ] Completed eval batch 1 to device 443.528 ms, 2.05 s total -[ 2023-09-21 09:29:39 ] Completed eval batch 1 forward 39.738 ms, 2.09 s total -[ 2023-09-21 09:29:39 ] Completed eval batch 1 recons_loss 0.557 ms, 2.09 s total -[ 2023-09-21 09:29:39 ] Completed eval batch 1 metrics update 231.386 ms, 2.32 s total -Saving checkpoint at epoch 111 val batch 1 -Epoch 111 val loss: 0.0766 -[ 2023-09-21 09:29:39 ] Completed saving temp checkpoint 403.129 ms, 2.72 s total -[ 2023-09-21 09:29:39 ] Completed replacing temp checkpoint with checkpoint 31.373 ms, 2.75 s total -[ 2023-09-21 09:29:39 ] Completed evaluating generator for epoch 111 56.488 ms, 2.81 s total - - -EPOCH :: 112 - - -[ 2023-09-21 09:29:39 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 112 from training step 0 - -[ 2023-09-21 09:29:43 ] Completed train batch 0 to device 3,129.755 ms, 3.13 s total -[ 2023-09-21 09:29:43 ] Completed train batch 0 generator forward 31.001 ms, 3.16 s total -[ 2023-09-21 09:29:43 ] Completed train batch 0 generator loss: 1.342 67.127 ms, 3.23 s total -[ 2023-09-21 09:29:43 ] Completed train batch 0 generator backward 762.577 ms, 3.99 s total -[ 2023-09-21 09:29:44 ] Completed train batch 0 discriminator loss 0.051 75.656 ms, 4.07 s total -[ 2023-09-21 09:29:44 ] Completed train batch 0 discriminator backward 66.036 ms, 4.13 s total -Epoch [112] Step [0/2], gen_loss: 0.436, disc_loss: 0.017 -[ 2023-09-21 09:29:44 ] Completed train batch 0 metrics update 4.724 ms, 4.14 s total -Saving checkpoint at epoch 112 train batch 0 -[ 2023-09-21 09:29:44 ] Completed saving temp checkpoint 432.910 ms, 4.57 s total -[ 2023-09-21 09:29:44 ] Completed replacing temp checkpoint with checkpoint 31.649 ms, 4.60 s total -[ 2023-09-21 09:29:45 ] Completed train batch 1 to device 1,289.529 ms, 5.89 s total -[ 2023-09-21 09:29:45 ] Completed train batch 1 generator forward 22.437 ms, 5.91 s total -[ 2023-09-21 09:29:45 ] Completed train batch 1 generator loss: 1.318 63.691 ms, 5.98 s total -[ 2023-09-21 09:29:47 ] Completed train batch 1 generator backward 1,458.594 ms, 7.44 s total -[ 2023-09-21 09:29:47 ] Completed train batch 1 discriminator loss 0.055 55.882 ms, 7.49 s total -[ 2023-09-21 09:29:47 ] Completed train batch 1 discriminator backward 71.071 ms, 7.56 s total -Epoch [112] Step [1/2], gen_loss: 0.440, disc_loss: 0.018 -[ 2023-09-21 09:29:47 ] Completed train batch 1 metrics update 5.357 ms, 7.57 s total -Saving checkpoint at epoch 112 train batch 1 -[ 2023-09-21 09:29:47 ] Completed saving temp checkpoint 398.932 ms, 7.97 s total -[ 2023-09-21 09:29:47 ] Completed replacing temp checkpoint with checkpoint 25.443 ms, 7.99 s total -Epoch [112] :: gen_loss: 0.438, disc_loss: 0.017 -[ 2023-09-21 09:29:48 ] Completed training generator for epoch 112 53.334 ms, 8.05 s total -[ 2023-09-21 09:29:48 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 112 from eval step 0 - -[ 2023-09-21 09:29:49 ] Completed eval batch 0 to device 1,109.065 ms, 1.11 s total -[ 2023-09-21 09:29:49 ] Completed eval batch 0 forward 42.095 ms, 1.15 s total -[ 2023-09-21 09:29:49 ] Completed eval batch 0 recons_loss 0.296 ms, 1.15 s total -[ 2023-09-21 09:29:49 ] Completed eval batch 0 metrics update 0.593 ms, 1.15 s total -Saving checkpoint at epoch 112 val batch 0 -[ 2023-09-21 09:29:49 ] Completed saving temp checkpoint 449.074 ms, 1.60 s total -[ 2023-09-21 09:29:49 ] Completed replacing temp checkpoint with checkpoint 35.128 ms, 1.64 s total -[ 2023-09-21 09:29:50 ] Completed eval batch 1 to device 405.268 ms, 2.04 s total -[ 2023-09-21 09:29:50 ] Completed eval batch 1 forward 40.583 ms, 2.08 s total -[ 2023-09-21 09:29:50 ] Completed eval batch 1 recons_loss 0.535 ms, 2.08 s total -[ 2023-09-21 09:29:50 ] Completed eval batch 1 metrics update 234.113 ms, 2.32 s total -Saving checkpoint at epoch 112 val batch 1 -Epoch 112 val loss: 0.0691 -[ 2023-09-21 09:29:50 ] Completed saving temp checkpoint 449.731 ms, 2.77 s total -[ 2023-09-21 09:29:50 ] Completed replacing temp checkpoint with checkpoint 17.203 ms, 2.78 s total -[ 2023-09-21 09:29:50 ] Completed evaluating generator for epoch 112 48.701 ms, 2.83 s total - - -EPOCH :: 113 - - -[ 2023-09-21 09:29:50 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 113 from training step 0 - -[ 2023-09-21 09:29:53 ] Completed train batch 0 to device 2,947.953 ms, 2.95 s total -[ 2023-09-21 09:29:53 ] Completed train batch 0 generator forward 41.328 ms, 2.99 s total -[ 2023-09-21 09:29:53 ] Completed train batch 0 generator loss: 1.230 65.831 ms, 3.06 s total -[ 2023-09-21 09:29:54 ] Completed train batch 0 generator backward 841.883 ms, 3.90 s total -[ 2023-09-21 09:29:54 ] Completed train batch 0 discriminator loss 0.052 76.799 ms, 3.97 s total -[ 2023-09-21 09:29:54 ] Completed train batch 0 discriminator backward 73.219 ms, 4.05 s total -Epoch [113] Step [0/2], gen_loss: 0.421, disc_loss: 0.018 -[ 2023-09-21 09:29:54 ] Completed train batch 0 metrics update 0.874 ms, 4.05 s total -Saving checkpoint at epoch 113 train batch 0 -[ 2023-09-21 09:29:55 ] Completed saving temp checkpoint 426.765 ms, 4.47 s total -[ 2023-09-21 09:29:55 ] Completed replacing temp checkpoint with checkpoint 22.149 ms, 4.50 s total -[ 2023-09-21 09:29:56 ] Completed train batch 1 to device 1,386.722 ms, 5.88 s total -[ 2023-09-21 09:29:56 ] Completed train batch 1 generator forward 38.725 ms, 5.92 s total -[ 2023-09-21 09:29:56 ] Completed train batch 1 generator loss: 1.287 64.720 ms, 5.99 s total -[ 2023-09-21 09:29:58 ] Completed train batch 1 generator backward 1,262.005 ms, 7.25 s total -[ 2023-09-21 09:29:58 ] Completed train batch 1 discriminator loss 0.054 56.503 ms, 7.31 s total -[ 2023-09-21 09:29:58 ] Completed train batch 1 discriminator backward 87.895 ms, 7.39 s total -Epoch [113] Step [1/2], gen_loss: 0.423, disc_loss: 0.019 -[ 2023-09-21 09:29:58 ] Completed train batch 1 metrics update 5.590 ms, 7.40 s total -Saving checkpoint at epoch 113 train batch 1 -[ 2023-09-21 09:29:58 ] Completed saving temp checkpoint 397.665 ms, 7.80 s total -[ 2023-09-21 09:29:58 ] Completed replacing temp checkpoint with checkpoint 15.114 ms, 7.81 s total -Epoch [113] :: gen_loss: 0.422, disc_loss: 0.018 -[ 2023-09-21 09:29:58 ] Completed training generator for epoch 113 61.667 ms, 7.87 s total -[ 2023-09-21 09:29:58 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 113 from eval step 0 - -[ 2023-09-21 09:29:59 ] Completed eval batch 0 to device 1,142.860 ms, 1.14 s total -[ 2023-09-21 09:29:59 ] Completed eval batch 0 forward 24.629 ms, 1.17 s total -[ 2023-09-21 09:29:59 ] Completed eval batch 0 recons_loss 0.304 ms, 1.17 s total -[ 2023-09-21 09:29:59 ] Completed eval batch 0 metrics update 0.561 ms, 1.17 s total -Saving checkpoint at epoch 113 val batch 0 -[ 2023-09-21 09:30:00 ] Completed saving temp checkpoint 331.855 ms, 1.50 s total -[ 2023-09-21 09:30:00 ] Completed replacing temp checkpoint with checkpoint 20.241 ms, 1.52 s total -[ 2023-09-21 09:30:00 ] Completed eval batch 1 to device 553.493 ms, 2.07 s total -[ 2023-09-21 09:30:00 ] Completed eval batch 1 forward 21.471 ms, 2.10 s total -[ 2023-09-21 09:30:00 ] Completed eval batch 1 recons_loss 0.359 ms, 2.10 s total -[ 2023-09-21 09:30:01 ] Completed eval batch 1 metrics update 206.059 ms, 2.30 s total -Saving checkpoint at epoch 113 val batch 1 -Epoch 113 val loss: 0.0568 -[ 2023-09-21 09:30:01 ] Completed saving temp checkpoint 510.531 ms, 2.81 s total -[ 2023-09-21 09:30:01 ] Completed replacing temp checkpoint with checkpoint 24.652 ms, 2.84 s total -[ 2023-09-21 09:30:01 ] Completed evaluating generator for epoch 113 53.658 ms, 2.89 s total - - -EPOCH :: 114 - - -[ 2023-09-21 09:30:01 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 114 from training step 0 - -[ 2023-09-21 09:30:04 ] Completed train batch 0 to device 3,024.969 ms, 3.02 s total -[ 2023-09-21 09:30:04 ] Completed train batch 0 generator forward 26.303 ms, 3.05 s total -[ 2023-09-21 09:30:04 ] Completed train batch 0 generator loss: 1.231 66.007 ms, 3.12 s total -[ 2023-09-21 09:30:05 ] Completed train batch 0 generator backward 658.250 ms, 3.78 s total -[ 2023-09-21 09:30:05 ] Completed train batch 0 discriminator loss 0.053 78.527 ms, 3.85 s total -[ 2023-09-21 09:30:05 ] Completed train batch 0 discriminator backward 65.050 ms, 3.92 s total -Epoch [114] Step [0/2], gen_loss: 0.424, disc_loss: 0.018 -[ 2023-09-21 09:30:05 ] Completed train batch 0 metrics update 5.692 ms, 3.92 s total -Saving checkpoint at epoch 114 train batch 0 -[ 2023-09-21 09:30:05 ] Completed saving temp checkpoint 431.092 ms, 4.36 s total -[ 2023-09-21 09:30:06 ] Completed replacing temp checkpoint with checkpoint 23.732 ms, 4.38 s total -[ 2023-09-21 09:30:07 ] Completed train batch 1 to device 1,471.484 ms, 5.85 s total -[ 2023-09-21 09:30:07 ] Completed train batch 1 generator forward 28.844 ms, 5.88 s total -[ 2023-09-21 09:30:07 ] Completed train batch 1 generator loss: 1.307 65.852 ms, 5.95 s total -[ 2023-09-21 09:30:08 ] Completed train batch 1 generator backward 1,323.229 ms, 7.27 s total -[ 2023-09-21 09:30:08 ] Completed train batch 1 discriminator loss 0.055 55.981 ms, 7.33 s total -[ 2023-09-21 09:30:09 ] Completed train batch 1 discriminator backward 79.360 ms, 7.40 s total -Epoch [114] Step [1/2], gen_loss: 0.440, disc_loss: 0.019 -[ 2023-09-21 09:30:09 ] Completed train batch 1 metrics update 2.170 ms, 7.41 s total -Saving checkpoint at epoch 114 train batch 1 -[ 2023-09-21 09:30:09 ] Completed saving temp checkpoint 478.892 ms, 7.89 s total -[ 2023-09-21 09:30:09 ] Completed replacing temp checkpoint with checkpoint 22.416 ms, 7.91 s total -Epoch [114] :: gen_loss: 0.432, disc_loss: 0.019 -[ 2023-09-21 09:30:09 ] Completed training generator for epoch 114 48.238 ms, 7.96 s total -[ 2023-09-21 09:30:09 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 114 from eval step 0 - -[ 2023-09-21 09:30:10 ] Completed eval batch 0 to device 1,149.237 ms, 1.15 s total -[ 2023-09-21 09:30:10 ] Completed eval batch 0 forward 39.951 ms, 1.19 s total -[ 2023-09-21 09:30:10 ] Completed eval batch 0 recons_loss 0.299 ms, 1.19 s total -[ 2023-09-21 09:30:10 ] Completed eval batch 0 metrics update 0.594 ms, 1.19 s total -Saving checkpoint at epoch 114 val batch 0 -[ 2023-09-21 09:30:11 ] Completed saving temp checkpoint 381.257 ms, 1.57 s total -[ 2023-09-21 09:30:11 ] Completed replacing temp checkpoint with checkpoint 20.949 ms, 1.59 s total -[ 2023-09-21 09:30:11 ] Completed eval batch 1 to device 412.089 ms, 2.00 s total -[ 2023-09-21 09:30:11 ] Completed eval batch 1 forward 38.897 ms, 2.04 s total -[ 2023-09-21 09:30:11 ] Completed eval batch 1 recons_loss 0.329 ms, 2.04 s total -[ 2023-09-21 09:30:11 ] Completed eval batch 1 metrics update 223.763 ms, 2.27 s total -Saving checkpoint at epoch 114 val batch 1 -Epoch 114 val loss: 0.0578 -[ 2023-09-21 09:30:12 ] Completed saving temp checkpoint 402.906 ms, 2.67 s total -[ 2023-09-21 09:30:12 ] Completed replacing temp checkpoint with checkpoint 24.547 ms, 2.69 s total -[ 2023-09-21 09:30:12 ] Completed evaluating generator for epoch 114 47.036 ms, 2.74 s total - - -EPOCH :: 115 - - -[ 2023-09-21 09:30:12 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 115 from training step 0 - -[ 2023-09-21 09:30:15 ] Completed train batch 0 to device 3,309.985 ms, 3.31 s total -[ 2023-09-21 09:30:15 ] Completed train batch 0 generator forward 32.214 ms, 3.34 s total -[ 2023-09-21 09:30:15 ] Completed train batch 0 generator loss: 1.253 66.632 ms, 3.41 s total -[ 2023-09-21 09:30:16 ] Completed train batch 0 generator backward 570.529 ms, 3.98 s total -[ 2023-09-21 09:30:16 ] Completed train batch 0 discriminator loss 0.061 56.856 ms, 4.04 s total -[ 2023-09-21 09:30:16 ] Completed train batch 0 discriminator backward 80.444 ms, 4.12 s total -Epoch [115] Step [0/2], gen_loss: 0.426, disc_loss: 0.020 -[ 2023-09-21 09:30:16 ] Completed train batch 0 metrics update 24.104 ms, 4.14 s total -Saving checkpoint at epoch 115 train batch 0 -[ 2023-09-21 09:30:16 ] Completed saving temp checkpoint 445.688 ms, 4.59 s total -[ 2023-09-21 09:30:16 ] Completed replacing temp checkpoint with checkpoint 31.571 ms, 4.62 s total -[ 2023-09-21 09:30:18 ] Completed train batch 1 to device 1,565.106 ms, 6.18 s total -[ 2023-09-21 09:30:18 ] Completed train batch 1 generator forward 42.553 ms, 6.23 s total -[ 2023-09-21 09:30:18 ] Completed train batch 1 generator loss: 1.252 63.711 ms, 6.29 s total -[ 2023-09-21 09:30:19 ] Completed train batch 1 generator backward 1,326.217 ms, 7.62 s total -[ 2023-09-21 09:30:19 ] Completed train batch 1 discriminator loss 0.057 56.108 ms, 7.67 s total -[ 2023-09-21 09:30:20 ] Completed train batch 1 discriminator backward 69.824 ms, 7.74 s total -Epoch [115] Step [1/2], gen_loss: 0.427, disc_loss: 0.020 -[ 2023-09-21 09:30:20 ] Completed train batch 1 metrics update 1.609 ms, 7.74 s total -Saving checkpoint at epoch 115 train batch 1 -[ 2023-09-21 09:30:20 ] Completed saving temp checkpoint 462.349 ms, 8.21 s total -[ 2023-09-21 09:30:20 ] Completed replacing temp checkpoint with checkpoint 32.442 ms, 8.24 s total -Epoch [115] :: gen_loss: 0.427, disc_loss: 0.020 -[ 2023-09-21 09:30:20 ] Completed training generator for epoch 115 52.448 ms, 8.29 s total -[ 2023-09-21 09:30:20 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 115 from eval step 0 - -[ 2023-09-21 09:30:21 ] Completed eval batch 0 to device 1,078.928 ms, 1.08 s total -[ 2023-09-21 09:30:21 ] Completed eval batch 0 forward 42.658 ms, 1.12 s total -[ 2023-09-21 09:30:21 ] Completed eval batch 0 recons_loss 0.305 ms, 1.12 s total -[ 2023-09-21 09:30:21 ] Completed eval batch 0 metrics update 0.593 ms, 1.12 s total -Saving checkpoint at epoch 115 val batch 0 -[ 2023-09-21 09:30:22 ] Completed saving temp checkpoint 430.575 ms, 1.55 s total -[ 2023-09-21 09:30:22 ] Completed replacing temp checkpoint with checkpoint 26.472 ms, 1.58 s total -[ 2023-09-21 09:30:22 ] Completed eval batch 1 to device 349.128 ms, 1.93 s total -[ 2023-09-21 09:30:22 ] Completed eval batch 1 forward 39.783 ms, 1.97 s total -[ 2023-09-21 09:30:22 ] Completed eval batch 1 recons_loss 0.575 ms, 1.97 s total -[ 2023-09-21 09:30:22 ] Completed eval batch 1 metrics update 359.061 ms, 2.33 s total -Saving checkpoint at epoch 115 val batch 1 -Epoch 115 val loss: 0.0636 -[ 2023-09-21 09:30:23 ] Completed saving temp checkpoint 440.609 ms, 2.77 s total -[ 2023-09-21 09:30:23 ] Completed replacing temp checkpoint with checkpoint 30.283 ms, 2.80 s total -[ 2023-09-21 09:30:23 ] Completed evaluating generator for epoch 115 50.499 ms, 2.85 s total - - -EPOCH :: 116 - - -[ 2023-09-21 09:30:23 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 116 from training step 0 - -[ 2023-09-21 09:30:26 ] Completed train batch 0 to device 3,105.298 ms, 3.11 s total -[ 2023-09-21 09:30:26 ] Completed train batch 0 generator forward 38.900 ms, 3.14 s total -[ 2023-09-21 09:30:26 ] Completed train batch 0 generator loss: 1.263 65.063 ms, 3.21 s total -[ 2023-09-21 09:30:27 ] Completed train batch 0 generator backward 747.558 ms, 3.96 s total -[ 2023-09-21 09:30:27 ] Completed train batch 0 discriminator loss 0.065 55.780 ms, 4.01 s total -[ 2023-09-21 09:30:27 ] Completed train batch 0 discriminator backward 107.789 ms, 4.12 s total -Epoch [116] Step [0/2], gen_loss: 0.428, disc_loss: 0.020 -[ 2023-09-21 09:30:27 ] Completed train batch 0 metrics update 6.341 ms, 4.13 s total -Saving checkpoint at epoch 116 train batch 0 -[ 2023-09-21 09:30:28 ] Completed saving temp checkpoint 437.934 ms, 4.56 s total -[ 2023-09-21 09:30:28 ] Completed replacing temp checkpoint with checkpoint 22.109 ms, 4.59 s total -[ 2023-09-21 09:30:29 ] Completed train batch 1 to device 1,332.801 ms, 5.92 s total -[ 2023-09-21 09:30:29 ] Completed train batch 1 generator forward 39.157 ms, 5.96 s total -[ 2023-09-21 09:30:29 ] Completed train batch 1 generator loss: 1.279 63.477 ms, 6.02 s total -[ 2023-09-21 09:30:31 ] Completed train batch 1 generator backward 1,540.720 ms, 7.56 s total -[ 2023-09-21 09:30:31 ] Completed train batch 1 discriminator loss 0.058 55.708 ms, 7.62 s total -[ 2023-09-21 09:30:31 ] Completed train batch 1 discriminator backward 90.337 ms, 7.71 s total -Epoch [116] Step [1/2], gen_loss: 0.434, disc_loss: 0.019 -[ 2023-09-21 09:30:31 ] Completed train batch 1 metrics update 5.925 ms, 7.71 s total -Saving checkpoint at epoch 116 train batch 1 -[ 2023-09-21 09:30:31 ] Completed saving temp checkpoint 439.608 ms, 8.15 s total -[ 2023-09-21 09:30:31 ] Completed replacing temp checkpoint with checkpoint 29.782 ms, 8.18 s total -Epoch [116] :: gen_loss: 0.431, disc_loss: 0.020 -[ 2023-09-21 09:30:31 ] Completed training generator for epoch 116 58.128 ms, 8.24 s total -[ 2023-09-21 09:30:31 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 116 from eval step 0 - -[ 2023-09-21 09:30:32 ] Completed eval batch 0 to device 1,161.594 ms, 1.16 s total -[ 2023-09-21 09:30:32 ] Completed eval batch 0 forward 26.062 ms, 1.19 s total -[ 2023-09-21 09:30:32 ] Completed eval batch 0 recons_loss 0.294 ms, 1.19 s total -[ 2023-09-21 09:30:32 ] Completed eval batch 0 metrics update 0.571 ms, 1.19 s total -Saving checkpoint at epoch 116 val batch 0 -[ 2023-09-21 09:30:33 ] Completed saving temp checkpoint 410.168 ms, 1.60 s total -[ 2023-09-21 09:30:33 ] Completed replacing temp checkpoint with checkpoint 30.178 ms, 1.63 s total -[ 2023-09-21 09:30:33 ] Completed eval batch 1 to device 427.139 ms, 2.06 s total -[ 2023-09-21 09:30:33 ] Completed eval batch 1 forward 39.811 ms, 2.10 s total -[ 2023-09-21 09:30:33 ] Completed eval batch 1 recons_loss 0.575 ms, 2.10 s total -[ 2023-09-21 09:30:34 ] Completed eval batch 1 metrics update 211.731 ms, 2.31 s total -Saving checkpoint at epoch 116 val batch 1 -Epoch 116 val loss: 0.0621 -[ 2023-09-21 09:30:34 ] Completed saving temp checkpoint 644.811 ms, 2.95 s total -[ 2023-09-21 09:30:34 ] Completed replacing temp checkpoint with checkpoint 34.986 ms, 2.99 s total -[ 2023-09-21 09:30:34 ] Completed evaluating generator for epoch 116 53.591 ms, 3.04 s total - - -EPOCH :: 117 - - -[ 2023-09-21 09:30:34 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 117 from training step 0 - -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 09:41:22 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 09:41:22 ] Completed importing Timer 0.023 ms, 0.00 s total -[ 2023-09-21 09:41:27 ] Completed importing everything else 4,728.866 ms, 4.73 s total -| distributed init (rank 1): env:// -| distributed init (rank 3): env:// -| distributed init (rank 5): env:// -| distributed init (rank 4): env:// -| distributed init (rank 0): env:// -| distributed init (rank 2): env:// -[ 2023-09-21 09:41:35 ] Completed preliminaries 7,930.269 ms, 12.66 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 09:41:35 ] Completed build datasets 11.411 ms, 12.67 s total -[ 2023-09-21 09:41:35 ] Completed build samplers 0.047 ms, 12.67 s total -[ 2023-09-21 09:41:35 ] Completed build dataloaders 0.131 ms, 12.67 s total -[ 2023-09-21 09:41:35 ] Completed generator to device 75.664 ms, 12.75 s total -[ 2023-09-21 09:41:35 ] Completed discriminator to device 9.736 ms, 12.76 s total -[ 2023-09-21 09:41:35 ] Completed loss functions 419.560 ms, 13.18 s total -[ 2023-09-21 09:41:35 ] Completed models prepped for distribution 81.504 ms, 13.26 s total -[ 2023-09-21 09:41:35 ] Completed optimizers 0.673 ms, 13.26 s total -[ 2023-09-21 09:41:35 ] Completed grad scalers 0.030 ms, 13.26 s total -[ 2023-09-21 09:41:36 ] Completed checkpoint retrieval 497.827 ms, 13.76 s total - - -EPOCH :: 116 - - -[ 2023-09-21 09:41:36 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 116 from training step 2 - -Epoch [116] :: gen_loss: 0.431, disc_loss: 0.020 -[ 2023-09-21 09:41:36 ] Completed training generator for epoch 116 145.785 ms, 0.15 s total -[ 2023-09-21 09:41:36 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 116 from eval step 2 - -[ 2023-09-21 09:41:36 ] Completed evaluating generator for epoch 116 127.267 ms, 0.13 s total - - -EPOCH :: 117 - - -[ 2023-09-21 09:41:36 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 117 from training step 0 - -[ 2023-09-21 09:41:40 ] Completed train batch 0 to device 3,998.699 ms, 4.00 s total -[ 2023-09-21 09:41:42 ] Completed train batch 0 generator forward 1,611.274 ms, 5.61 s total -[ 2023-09-21 09:41:42 ] Completed train batch 0 generator loss: 1.239 81.843 ms, 5.69 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 09:41:42 ] Completed train batch 0 generator backward 563.097 ms, 6.25 s total -[ 2023-09-21 09:41:43 ] Completed train batch 0 discriminator loss 0.058 57.417 ms, 6.31 s total -[ 2023-09-21 09:41:43 ] Completed train batch 0 discriminator backward 106.414 ms, 6.42 s total -Epoch [117] Step [0/2], gen_loss: 0.418, disc_loss: 0.019 -[ 2023-09-21 09:41:43 ] Completed train batch 0 metrics update 7.296 ms, 6.43 s total -Saving checkpoint at epoch 117 train batch 0 -[ 2023-09-21 09:41:43 ] Completed saving temp checkpoint 441.549 ms, 6.87 s total -[ 2023-09-21 09:41:43 ] Completed replacing temp checkpoint with checkpoint 129.946 ms, 7.00 s total -[ 2023-09-21 09:41:44 ] Completed train batch 1 to device 595.843 ms, 7.59 s total -[ 2023-09-21 09:41:44 ] Completed train batch 1 generator forward 22.147 ms, 7.62 s total -[ 2023-09-21 09:41:44 ] Completed train batch 1 generator loss: 1.238 61.605 ms, 7.68 s total -[ 2023-09-21 09:41:44 ] Completed train batch 1 generator backward 516.666 ms, 8.19 s total -[ 2023-09-21 09:41:44 ] Completed train batch 1 discriminator loss 0.055 56.108 ms, 8.25 s total -[ 2023-09-21 09:41:45 ] Completed train batch 1 discriminator backward 85.853 ms, 8.34 s total -Epoch [117] Step [1/2], gen_loss: 0.411, disc_loss: 0.018 -[ 2023-09-21 09:41:45 ] Completed train batch 1 metrics update 5.933 ms, 8.34 s total -Saving checkpoint at epoch 117 train batch 1 -[ 2023-09-21 09:41:45 ] Completed saving temp checkpoint 415.934 ms, 8.76 s total -[ 2023-09-21 09:41:45 ] Completed replacing temp checkpoint with checkpoint 24.245 ms, 8.78 s total -Epoch [117] :: gen_loss: 0.415, disc_loss: 0.019 -[ 2023-09-21 09:41:45 ] Completed training generator for epoch 117 43.606 ms, 8.83 s total -[ 2023-09-21 09:41:45 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 117 from eval step 0 - -[ 2023-09-21 09:41:46 ] Completed eval batch 0 to device 1,142.096 ms, 1.14 s total -[ 2023-09-21 09:41:46 ] Completed eval batch 0 forward 25.142 ms, 1.17 s total -[ 2023-09-21 09:41:46 ] Completed eval batch 0 recons_loss 0.304 ms, 1.17 s total -[ 2023-09-21 09:41:46 ] Completed eval batch 0 metrics update 0.619 ms, 1.17 s total -Saving checkpoint at epoch 117 val batch 0 -[ 2023-09-21 09:41:47 ] Completed saving temp checkpoint 391.948 ms, 1.56 s total -[ 2023-09-21 09:41:47 ] Completed replacing temp checkpoint with checkpoint 25.718 ms, 1.59 s total -[ 2023-09-21 09:41:47 ] Completed eval batch 1 to device 521.894 ms, 2.11 s total -[ 2023-09-21 09:41:47 ] Completed eval batch 1 forward 39.687 ms, 2.15 s total -[ 2023-09-21 09:41:47 ] Completed eval batch 1 recons_loss 0.573 ms, 2.15 s total -[ 2023-09-21 09:41:47 ] Completed eval batch 1 metrics update 5.144 ms, 2.15 s total -Saving checkpoint at epoch 117 val batch 1 -Epoch 117 val loss: 0.0627 -[ 2023-09-21 09:41:48 ] Completed saving temp checkpoint 473.784 ms, 2.63 s total -[ 2023-09-21 09:41:48 ] Completed replacing temp checkpoint with checkpoint 30.960 ms, 2.66 s total -[ 2023-09-21 09:41:48 ] Completed evaluating generator for epoch 117 49.776 ms, 2.71 s total - - -EPOCH :: 118 - - -[ 2023-09-21 09:41:48 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 118 from training step 0 - -[ 2023-09-21 09:41:51 ] Completed train batch 0 to device 3,150.309 ms, 3.15 s total -[ 2023-09-21 09:41:51 ] Completed train batch 0 generator forward 31.580 ms, 3.18 s total -[ 2023-09-21 09:41:51 ] Completed train batch 0 generator loss: 1.214 66.743 ms, 3.25 s total -[ 2023-09-21 09:41:51 ] Completed train batch 0 generator backward 288.448 ms, 3.54 s total -[ 2023-09-21 09:41:51 ] Completed train batch 0 discriminator loss 0.055 66.255 ms, 3.60 s total -[ 2023-09-21 09:41:51 ] Completed train batch 0 discriminator backward 85.284 ms, 3.69 s total -Epoch [118] Step [0/2], gen_loss: 0.416, disc_loss: 0.018 -[ 2023-09-21 09:41:51 ] Completed train batch 0 metrics update 6.930 ms, 3.70 s total -Saving checkpoint at epoch 118 train batch 0 -[ 2023-09-21 09:41:52 ] Completed saving temp checkpoint 467.222 ms, 4.16 s total -[ 2023-09-21 09:41:52 ] Completed replacing temp checkpoint with checkpoint 23.350 ms, 4.19 s total -[ 2023-09-21 09:41:54 ] Completed train batch 1 to device 1,985.198 ms, 6.17 s total -[ 2023-09-21 09:41:54 ] Completed train batch 1 generator forward 41.170 ms, 6.21 s total -[ 2023-09-21 09:41:54 ] Completed train batch 1 generator loss: 1.237 65.028 ms, 6.28 s total -[ 2023-09-21 09:41:55 ] Completed train batch 1 generator backward 489.596 ms, 6.77 s total -[ 2023-09-21 09:41:55 ] Completed train batch 1 discriminator loss 0.053 55.602 ms, 6.82 s total -[ 2023-09-21 09:41:55 ] Completed train batch 1 discriminator backward 88.643 ms, 6.91 s total -Epoch [118] Step [1/2], gen_loss: 0.415, disc_loss: 0.018 -[ 2023-09-21 09:41:55 ] Completed train batch 1 metrics update 4.263 ms, 6.92 s total -Saving checkpoint at epoch 118 train batch 1 -[ 2023-09-21 09:41:55 ] Completed saving temp checkpoint 452.248 ms, 7.37 s total -[ 2023-09-21 09:41:55 ] Completed replacing temp checkpoint with checkpoint 27.271 ms, 7.40 s total -Epoch [118] :: gen_loss: 0.416, disc_loss: 0.018 -[ 2023-09-21 09:41:55 ] Completed training generator for epoch 118 58.104 ms, 7.45 s total -[ 2023-09-21 09:41:55 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 118 from eval step 0 - -[ 2023-09-21 09:41:56 ] Completed eval batch 0 to device 1,121.893 ms, 1.12 s total -[ 2023-09-21 09:41:56 ] Completed eval batch 0 forward 40.152 ms, 1.16 s total -[ 2023-09-21 09:41:56 ] Completed eval batch 0 recons_loss 0.304 ms, 1.16 s total -[ 2023-09-21 09:41:56 ] Completed eval batch 0 metrics update 0.669 ms, 1.16 s total -Saving checkpoint at epoch 118 val batch 0 -[ 2023-09-21 09:41:57 ] Completed saving temp checkpoint 426.901 ms, 1.59 s total -[ 2023-09-21 09:41:57 ] Completed replacing temp checkpoint with checkpoint 32.480 ms, 1.62 s total -[ 2023-09-21 09:41:57 ] Completed eval batch 1 to device 422.905 ms, 2.05 s total -[ 2023-09-21 09:41:57 ] Completed eval batch 1 forward 38.696 ms, 2.08 s total -[ 2023-09-21 09:41:57 ] Completed eval batch 1 recons_loss 0.355 ms, 2.08 s total -[ 2023-09-21 09:41:57 ] Completed eval batch 1 metrics update 94.259 ms, 2.18 s total -Saving checkpoint at epoch 118 val batch 1 -Epoch 118 val loss: 0.0611 -[ 2023-09-21 09:41:58 ] Completed saving temp checkpoint 462.384 ms, 2.64 s total -[ 2023-09-21 09:41:58 ] Completed replacing temp checkpoint with checkpoint 29.998 ms, 2.67 s total -[ 2023-09-21 09:41:58 ] Completed evaluating generator for epoch 118 54.379 ms, 2.73 s total - - -EPOCH :: 119 - - -[ 2023-09-21 09:41:58 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 119 from training step 0 - -[ 2023-09-21 09:42:01 ] Completed train batch 0 to device 3,110.731 ms, 3.11 s total -[ 2023-09-21 09:42:01 ] Completed train batch 0 generator forward 30.804 ms, 3.14 s total -[ 2023-09-21 09:42:01 ] Completed train batch 0 generator loss: 1.257 67.162 ms, 3.21 s total -[ 2023-09-21 09:42:02 ] Completed train batch 0 generator backward 422.513 ms, 3.63 s total -[ 2023-09-21 09:42:02 ] Completed train batch 0 discriminator loss 0.056 77.229 ms, 3.71 s total -[ 2023-09-21 09:42:02 ] Completed train batch 0 discriminator backward 79.712 ms, 3.79 s total -Epoch [119] Step [0/2], gen_loss: 0.412, disc_loss: 0.018 -[ 2023-09-21 09:42:02 ] Completed train batch 0 metrics update 0.890 ms, 3.79 s total -Saving checkpoint at epoch 119 train batch 0 -[ 2023-09-21 09:42:02 ] Completed saving temp checkpoint 443.666 ms, 4.23 s total -[ 2023-09-21 09:42:02 ] Completed replacing temp checkpoint with checkpoint 27.612 ms, 4.26 s total -[ 2023-09-21 09:42:04 ] Completed train batch 1 to device 1,527.336 ms, 5.79 s total -[ 2023-09-21 09:42:04 ] Completed train batch 1 generator forward 40.981 ms, 5.83 s total -[ 2023-09-21 09:42:04 ] Completed train batch 1 generator loss: 1.242 63.777 ms, 5.89 s total -[ 2023-09-21 09:42:05 ] Completed train batch 1 generator backward 1,083.763 ms, 6.98 s total -[ 2023-09-21 09:42:05 ] Completed train batch 1 discriminator loss 0.056 55.938 ms, 7.03 s total -[ 2023-09-21 09:42:05 ] Completed train batch 1 discriminator backward 68.607 ms, 7.10 s total -Epoch [119] Step [1/2], gen_loss: 0.414, disc_loss: 0.019 -[ 2023-09-21 09:42:05 ] Completed train batch 1 metrics update 2.343 ms, 7.10 s total -Saving checkpoint at epoch 119 train batch 1 -[ 2023-09-21 09:42:06 ] Completed saving temp checkpoint 463.136 ms, 7.57 s total -[ 2023-09-21 09:42:06 ] Completed replacing temp checkpoint with checkpoint 16.297 ms, 7.58 s total -Epoch [119] :: gen_loss: 0.413, disc_loss: 0.018 -[ 2023-09-21 09:42:06 ] Completed training generator for epoch 119 50.442 ms, 7.63 s total -[ 2023-09-21 09:42:06 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 119 from eval step 0 - -[ 2023-09-21 09:42:07 ] Completed eval batch 0 to device 1,046.052 ms, 1.05 s total -[ 2023-09-21 09:42:07 ] Completed eval batch 0 forward 41.174 ms, 1.09 s total -[ 2023-09-21 09:42:07 ] Completed eval batch 0 recons_loss 0.233 ms, 1.09 s total -[ 2023-09-21 09:42:07 ] Completed eval batch 0 metrics update 0.523 ms, 1.09 s total -Saving checkpoint at epoch 119 val batch 0 -[ 2023-09-21 09:42:07 ] Completed saving temp checkpoint 446.825 ms, 1.53 s total -[ 2023-09-21 09:42:07 ] Completed replacing temp checkpoint with checkpoint 25.971 ms, 1.56 s total -[ 2023-09-21 09:42:07 ] Completed eval batch 1 to device 353.976 ms, 1.91 s total -[ 2023-09-21 09:42:08 ] Completed eval batch 1 forward 40.941 ms, 1.96 s total -[ 2023-09-21 09:42:08 ] Completed eval batch 1 recons_loss 0.528 ms, 1.96 s total -[ 2023-09-21 09:42:08 ] Completed eval batch 1 metrics update 181.165 ms, 2.14 s total -Saving checkpoint at epoch 119 val batch 1 -Epoch 119 val loss: 0.0591 -[ 2023-09-21 09:42:08 ] Completed saving temp checkpoint 440.291 ms, 2.58 s total -[ 2023-09-21 09:42:08 ] Completed replacing temp checkpoint with checkpoint 18.109 ms, 2.60 s total -[ 2023-09-21 09:42:08 ] Completed evaluating generator for epoch 119 46.998 ms, 2.64 s total - - -EPOCH :: 120 - - -[ 2023-09-21 09:42:08 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 120 from training step 0 - -[ 2023-09-21 09:42:11 ] Completed train batch 0 to device 3,099.790 ms, 3.10 s total -[ 2023-09-21 09:42:11 ] Completed train batch 0 generator forward 31.159 ms, 3.13 s total -[ 2023-09-21 09:42:11 ] Completed train batch 0 generator loss: 1.203 65.874 ms, 3.20 s total -[ 2023-09-21 09:42:12 ] Completed train batch 0 generator backward 392.938 ms, 3.59 s total -[ 2023-09-21 09:42:12 ] Completed train batch 0 discriminator loss 0.057 56.638 ms, 3.65 s total -[ 2023-09-21 09:42:12 ] Completed train batch 0 discriminator backward 106.567 ms, 3.75 s total -Epoch [120] Step [0/2], gen_loss: 0.414, disc_loss: 0.019 -[ 2023-09-21 09:42:12 ] Completed train batch 0 metrics update 9.789 ms, 3.76 s total -Saving checkpoint at epoch 120 train batch 0 -[ 2023-09-21 09:42:12 ] Completed saving temp checkpoint 450.921 ms, 4.21 s total -[ 2023-09-21 09:42:12 ] Completed replacing temp checkpoint with checkpoint 26.182 ms, 4.24 s total -[ 2023-09-21 09:42:14 ] Completed train batch 1 to device 1,776.811 ms, 6.02 s total -[ 2023-09-21 09:42:14 ] Completed train batch 1 generator forward 27.238 ms, 6.04 s total -[ 2023-09-21 09:42:14 ] Completed train batch 1 generator loss: 1.244 63.167 ms, 6.11 s total -[ 2023-09-21 09:42:15 ] Completed train batch 1 generator backward 882.468 ms, 6.99 s total -[ 2023-09-21 09:42:15 ] Completed train batch 1 discriminator loss 0.057 55.840 ms, 7.05 s total -[ 2023-09-21 09:42:15 ] Completed train batch 1 discriminator backward 90.307 ms, 7.14 s total -Epoch [120] Step [1/2], gen_loss: 0.417, disc_loss: 0.019 -[ 2023-09-21 09:42:15 ] Completed train batch 1 metrics update 6.269 ms, 7.14 s total -Saving checkpoint at epoch 120 train batch 1 -[ 2023-09-21 09:42:16 ] Completed saving temp checkpoint 431.863 ms, 7.57 s total -[ 2023-09-21 09:42:16 ] Completed replacing temp checkpoint with checkpoint 28.227 ms, 7.60 s total -Epoch [120] :: gen_loss: 0.415, disc_loss: 0.019 -[ 2023-09-21 09:42:16 ] Completed training generator for epoch 120 52.915 ms, 7.65 s total -[ 2023-09-21 09:42:16 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 120 from eval step 0 - -[ 2023-09-21 09:42:17 ] Completed eval batch 0 to device 1,102.010 ms, 1.10 s total -[ 2023-09-21 09:42:17 ] Completed eval batch 0 forward 43.501 ms, 1.15 s total -[ 2023-09-21 09:42:17 ] Completed eval batch 0 recons_loss 0.296 ms, 1.15 s total -[ 2023-09-21 09:42:17 ] Completed eval batch 0 metrics update 0.611 ms, 1.15 s total -Saving checkpoint at epoch 120 val batch 0 -[ 2023-09-21 09:42:17 ] Completed saving temp checkpoint 444.201 ms, 1.59 s total -[ 2023-09-21 09:42:17 ] Completed replacing temp checkpoint with checkpoint 32.815 ms, 1.62 s total -[ 2023-09-21 09:42:18 ] Completed eval batch 1 to device 345.716 ms, 1.97 s total -[ 2023-09-21 09:42:18 ] Completed eval batch 1 forward 39.821 ms, 2.01 s total -[ 2023-09-21 09:42:18 ] Completed eval batch 1 recons_loss 0.553 ms, 2.01 s total -[ 2023-09-21 09:42:18 ] Completed eval batch 1 metrics update 190.525 ms, 2.20 s total -Saving checkpoint at epoch 120 val batch 1 -Epoch 120 val loss: 0.0640 -[ 2023-09-21 09:42:19 ] Completed saving temp checkpoint 448.037 ms, 2.65 s total -[ 2023-09-21 09:42:19 ] Completed replacing temp checkpoint with checkpoint 15.329 ms, 2.66 s total -[ 2023-09-21 09:42:19 ] Completed evaluating generator for epoch 120 49.480 ms, 2.71 s total - - -EPOCH :: 121 - - -[ 2023-09-21 09:42:19 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 121 from training step 0 - -[ 2023-09-21 09:42:22 ] Completed train batch 0 to device 3,104.702 ms, 3.10 s total -[ 2023-09-21 09:42:22 ] Completed train batch 0 generator forward 44.872 ms, 3.15 s total -[ 2023-09-21 09:42:22 ] Completed train batch 0 generator loss: 1.279 66.384 ms, 3.22 s total -[ 2023-09-21 09:42:22 ] Completed train batch 0 generator backward 397.272 ms, 3.61 s total -[ 2023-09-21 09:42:22 ] Completed train batch 0 discriminator loss 0.055 55.774 ms, 3.67 s total -[ 2023-09-21 09:42:22 ] Completed train batch 0 discriminator backward 103.723 ms, 3.77 s total -Epoch [121] Step [0/2], gen_loss: 0.416, disc_loss: 0.019 -[ 2023-09-21 09:42:22 ] Completed train batch 0 metrics update 16.299 ms, 3.79 s total -Saving checkpoint at epoch 121 train batch 0 -[ 2023-09-21 09:42:23 ] Completed saving temp checkpoint 475.651 ms, 4.26 s total -[ 2023-09-21 09:42:23 ] Completed replacing temp checkpoint with checkpoint 35.972 ms, 4.30 s total -[ 2023-09-21 09:42:25 ] Completed train batch 1 to device 1,620.723 ms, 5.92 s total -[ 2023-09-21 09:42:25 ] Completed train batch 1 generator forward 22.269 ms, 5.94 s total -[ 2023-09-21 09:42:25 ] Completed train batch 1 generator loss: 1.257 64.389 ms, 6.01 s total -[ 2023-09-21 09:42:26 ] Completed train batch 1 generator backward 1,051.652 ms, 7.06 s total -[ 2023-09-21 09:42:26 ] Completed train batch 1 discriminator loss 0.056 55.965 ms, 7.12 s total -[ 2023-09-21 09:42:26 ] Completed train batch 1 discriminator backward 72.346 ms, 7.19 s total -Epoch [121] Step [1/2], gen_loss: 0.422, disc_loss: 0.018 -[ 2023-09-21 09:42:26 ] Completed train batch 1 metrics update 1.197 ms, 7.19 s total -Saving checkpoint at epoch 121 train batch 1 -[ 2023-09-21 09:42:26 ] Completed saving temp checkpoint 467.262 ms, 7.66 s total -[ 2023-09-21 09:42:26 ] Completed replacing temp checkpoint with checkpoint 22.260 ms, 7.68 s total -Epoch [121] :: gen_loss: 0.419, disc_loss: 0.019 -[ 2023-09-21 09:42:26 ] Completed training generator for epoch 121 55.781 ms, 7.73 s total -[ 2023-09-21 09:42:26 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 121 from eval step 0 - -[ 2023-09-21 09:42:27 ] Completed eval batch 0 to device 1,139.959 ms, 1.14 s total -[ 2023-09-21 09:42:27 ] Completed eval batch 0 forward 28.825 ms, 1.17 s total -[ 2023-09-21 09:42:27 ] Completed eval batch 0 recons_loss 0.316 ms, 1.17 s total -[ 2023-09-21 09:42:27 ] Completed eval batch 0 metrics update 0.647 ms, 1.17 s total -Saving checkpoint at epoch 121 val batch 0 -[ 2023-09-21 09:42:28 ] Completed saving temp checkpoint 442.430 ms, 1.61 s total -[ 2023-09-21 09:42:28 ] Completed replacing temp checkpoint with checkpoint 25.393 ms, 1.64 s total -[ 2023-09-21 09:42:28 ] Completed eval batch 1 to device 394.040 ms, 2.03 s total -[ 2023-09-21 09:42:28 ] Completed eval batch 1 forward 39.568 ms, 2.07 s total -[ 2023-09-21 09:42:28 ] Completed eval batch 1 recons_loss 0.511 ms, 2.07 s total -[ 2023-09-21 09:42:28 ] Completed eval batch 1 metrics update 52.854 ms, 2.12 s total -Saving checkpoint at epoch 121 val batch 1 -Epoch 121 val loss: 0.0662 -[ 2023-09-21 09:42:29 ] Completed saving temp checkpoint 471.384 ms, 2.60 s total -[ 2023-09-21 09:42:29 ] Completed replacing temp checkpoint with checkpoint 22.705 ms, 2.62 s total -[ 2023-09-21 09:42:29 ] Completed evaluating generator for epoch 121 48.796 ms, 2.67 s total - - -EPOCH :: 122 - - -[ 2023-09-21 09:42:29 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 122 from training step 0 - -[ 2023-09-21 09:42:32 ] Completed train batch 0 to device 3,009.339 ms, 3.01 s total -[ 2023-09-21 09:42:32 ] Completed train batch 0 generator forward 35.160 ms, 3.04 s total -[ 2023-09-21 09:42:32 ] Completed train batch 0 generator loss: 1.214 66.214 ms, 3.11 s total -[ 2023-09-21 09:42:33 ] Completed train batch 0 generator backward 515.705 ms, 3.63 s total -[ 2023-09-21 09:42:33 ] Completed train batch 0 discriminator loss 0.056 57.198 ms, 3.68 s total -[ 2023-09-21 09:42:33 ] Completed train batch 0 discriminator backward 94.805 ms, 3.78 s total -Epoch [122] Step [0/2], gen_loss: 0.411, disc_loss: 0.018 -[ 2023-09-21 09:42:33 ] Completed train batch 0 metrics update 14.376 ms, 3.79 s total -Saving checkpoint at epoch 122 train batch 0 -[ 2023-09-21 09:42:33 ] Completed saving temp checkpoint 411.546 ms, 4.20 s total -[ 2023-09-21 09:42:33 ] Completed replacing temp checkpoint with checkpoint 24.044 ms, 4.23 s total -[ 2023-09-21 09:42:35 ] Completed train batch 1 to device 1,576.178 ms, 5.80 s total -[ 2023-09-21 09:42:35 ] Completed train batch 1 generator forward 23.123 ms, 5.83 s total -[ 2023-09-21 09:42:35 ] Completed train batch 1 generator loss: 1.216 64.955 ms, 5.89 s total -[ 2023-09-21 09:42:36 ] Completed train batch 1 generator backward 1,104.728 ms, 7.00 s total -[ 2023-09-21 09:42:36 ] Completed train batch 1 discriminator loss 0.055 56.585 ms, 7.05 s total -[ 2023-09-21 09:42:36 ] Completed train batch 1 discriminator backward 88.883 ms, 7.14 s total -Epoch [122] Step [1/2], gen_loss: 0.421, disc_loss: 0.018 -[ 2023-09-21 09:42:36 ] Completed train batch 1 metrics update 5.167 ms, 7.15 s total -Saving checkpoint at epoch 122 train batch 1 -[ 2023-09-21 09:42:36 ] Completed saving temp checkpoint 356.218 ms, 7.50 s total -[ 2023-09-21 09:42:37 ] Completed replacing temp checkpoint with checkpoint 18.578 ms, 7.52 s total -Epoch [122] :: gen_loss: 0.416, disc_loss: 0.018 -[ 2023-09-21 09:42:37 ] Completed training generator for epoch 122 55.207 ms, 7.58 s total -[ 2023-09-21 09:42:37 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 122 from eval step 0 - -[ 2023-09-21 09:42:38 ] Completed eval batch 0 to device 1,175.167 ms, 1.18 s total -[ 2023-09-21 09:42:38 ] Completed eval batch 0 forward 25.958 ms, 1.20 s total -[ 2023-09-21 09:42:38 ] Completed eval batch 0 recons_loss 0.225 ms, 1.20 s total -[ 2023-09-21 09:42:38 ] Completed eval batch 0 metrics update 0.489 ms, 1.20 s total -Saving checkpoint at epoch 122 val batch 0 -[ 2023-09-21 09:42:38 ] Completed saving temp checkpoint 348.221 ms, 1.55 s total -[ 2023-09-21 09:42:38 ] Completed replacing temp checkpoint with checkpoint 21.901 ms, 1.57 s total -[ 2023-09-21 09:42:39 ] Completed eval batch 1 to device 792.135 ms, 2.36 s total -[ 2023-09-21 09:42:39 ] Completed eval batch 1 forward 21.678 ms, 2.39 s total -[ 2023-09-21 09:42:39 ] Completed eval batch 1 recons_loss 0.371 ms, 2.39 s total -[ 2023-09-21 09:42:39 ] Completed eval batch 1 metrics update 0.340 ms, 2.39 s total -Saving checkpoint at epoch 122 val batch 1 -Epoch 122 val loss: 0.0668 -[ 2023-09-21 09:42:39 ] Completed saving temp checkpoint 354.305 ms, 2.74 s total -[ 2023-09-21 09:42:39 ] Completed replacing temp checkpoint with checkpoint 17.679 ms, 2.76 s total -[ 2023-09-21 09:42:39 ] Completed evaluating generator for epoch 122 53.271 ms, 2.81 s total - - -EPOCH :: 123 - - -[ 2023-09-21 09:42:39 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 123 from training step 0 - -[ 2023-09-21 09:42:42 ] Completed train batch 0 to device 3,105.105 ms, 3.11 s total -[ 2023-09-21 09:42:43 ] Completed train batch 0 generator forward 25.168 ms, 3.13 s total -[ 2023-09-21 09:42:43 ] Completed train batch 0 generator loss: 1.235 64.349 ms, 3.19 s total -[ 2023-09-21 09:42:43 ] Completed train batch 0 generator backward 570.173 ms, 3.76 s total -[ 2023-09-21 09:42:43 ] Completed train batch 0 discriminator loss 0.054 58.643 ms, 3.82 s total -[ 2023-09-21 09:42:43 ] Completed train batch 0 discriminator backward 96.361 ms, 3.92 s total -Epoch [123] Step [0/2], gen_loss: 0.415, disc_loss: 0.018 -[ 2023-09-21 09:42:43 ] Completed train batch 0 metrics update 3.335 ms, 3.92 s total -Saving checkpoint at epoch 123 train batch 0 -[ 2023-09-21 09:42:44 ] Completed saving temp checkpoint 420.114 ms, 4.34 s total -[ 2023-09-21 09:42:44 ] Completed replacing temp checkpoint with checkpoint 33.943 ms, 4.38 s total -[ 2023-09-21 09:42:45 ] Completed train batch 1 to device 1,460.592 ms, 5.84 s total -[ 2023-09-21 09:42:45 ] Completed train batch 1 generator forward 23.228 ms, 5.86 s total -[ 2023-09-21 09:42:45 ] Completed train batch 1 generator loss: 1.273 65.242 ms, 5.93 s total -[ 2023-09-21 09:42:47 ] Completed train batch 1 generator backward 1,222.177 ms, 7.15 s total -[ 2023-09-21 09:42:47 ] Completed train batch 1 discriminator loss 0.056 55.688 ms, 7.20 s total -[ 2023-09-21 09:42:47 ] Completed train batch 1 discriminator backward 88.689 ms, 7.29 s total -Epoch [123] Step [1/2], gen_loss: 0.417, disc_loss: 0.018 -[ 2023-09-21 09:42:47 ] Completed train batch 1 metrics update 1.555 ms, 7.29 s total -Saving checkpoint at epoch 123 train batch 1 -[ 2023-09-21 09:42:47 ] Completed saving temp checkpoint 448.740 ms, 7.74 s total -[ 2023-09-21 09:42:47 ] Completed replacing temp checkpoint with checkpoint 20.248 ms, 7.76 s total -Epoch [123] :: gen_loss: 0.416, disc_loss: 0.018 -[ 2023-09-21 09:42:47 ] Completed training generator for epoch 123 56.021 ms, 7.82 s total -[ 2023-09-21 09:42:47 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 123 from eval step 0 - -[ 2023-09-21 09:42:48 ] Completed eval batch 0 to device 1,131.250 ms, 1.13 s total -[ 2023-09-21 09:42:48 ] Completed eval batch 0 forward 27.024 ms, 1.16 s total -[ 2023-09-21 09:42:48 ] Completed eval batch 0 recons_loss 0.246 ms, 1.16 s total -[ 2023-09-21 09:42:48 ] Completed eval batch 0 metrics update 0.479 ms, 1.16 s total -Saving checkpoint at epoch 123 val batch 0 -[ 2023-09-21 09:42:49 ] Completed saving temp checkpoint 416.929 ms, 1.58 s total -[ 2023-09-21 09:42:49 ] Completed replacing temp checkpoint with checkpoint 27.148 ms, 1.60 s total -[ 2023-09-21 09:42:49 ] Completed eval batch 1 to device 449.488 ms, 2.05 s total -[ 2023-09-21 09:42:49 ] Completed eval batch 1 forward 37.178 ms, 2.09 s total -[ 2023-09-21 09:42:49 ] Completed eval batch 1 recons_loss 0.357 ms, 2.09 s total -[ 2023-09-21 09:42:49 ] Completed eval batch 1 metrics update 97.085 ms, 2.19 s total -Saving checkpoint at epoch 123 val batch 1 -Epoch 123 val loss: 0.0676 -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 09:55:05 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 09:55:05 ] Completed importing Timer 0.023 ms, 0.00 s total -[ 2023-09-21 09:55:10 ] Completed importing everything else 4,731.080 ms, 4.73 s total -| distributed init (rank 4): env:// -| distributed init (rank 5): env:// -| distributed init (rank 1): env:// -| distributed init (rank 0): env:// -| distributed init (rank 3): env:// -| distributed init (rank 2): env:// -[ 2023-09-21 09:55:18 ] Completed preliminaries 7,863.634 ms, 12.59 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 09:55:18 ] Completed build datasets 11.538 ms, 12.61 s total -[ 2023-09-21 09:55:18 ] Completed build samplers 0.045 ms, 12.61 s total -[ 2023-09-21 09:55:18 ] Completed build dataloaders 0.135 ms, 12.61 s total -[ 2023-09-21 09:55:18 ] Completed generator to device 94.229 ms, 12.70 s total -[ 2023-09-21 09:55:18 ] Completed discriminator to device 7.300 ms, 12.71 s total -[ 2023-09-21 09:55:18 ] Completed loss functions 393.820 ms, 13.10 s total -[ 2023-09-21 09:55:18 ] Completed models prepped for distribution 64.785 ms, 13.17 s total -[ 2023-09-21 09:55:18 ] Completed optimizers 0.671 ms, 13.17 s total -[ 2023-09-21 09:55:18 ] Completed grad scalers 0.028 ms, 13.17 s total -[ 2023-09-21 09:55:19 ] Completed checkpoint retrieval 486.603 ms, 13.65 s total - - -EPOCH :: 123 - - -[ 2023-09-21 09:55:19 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 123 from training step 2 - -Epoch [123] :: gen_loss: 0.416, disc_loss: 0.018 -[ 2023-09-21 09:55:19 ] Completed training generator for epoch 123 124.304 ms, 0.12 s total -[ 2023-09-21 09:55:19 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 123 from eval step 1 - -[ 2023-09-21 09:55:20 ] Completed eval batch 1 to device 1,350.797 ms, 1.35 s total -[ 2023-09-21 09:55:22 ] Completed eval batch 1 forward 1,128.760 ms, 2.48 s total -[ 2023-09-21 09:55:22 ] Completed eval batch 1 recons_loss 0.282 ms, 2.48 s total -[ 2023-09-21 09:55:22 ] Completed eval batch 1 metrics update 347.300 ms, 2.83 s total -Saving checkpoint at epoch 123 val batch 1 -Epoch 123 val loss: 0.0662 -[ 2023-09-21 09:55:22 ] Completed saving temp checkpoint 444.682 ms, 3.27 s total -[ 2023-09-21 09:55:22 ] Completed replacing temp checkpoint with checkpoint 138.575 ms, 3.41 s total -[ 2023-09-21 09:55:22 ] Completed evaluating generator for epoch 123 39.736 ms, 3.45 s total - - -EPOCH :: 124 - - -[ 2023-09-21 09:55:22 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 124 from training step 0 - -[ 2023-09-21 09:55:26 ] Completed train batch 0 to device 3,034.987 ms, 3.03 s total -[ 2023-09-21 09:55:26 ] Completed train batch 0 generator forward 43.988 ms, 3.08 s total -[ 2023-09-21 09:55:26 ] Completed train batch 0 generator loss: 1.308 79.120 ms, 3.16 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 09:55:26 ] Completed train batch 0 generator backward 645.645 ms, 3.80 s total -[ 2023-09-21 09:55:26 ] Completed train batch 0 discriminator loss 0.055 57.439 ms, 3.86 s total -[ 2023-09-21 09:55:26 ] Completed train batch 0 discriminator backward 105.543 ms, 3.97 s total -Epoch [124] Step [0/2], gen_loss: 0.416, disc_loss: 0.019 -[ 2023-09-21 09:55:26 ] Completed train batch 0 metrics update 8.310 ms, 3.98 s total -Saving checkpoint at epoch 124 train batch 0 -[ 2023-09-21 09:55:27 ] Completed saving temp checkpoint 445.434 ms, 4.42 s total -[ 2023-09-21 09:55:27 ] Completed replacing temp checkpoint with checkpoint 31.335 ms, 4.45 s total -[ 2023-09-21 09:55:28 ] Completed train batch 1 to device 1,367.709 ms, 5.82 s total -[ 2023-09-21 09:55:28 ] Completed train batch 1 generator forward 40.946 ms, 5.86 s total -[ 2023-09-21 09:55:28 ] Completed train batch 1 generator loss: 1.211 63.407 ms, 5.92 s total -[ 2023-09-21 09:55:30 ] Completed train batch 1 generator backward 1,407.246 ms, 7.33 s total -[ 2023-09-21 09:55:30 ] Completed train batch 1 discriminator loss 0.056 56.424 ms, 7.39 s total -[ 2023-09-21 09:55:30 ] Completed train batch 1 discriminator backward 73.683 ms, 7.46 s total -Epoch [124] Step [1/2], gen_loss: 0.418, disc_loss: 0.019 -[ 2023-09-21 09:55:30 ] Completed train batch 1 metrics update 0.795 ms, 7.46 s total -Saving checkpoint at epoch 124 train batch 1 -[ 2023-09-21 09:55:30 ] Completed saving temp checkpoint 435.254 ms, 7.90 s total -[ 2023-09-21 09:55:30 ] Completed replacing temp checkpoint with checkpoint 27.209 ms, 7.92 s total -Epoch [124] :: gen_loss: 0.417, disc_loss: 0.019 -[ 2023-09-21 09:55:30 ] Completed training generator for epoch 124 63.480 ms, 7.99 s total -[ 2023-09-21 09:55:30 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 124 from eval step 0 - -[ 2023-09-21 09:55:32 ] Completed eval batch 0 to device 1,196.444 ms, 1.20 s total -[ 2023-09-21 09:55:32 ] Completed eval batch 0 forward 25.119 ms, 1.22 s total -[ 2023-09-21 09:55:32 ] Completed eval batch 0 recons_loss 0.322 ms, 1.22 s total -[ 2023-09-21 09:55:32 ] Completed eval batch 0 metrics update 0.668 ms, 1.22 s total -Saving checkpoint at epoch 124 val batch 0 -[ 2023-09-21 09:55:32 ] Completed saving temp checkpoint 398.536 ms, 1.62 s total -[ 2023-09-21 09:55:32 ] Completed replacing temp checkpoint with checkpoint 29.409 ms, 1.65 s total -[ 2023-09-21 09:55:33 ] Completed eval batch 1 to device 441.231 ms, 2.09 s total -[ 2023-09-21 09:55:33 ] Completed eval batch 1 forward 28.997 ms, 2.12 s total -[ 2023-09-21 09:55:33 ] Completed eval batch 1 recons_loss 0.358 ms, 2.12 s total -[ 2023-09-21 09:55:33 ] Completed eval batch 1 metrics update 136.665 ms, 2.26 s total -Saving checkpoint at epoch 124 val batch 1 -Epoch 124 val loss: 0.0642 -[ 2023-09-21 09:55:33 ] Completed saving temp checkpoint 463.719 ms, 2.72 s total -[ 2023-09-21 09:55:33 ] Completed replacing temp checkpoint with checkpoint 22.223 ms, 2.74 s total -[ 2023-09-21 09:55:33 ] Completed evaluating generator for epoch 124 55.286 ms, 2.80 s total - - -EPOCH :: 125 - - -[ 2023-09-21 09:55:33 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 125 from training step 0 - -[ 2023-09-21 09:55:37 ] Completed train batch 0 to device 3,231.130 ms, 3.23 s total -[ 2023-09-21 09:55:37 ] Completed train batch 0 generator forward 27.254 ms, 3.26 s total -[ 2023-09-21 09:55:37 ] Completed train batch 0 generator loss: 1.222 66.817 ms, 3.33 s total -[ 2023-09-21 09:55:37 ] Completed train batch 0 generator backward 316.012 ms, 3.64 s total -[ 2023-09-21 09:55:37 ] Completed train batch 0 discriminator loss 0.053 79.072 ms, 3.72 s total -[ 2023-09-21 09:55:37 ] Completed train batch 0 discriminator backward 67.387 ms, 3.79 s total -Epoch [125] Step [0/2], gen_loss: 0.411, disc_loss: 0.018 -[ 2023-09-21 09:55:37 ] Completed train batch 0 metrics update 1.740 ms, 3.79 s total -Saving checkpoint at epoch 125 train batch 0 -[ 2023-09-21 09:55:37 ] Completed saving temp checkpoint 411.615 ms, 4.20 s total -[ 2023-09-21 09:55:38 ] Completed replacing temp checkpoint with checkpoint 23.300 ms, 4.22 s total -[ 2023-09-21 09:55:40 ] Completed train batch 1 to device 2,007.170 ms, 6.23 s total -[ 2023-09-21 09:55:40 ] Completed train batch 1 generator forward 41.872 ms, 6.27 s total -[ 2023-09-21 09:55:40 ] Completed train batch 1 generator loss: 1.224 64.070 ms, 6.34 s total -[ 2023-09-21 09:55:40 ] Completed train batch 1 generator backward 674.821 ms, 7.01 s total -[ 2023-09-21 09:55:40 ] Completed train batch 1 discriminator loss 0.054 56.633 ms, 7.07 s total -[ 2023-09-21 09:55:40 ] Completed train batch 1 discriminator backward 59.327 ms, 7.13 s total -Epoch [125] Step [1/2], gen_loss: 0.414, disc_loss: 0.018 -[ 2023-09-21 09:55:40 ] Completed train batch 1 metrics update 2.278 ms, 7.13 s total -Saving checkpoint at epoch 125 train batch 1 -[ 2023-09-21 09:55:41 ] Completed saving temp checkpoint 454.221 ms, 7.58 s total -[ 2023-09-21 09:55:41 ] Completed replacing temp checkpoint with checkpoint 20.753 ms, 7.61 s total -Epoch [125] :: gen_loss: 0.413, disc_loss: 0.018 -[ 2023-09-21 09:55:41 ] Completed training generator for epoch 125 49.884 ms, 7.66 s total -[ 2023-09-21 09:55:41 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 125 from eval step 0 - -[ 2023-09-21 09:55:42 ] Completed eval batch 0 to device 1,127.027 ms, 1.13 s total -[ 2023-09-21 09:55:42 ] Completed eval batch 0 forward 36.817 ms, 1.16 s total -[ 2023-09-21 09:55:42 ] Completed eval batch 0 recons_loss 0.308 ms, 1.16 s total -[ 2023-09-21 09:55:42 ] Completed eval batch 0 metrics update 0.570 ms, 1.16 s total -Saving checkpoint at epoch 125 val batch 0 -[ 2023-09-21 09:55:43 ] Completed saving temp checkpoint 416.620 ms, 1.58 s total -[ 2023-09-21 09:55:43 ] Completed replacing temp checkpoint with checkpoint 35.542 ms, 1.62 s total -[ 2023-09-21 09:55:43 ] Completed eval batch 1 to device 478.553 ms, 2.10 s total -[ 2023-09-21 09:55:43 ] Completed eval batch 1 forward 39.170 ms, 2.13 s total -[ 2023-09-21 09:55:43 ] Completed eval batch 1 recons_loss 0.336 ms, 2.13 s total -[ 2023-09-21 09:55:43 ] Completed eval batch 1 metrics update 119.637 ms, 2.25 s total -Saving checkpoint at epoch 125 val batch 1 -Epoch 125 val loss: 0.0647 -[ 2023-09-21 09:55:44 ] Completed saving temp checkpoint 436.897 ms, 2.69 s total -[ 2023-09-21 09:55:44 ] Completed replacing temp checkpoint with checkpoint 23.969 ms, 2.72 s total -[ 2023-09-21 09:55:44 ] Completed evaluating generator for epoch 125 48.144 ms, 2.76 s total - - -EPOCH :: 126 - - -[ 2023-09-21 09:55:44 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 126 from training step 0 - -[ 2023-09-21 09:55:47 ] Completed train batch 0 to device 3,149.913 ms, 3.15 s total -[ 2023-09-21 09:55:47 ] Completed train batch 0 generator forward 44.430 ms, 3.19 s total -[ 2023-09-21 09:55:47 ] Completed train batch 0 generator loss: 1.265 66.538 ms, 3.26 s total -[ 2023-09-21 09:55:48 ] Completed train batch 0 generator backward 605.451 ms, 3.87 s total -[ 2023-09-21 09:55:48 ] Completed train batch 0 discriminator loss 0.054 59.241 ms, 3.93 s total -[ 2023-09-21 09:55:48 ] Completed train batch 0 discriminator backward 93.791 ms, 4.02 s total -Epoch [126] Step [0/2], gen_loss: 0.411, disc_loss: 0.018 -[ 2023-09-21 09:55:48 ] Completed train batch 0 metrics update 9.566 ms, 4.03 s total -Saving checkpoint at epoch 126 train batch 0 -[ 2023-09-21 09:55:48 ] Completed saving temp checkpoint 406.955 ms, 4.44 s total -[ 2023-09-21 09:55:48 ] Completed replacing temp checkpoint with checkpoint 23.871 ms, 4.46 s total -[ 2023-09-21 09:55:50 ] Completed train batch 1 to device 1,482.626 ms, 5.94 s total -[ 2023-09-21 09:55:50 ] Completed train batch 1 generator forward 41.397 ms, 5.98 s total -[ 2023-09-21 09:55:50 ] Completed train batch 1 generator loss: 1.230 64.364 ms, 6.05 s total -[ 2023-09-21 09:55:51 ] Completed train batch 1 generator backward 1,270.647 ms, 7.32 s total -[ 2023-09-21 09:55:51 ] Completed train batch 1 discriminator loss 0.052 56.334 ms, 7.38 s total -[ 2023-09-21 09:55:51 ] Completed train batch 1 discriminator backward 75.701 ms, 7.45 s total -Epoch [126] Step [1/2], gen_loss: 0.418, disc_loss: 0.017 -[ 2023-09-21 09:55:51 ] Completed train batch 1 metrics update 1.591 ms, 7.45 s total -Saving checkpoint at epoch 126 train batch 1 -[ 2023-09-21 09:55:52 ] Completed saving temp checkpoint 466.438 ms, 7.92 s total -[ 2023-09-21 09:55:52 ] Completed replacing temp checkpoint with checkpoint 16.924 ms, 7.94 s total -Epoch [126] :: gen_loss: 0.415, disc_loss: 0.018 -[ 2023-09-21 09:55:52 ] Completed training generator for epoch 126 48.697 ms, 7.98 s total -[ 2023-09-21 09:55:52 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 126 from eval step 0 - -[ 2023-09-21 09:55:53 ] Completed eval batch 0 to device 1,128.666 ms, 1.13 s total -[ 2023-09-21 09:55:53 ] Completed eval batch 0 forward 25.553 ms, 1.15 s total -[ 2023-09-21 09:55:53 ] Completed eval batch 0 recons_loss 0.293 ms, 1.15 s total -[ 2023-09-21 09:55:53 ] Completed eval batch 0 metrics update 0.608 ms, 1.16 s total -Saving checkpoint at epoch 126 val batch 0 -[ 2023-09-21 09:55:53 ] Completed saving temp checkpoint 450.706 ms, 1.61 s total -[ 2023-09-21 09:55:53 ] Completed replacing temp checkpoint with checkpoint 36.316 ms, 1.64 s total -[ 2023-09-21 09:55:54 ] Completed eval batch 1 to device 411.116 ms, 2.05 s total -[ 2023-09-21 09:55:54 ] Completed eval batch 1 forward 22.177 ms, 2.08 s total -[ 2023-09-21 09:55:54 ] Completed eval batch 1 recons_loss 0.352 ms, 2.08 s total -[ 2023-09-21 09:55:54 ] Completed eval batch 1 metrics update 217.471 ms, 2.29 s total -Saving checkpoint at epoch 126 val batch 1 -Epoch 126 val loss: 0.0652 -[ 2023-09-21 09:55:55 ] Completed saving temp checkpoint 534.973 ms, 2.83 s total -[ 2023-09-21 09:55:55 ] Completed replacing temp checkpoint with checkpoint 35.442 ms, 2.86 s total -[ 2023-09-21 09:55:55 ] Completed evaluating generator for epoch 126 57.375 ms, 2.92 s total - - -EPOCH :: 127 - - -[ 2023-09-21 09:55:55 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 127 from training step 0 - -[ 2023-09-21 09:55:58 ] Completed train batch 0 to device 3,197.533 ms, 3.20 s total -[ 2023-09-21 09:55:58 ] Completed train batch 0 generator forward 30.449 ms, 3.23 s total -[ 2023-09-21 09:55:58 ] Completed train batch 0 generator loss: 1.278 66.463 ms, 3.29 s total -[ 2023-09-21 09:55:58 ] Completed train batch 0 generator backward 267.470 ms, 3.56 s total -[ 2023-09-21 09:55:58 ] Completed train batch 0 discriminator loss 0.051 67.319 ms, 3.63 s total -[ 2023-09-21 09:55:58 ] Completed train batch 0 discriminator backward 84.362 ms, 3.71 s total -Epoch [127] Step [0/2], gen_loss: 0.416, disc_loss: 0.017 -[ 2023-09-21 09:55:58 ] Completed train batch 0 metrics update 3.903 ms, 3.72 s total -Saving checkpoint at epoch 127 train batch 0 -[ 2023-09-21 09:55:59 ] Completed saving temp checkpoint 413.844 ms, 4.13 s total -[ 2023-09-21 09:55:59 ] Completed replacing temp checkpoint with checkpoint 31.304 ms, 4.16 s total -[ 2023-09-21 09:56:01 ] Completed train batch 1 to device 1,798.400 ms, 5.96 s total -[ 2023-09-21 09:56:01 ] Completed train batch 1 generator forward 41.579 ms, 6.00 s total -[ 2023-09-21 09:56:01 ] Completed train batch 1 generator loss: 1.301 63.885 ms, 6.07 s total -[ 2023-09-21 09:56:01 ] Completed train batch 1 generator backward 809.457 ms, 6.88 s total -[ 2023-09-21 09:56:02 ] Completed train batch 1 discriminator loss 0.050 56.398 ms, 6.93 s total -[ 2023-09-21 09:56:02 ] Completed train batch 1 discriminator backward 89.802 ms, 7.02 s total -Epoch [127] Step [1/2], gen_loss: 0.423, disc_loss: 0.017 -[ 2023-09-21 09:56:02 ] Completed train batch 1 metrics update 2.067 ms, 7.02 s total -Saving checkpoint at epoch 127 train batch 1 -[ 2023-09-21 09:56:02 ] Completed saving temp checkpoint 425.376 ms, 7.45 s total -[ 2023-09-21 09:56:02 ] Completed replacing temp checkpoint with checkpoint 17.846 ms, 7.47 s total -Epoch [127] :: gen_loss: 0.420, disc_loss: 0.017 -[ 2023-09-21 09:56:02 ] Completed training generator for epoch 127 60.423 ms, 7.53 s total -[ 2023-09-21 09:56:02 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 127 from eval step 0 - -[ 2023-09-21 09:56:03 ] Completed eval batch 0 to device 1,131.378 ms, 1.13 s total -[ 2023-09-21 09:56:03 ] Completed eval batch 0 forward 25.707 ms, 1.16 s total -[ 2023-09-21 09:56:03 ] Completed eval batch 0 recons_loss 0.294 ms, 1.16 s total -[ 2023-09-21 09:56:03 ] Completed eval batch 0 metrics update 0.558 ms, 1.16 s total -Saving checkpoint at epoch 127 val batch 0 -[ 2023-09-21 09:56:04 ] Completed saving temp checkpoint 383.276 ms, 1.54 s total -[ 2023-09-21 09:56:04 ] Completed replacing temp checkpoint with checkpoint 30.145 ms, 1.57 s total -[ 2023-09-21 09:56:04 ] Completed eval batch 1 to device 484.643 ms, 2.06 s total -[ 2023-09-21 09:56:04 ] Completed eval batch 1 forward 21.708 ms, 2.08 s total -[ 2023-09-21 09:56:04 ] Completed eval batch 1 recons_loss 0.345 ms, 2.08 s total -[ 2023-09-21 09:56:04 ] Completed eval batch 1 metrics update 124.556 ms, 2.20 s total -Saving checkpoint at epoch 127 val batch 1 -Epoch 127 val loss: 0.0647 -[ 2023-09-21 09:56:05 ] Completed saving temp checkpoint 407.069 ms, 2.61 s total -[ 2023-09-21 09:56:05 ] Completed replacing temp checkpoint with checkpoint 17.925 ms, 2.63 s total -[ 2023-09-21 09:56:05 ] Completed evaluating generator for epoch 127 48.764 ms, 2.68 s total - - -EPOCH :: 128 - - -[ 2023-09-21 09:56:05 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 128 from training step 0 - -[ 2023-09-21 09:56:08 ] Completed train batch 0 to device 3,096.259 ms, 3.10 s total -[ 2023-09-21 09:56:08 ] Completed train batch 0 generator forward 43.857 ms, 3.14 s total -[ 2023-09-21 09:56:08 ] Completed train batch 0 generator loss: 1.321 66.214 ms, 3.21 s total -[ 2023-09-21 09:56:09 ] Completed train batch 0 generator backward 577.830 ms, 3.78 s total -[ 2023-09-21 09:56:09 ] Completed train batch 0 discriminator loss 0.051 56.866 ms, 3.84 s total -[ 2023-09-21 09:56:09 ] Completed train batch 0 discriminator backward 116.872 ms, 3.96 s total -Epoch [128] Step [0/2], gen_loss: 0.414, disc_loss: 0.017 -[ 2023-09-21 09:56:09 ] Completed train batch 0 metrics update 5.775 ms, 3.96 s total -Saving checkpoint at epoch 128 train batch 0 -[ 2023-09-21 09:56:09 ] Completed saving temp checkpoint 406.076 ms, 4.37 s total -[ 2023-09-21 09:56:09 ] Completed replacing temp checkpoint with checkpoint 20.639 ms, 4.39 s total -[ 2023-09-21 09:56:11 ] Completed train batch 1 to device 1,641.592 ms, 6.03 s total -[ 2023-09-21 09:56:11 ] Completed train batch 1 generator forward 42.914 ms, 6.07 s total -[ 2023-09-21 09:56:11 ] Completed train batch 1 generator loss: 1.322 65.607 ms, 6.14 s total -[ 2023-09-21 09:56:12 ] Completed train batch 1 generator backward 944.152 ms, 7.08 s total -[ 2023-09-21 09:56:12 ] Completed train batch 1 discriminator loss 0.050 57.775 ms, 7.14 s total -[ 2023-09-21 09:56:12 ] Completed train batch 1 discriminator backward 59.706 ms, 7.20 s total -Epoch [128] Step [1/2], gen_loss: 0.423, disc_loss: 0.017 -[ 2023-09-21 09:56:12 ] Completed train batch 1 metrics update 1.580 ms, 7.20 s total -Saving checkpoint at epoch 128 train batch 1 -[ 2023-09-21 09:56:13 ] Completed saving temp checkpoint 707.811 ms, 7.91 s total -[ 2023-09-21 09:56:13 ] Completed replacing temp checkpoint with checkpoint 40.229 ms, 7.95 s total -Epoch [128] :: gen_loss: 0.419, disc_loss: 0.017 -[ 2023-09-21 09:56:13 ] Completed training generator for epoch 128 58.350 ms, 8.01 s total -[ 2023-09-21 09:56:13 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 128 from eval step 0 - -[ 2023-09-21 09:56:14 ] Completed eval batch 0 to device 1,136.050 ms, 1.14 s total -[ 2023-09-21 09:56:14 ] Completed eval batch 0 forward 30.697 ms, 1.17 s total -[ 2023-09-21 09:56:14 ] Completed eval batch 0 recons_loss 0.322 ms, 1.17 s total -[ 2023-09-21 09:56:14 ] Completed eval batch 0 metrics update 0.609 ms, 1.17 s total -Saving checkpoint at epoch 128 val batch 0 -[ 2023-09-21 09:56:14 ] Completed saving temp checkpoint 411.503 ms, 1.58 s total -[ 2023-09-21 09:56:14 ] Completed replacing temp checkpoint with checkpoint 31.088 ms, 1.61 s total -[ 2023-09-21 09:56:15 ] Completed eval batch 1 to device 351.731 ms, 1.96 s total -[ 2023-09-21 09:56:15 ] Completed eval batch 1 forward 22.107 ms, 1.98 s total -[ 2023-09-21 09:56:15 ] Completed eval batch 1 recons_loss 0.385 ms, 1.98 s total -[ 2023-09-21 09:56:15 ] Completed eval batch 1 metrics update 0.327 ms, 1.98 s total -Saving checkpoint at epoch 128 val batch 1 -Epoch 128 val loss: 0.0612 -[ 2023-09-21 09:56:15 ] Completed saving temp checkpoint 423.335 ms, 2.41 s total -[ 2023-09-21 09:56:15 ] Completed replacing temp checkpoint with checkpoint 31.935 ms, 2.44 s total -[ 2023-09-21 09:56:15 ] Completed evaluating generator for epoch 128 56.841 ms, 2.50 s total - - -EPOCH :: 129 - - -[ 2023-09-21 09:56:15 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 129 from training step 0 - -[ 2023-09-21 09:56:18 ] Completed train batch 0 to device 3,099.898 ms, 3.10 s total -[ 2023-09-21 09:56:18 ] Completed train batch 0 generator forward 44.432 ms, 3.14 s total -[ 2023-09-21 09:56:19 ] Completed train batch 0 generator loss: 1.172 66.718 ms, 3.21 s total -[ 2023-09-21 09:56:19 ] Completed train batch 0 generator backward 590.672 ms, 3.80 s total -[ 2023-09-21 09:56:19 ] Completed train batch 0 discriminator loss 0.048 57.235 ms, 3.86 s total -[ 2023-09-21 09:56:19 ] Completed train batch 0 discriminator backward 81.033 ms, 3.94 s total -Epoch [129] Step [0/2], gen_loss: 0.411, disc_loss: 0.016 -[ 2023-09-21 09:56:19 ] Completed train batch 0 metrics update 22.213 ms, 3.96 s total -Saving checkpoint at epoch 129 train batch 0 -[ 2023-09-21 09:56:20 ] Completed saving temp checkpoint 426.501 ms, 4.39 s total -[ 2023-09-21 09:56:20 ] Completed replacing temp checkpoint with checkpoint 28.303 ms, 4.42 s total -[ 2023-09-21 09:56:21 ] Completed train batch 1 to device 1,675.900 ms, 6.09 s total -[ 2023-09-21 09:56:21 ] Completed train batch 1 generator forward 22.537 ms, 6.12 s total -[ 2023-09-21 09:56:21 ] Completed train batch 1 generator loss: 1.238 63.444 ms, 6.18 s total -[ 2023-09-21 09:56:23 ] Completed train batch 1 generator backward 1,096.348 ms, 7.28 s total -[ 2023-09-21 09:56:23 ] Completed train batch 1 discriminator loss 0.048 56.828 ms, 7.33 s total -[ 2023-09-21 09:56:23 ] Completed train batch 1 discriminator backward 77.028 ms, 7.41 s total -Epoch [129] Step [1/2], gen_loss: 0.417, disc_loss: 0.016 -[ 2023-09-21 09:56:23 ] Completed train batch 1 metrics update 4.991 ms, 7.41 s total -Saving checkpoint at epoch 129 train batch 1 -[ 2023-09-21 09:56:23 ] Completed saving temp checkpoint 428.368 ms, 7.84 s total -[ 2023-09-21 09:56:23 ] Completed replacing temp checkpoint with checkpoint 25.891 ms, 7.87 s total -Epoch [129] :: gen_loss: 0.414, disc_loss: 0.016 -[ 2023-09-21 09:56:23 ] Completed training generator for epoch 129 55.755 ms, 7.92 s total -[ 2023-09-21 09:56:23 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 129 from eval step 0 - -[ 2023-09-21 09:56:24 ] Completed eval batch 0 to device 1,168.545 ms, 1.17 s total -[ 2023-09-21 09:56:24 ] Completed eval batch 0 forward 25.004 ms, 1.19 s total -[ 2023-09-21 09:56:24 ] Completed eval batch 0 recons_loss 0.297 ms, 1.19 s total -[ 2023-09-21 09:56:24 ] Completed eval batch 0 metrics update 0.591 ms, 1.19 s total -Saving checkpoint at epoch 129 val batch 0 -[ 2023-09-21 09:56:25 ] Completed saving temp checkpoint 368.532 ms, 1.56 s total -[ 2023-09-21 09:56:25 ] Completed replacing temp checkpoint with checkpoint 26.831 ms, 1.59 s total -[ 2023-09-21 09:56:25 ] Completed eval batch 1 to device 482.123 ms, 2.07 s total -[ 2023-09-21 09:56:25 ] Completed eval batch 1 forward 39.958 ms, 2.11 s total -[ 2023-09-21 09:56:25 ] Completed eval batch 1 recons_loss 0.368 ms, 2.11 s total -[ 2023-09-21 09:56:25 ] Completed eval batch 1 metrics update 86.107 ms, 2.20 s total -Saving checkpoint at epoch 129 val batch 1 -Epoch 129 val loss: 0.0557 -[ 2023-09-21 09:56:26 ] Completed saving temp checkpoint 394.928 ms, 2.59 s total -[ 2023-09-21 09:56:26 ] Completed replacing temp checkpoint with checkpoint 31.003 ms, 2.62 s total -[ 2023-09-21 09:56:26 ] Completed evaluating generator for epoch 129 50.587 ms, 2.67 s total - - -EPOCH :: 130 - - -[ 2023-09-21 09:56:26 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 130 from training step 0 - -[ 2023-09-21 09:56:29 ] Completed train batch 0 to device 2,913.267 ms, 2.91 s total -[ 2023-09-21 09:56:29 ] Completed train batch 0 generator forward 44.340 ms, 2.96 s total -[ 2023-09-21 09:56:29 ] Completed train batch 0 generator loss: 1.217 66.110 ms, 3.02 s total -[ 2023-09-21 09:56:30 ] Completed train batch 0 generator backward 779.375 ms, 3.80 s total -[ 2023-09-21 09:56:30 ] Completed train batch 0 discriminator loss 0.049 58.615 ms, 3.86 s total -[ 2023-09-21 09:56:30 ] Completed train batch 0 discriminator backward 99.349 ms, 3.96 s total -Epoch [130] Step [0/2], gen_loss: 0.413, disc_loss: 0.016 -[ 2023-09-21 09:56:30 ] Completed train batch 0 metrics update 4.978 ms, 3.97 s total -Saving checkpoint at epoch 130 train batch 0 -[ 2023-09-21 09:56:30 ] Completed saving temp checkpoint 385.588 ms, 4.35 s total -[ 2023-09-21 09:56:30 ] Completed replacing temp checkpoint with checkpoint 29.772 ms, 4.38 s total -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 10:08:35 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 10:08:35 ] Completed importing Timer 0.023 ms, 0.00 s total -[ 2023-09-21 10:08:40 ] Completed importing everything else 4,672.730 ms, 4.67 s total -| distributed init (rank 0): env:// -| distributed init (rank 3): env:// -| distributed init (rank 1): env:// -| distributed init (rank 4): env:// -| distributed init (rank 2): env:// -| distributed init (rank 5): env:// -[ 2023-09-21 10:08:49 ] Completed preliminaries 8,675.519 ms, 13.35 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 10:08:49 ] Completed build datasets 12.574 ms, 13.36 s total -[ 2023-09-21 10:08:49 ] Completed build samplers 0.047 ms, 13.36 s total -[ 2023-09-21 10:08:49 ] Completed build dataloaders 0.131 ms, 13.36 s total -[ 2023-09-21 10:08:49 ] Completed generator to device 73.679 ms, 13.43 s total -[ 2023-09-21 10:08:49 ] Completed discriminator to device 7.225 ms, 13.44 s total -[ 2023-09-21 10:08:49 ] Completed loss functions 394.300 ms, 13.84 s total -[ 2023-09-21 10:08:49 ] Completed models prepped for distribution 73.495 ms, 13.91 s total -[ 2023-09-21 10:08:49 ] Completed optimizers 0.663 ms, 13.91 s total -[ 2023-09-21 10:08:49 ] Completed grad scalers 0.021 ms, 13.91 s total -[ 2023-09-21 10:08:50 ] Completed checkpoint retrieval 469.713 ms, 14.38 s total - - -EPOCH :: 130 - - -[ 2023-09-21 10:08:50 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 130 from training step 1 - -[ 2023-09-21 10:08:53 ] Completed train batch 1 to device 3,464.454 ms, 3.46 s total -[ 2023-09-21 10:08:54 ] Completed train batch 1 generator forward 1,098.096 ms, 4.56 s total -[ 2023-09-21 10:08:54 ] Completed train batch 1 generator loss: 1.225 72.154 ms, 4.63 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 10:08:55 ] Completed train batch 1 generator backward 888.534 ms, 5.52 s total -[ 2023-09-21 10:08:55 ] Completed train batch 1 discriminator loss 0.051 65.729 ms, 5.59 s total -[ 2023-09-21 10:08:55 ] Completed train batch 1 discriminator backward 71.616 ms, 5.66 s total -Epoch [130] Step [1/2], gen_loss: 0.418, disc_loss: 0.017 -[ 2023-09-21 10:08:55 ] Completed train batch 1 metrics update 4.658 ms, 5.67 s total -Saving checkpoint at epoch 130 train batch 1 -[ 2023-09-21 10:08:56 ] Completed saving temp checkpoint 413.657 ms, 6.08 s total -[ 2023-09-21 10:08:56 ] Completed replacing temp checkpoint with checkpoint 131.643 ms, 6.21 s total -Epoch [130] :: gen_loss: 0.415, disc_loss: 0.017 -[ 2023-09-21 10:08:56 ] Completed training generator for epoch 130 50.429 ms, 6.26 s total -[ 2023-09-21 10:08:56 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 130 from eval step 0 - -[ 2023-09-21 10:08:57 ] Completed eval batch 0 to device 1,092.074 ms, 1.09 s total -[ 2023-09-21 10:08:57 ] Completed eval batch 0 forward 24.941 ms, 1.12 s total -[ 2023-09-21 10:08:57 ] Completed eval batch 0 recons_loss 0.301 ms, 1.12 s total -[ 2023-09-21 10:08:57 ] Completed eval batch 0 metrics update 0.606 ms, 1.12 s total -Saving checkpoint at epoch 130 val batch 0 -[ 2023-09-21 10:08:57 ] Completed saving temp checkpoint 394.198 ms, 1.51 s total -[ 2023-09-21 10:08:57 ] Completed replacing temp checkpoint with checkpoint 18.653 ms, 1.53 s total -[ 2023-09-21 10:08:58 ] Completed eval batch 1 to device 411.682 ms, 1.94 s total -[ 2023-09-21 10:08:58 ] Completed eval batch 1 forward 39.716 ms, 1.98 s total -[ 2023-09-21 10:08:58 ] Completed eval batch 1 recons_loss 0.551 ms, 1.98 s total -[ 2023-09-21 10:08:58 ] Completed eval batch 1 metrics update 166.134 ms, 2.15 s total -Saving checkpoint at epoch 130 val batch 1 -Epoch 130 val loss: 0.0559 -[ 2023-09-21 10:08:58 ] Completed saving temp checkpoint 402.095 ms, 2.55 s total -[ 2023-09-21 10:08:58 ] Completed replacing temp checkpoint with checkpoint 25.718 ms, 2.58 s total -[ 2023-09-21 10:08:58 ] Completed evaluating generator for epoch 130 55.605 ms, 2.63 s total - - -EPOCH :: 131 - - -[ 2023-09-21 10:08:58 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 131 from training step 0 - -[ 2023-09-21 10:09:02 ] Completed train batch 0 to device 3,104.007 ms, 3.10 s total -[ 2023-09-21 10:09:02 ] Completed train batch 0 generator forward 41.964 ms, 3.15 s total -[ 2023-09-21 10:09:02 ] Completed train batch 0 generator loss: 1.259 66.184 ms, 3.21 s total -[ 2023-09-21 10:09:02 ] Completed train batch 0 generator backward 503.805 ms, 3.72 s total -[ 2023-09-21 10:09:02 ] Completed train batch 0 discriminator loss 0.054 57.413 ms, 3.77 s total -[ 2023-09-21 10:09:02 ] Completed train batch 0 discriminator backward 113.193 ms, 3.89 s total -Epoch [131] Step [0/2], gen_loss: 0.412, disc_loss: 0.017 -[ 2023-09-21 10:09:02 ] Completed train batch 0 metrics update 23.786 ms, 3.91 s total -Saving checkpoint at epoch 131 train batch 0 -[ 2023-09-21 10:09:03 ] Completed saving temp checkpoint 408.243 ms, 4.32 s total -[ 2023-09-21 10:09:03 ] Completed replacing temp checkpoint with checkpoint 28.782 ms, 4.35 s total -[ 2023-09-21 10:09:04 ] Completed train batch 1 to device 1,573.176 ms, 5.92 s total -[ 2023-09-21 10:09:04 ] Completed train batch 1 generator forward 24.096 ms, 5.94 s total -[ 2023-09-21 10:09:04 ] Completed train batch 1 generator loss: 1.283 63.137 ms, 6.01 s total -[ 2023-09-21 10:09:06 ] Completed train batch 1 generator backward 1,099.368 ms, 7.11 s total -[ 2023-09-21 10:09:06 ] Completed train batch 1 discriminator loss 0.052 56.600 ms, 7.16 s total -[ 2023-09-21 10:09:06 ] Completed train batch 1 discriminator backward 72.646 ms, 7.24 s total -Epoch [131] Step [1/2], gen_loss: 0.412, disc_loss: 0.017 -[ 2023-09-21 10:09:06 ] Completed train batch 1 metrics update 5.065 ms, 7.24 s total -Saving checkpoint at epoch 131 train batch 1 -[ 2023-09-21 10:09:06 ] Completed saving temp checkpoint 425.724 ms, 7.67 s total -[ 2023-09-21 10:09:06 ] Completed replacing temp checkpoint with checkpoint 28.199 ms, 7.70 s total -Epoch [131] :: gen_loss: 0.412, disc_loss: 0.017 -[ 2023-09-21 10:09:06 ] Completed training generator for epoch 131 53.083 ms, 7.75 s total -[ 2023-09-21 10:09:06 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 131 from eval step 0 - -[ 2023-09-21 10:09:07 ] Completed eval batch 0 to device 1,075.652 ms, 1.08 s total -[ 2023-09-21 10:09:07 ] Completed eval batch 0 forward 23.732 ms, 1.10 s total -[ 2023-09-21 10:09:07 ] Completed eval batch 0 recons_loss 0.226 ms, 1.10 s total -[ 2023-09-21 10:09:07 ] Completed eval batch 0 metrics update 0.496 ms, 1.10 s total -Saving checkpoint at epoch 131 val batch 0 -[ 2023-09-21 10:09:08 ] Completed saving temp checkpoint 378.312 ms, 1.48 s total -[ 2023-09-21 10:09:08 ] Completed replacing temp checkpoint with checkpoint 23.204 ms, 1.50 s total -[ 2023-09-21 10:09:08 ] Completed eval batch 1 to device 477.528 ms, 1.98 s total -[ 2023-09-21 10:09:08 ] Completed eval batch 1 forward 39.911 ms, 2.02 s total -[ 2023-09-21 10:09:08 ] Completed eval batch 1 recons_loss 0.554 ms, 2.02 s total -[ 2023-09-21 10:09:08 ] Completed eval batch 1 metrics update 117.433 ms, 2.14 s total -Saving checkpoint at epoch 131 val batch 1 -Epoch 131 val loss: 0.0595 -[ 2023-09-21 10:09:09 ] Completed saving temp checkpoint 486.625 ms, 2.62 s total -[ 2023-09-21 10:09:09 ] Completed replacing temp checkpoint with checkpoint 24.007 ms, 2.65 s total -[ 2023-09-21 10:09:09 ] Completed evaluating generator for epoch 131 64.320 ms, 2.71 s total - - -EPOCH :: 132 - - -[ 2023-09-21 10:09:09 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 132 from training step 0 - -[ 2023-09-21 10:09:12 ] Completed train batch 0 to device 2,969.579 ms, 2.97 s total -[ 2023-09-21 10:09:12 ] Completed train batch 0 generator forward 27.441 ms, 3.00 s total -[ 2023-09-21 10:09:12 ] Completed train batch 0 generator loss: 1.279 75.189 ms, 3.07 s total -[ 2023-09-21 10:09:13 ] Completed train batch 0 generator backward 605.966 ms, 3.68 s total -[ 2023-09-21 10:09:13 ] Completed train batch 0 discriminator loss 0.050 57.210 ms, 3.74 s total -[ 2023-09-21 10:09:13 ] Completed train batch 0 discriminator backward 106.258 ms, 3.84 s total -Epoch [132] Step [0/2], gen_loss: 0.416, disc_loss: 0.017 -[ 2023-09-21 10:09:13 ] Completed train batch 0 metrics update 5.468 ms, 3.85 s total -Saving checkpoint at epoch 132 train batch 0 -[ 2023-09-21 10:09:13 ] Completed saving temp checkpoint 443.045 ms, 4.29 s total -[ 2023-09-21 10:09:13 ] Completed replacing temp checkpoint with checkpoint 23.555 ms, 4.31 s total -[ 2023-09-21 10:09:15 ] Completed train batch 1 to device 1,442.636 ms, 5.76 s total -[ 2023-09-21 10:09:15 ] Completed train batch 1 generator forward 39.882 ms, 5.80 s total -[ 2023-09-21 10:09:15 ] Completed train batch 1 generator loss: 1.276 63.512 ms, 5.86 s total -[ 2023-09-21 10:09:16 ] Completed train batch 1 generator backward 1,409.417 ms, 7.27 s total -[ 2023-09-21 10:09:16 ] Completed train batch 1 discriminator loss 0.049 56.689 ms, 7.33 s total -[ 2023-09-21 10:09:16 ] Completed train batch 1 discriminator backward 74.761 ms, 7.40 s total -Epoch [132] Step [1/2], gen_loss: 0.415, disc_loss: 0.016 -[ 2023-09-21 10:09:16 ] Completed train batch 1 metrics update 2.243 ms, 7.40 s total -Saving checkpoint at epoch 132 train batch 1 -[ 2023-09-21 10:09:17 ] Completed saving temp checkpoint 453.911 ms, 7.86 s total -[ 2023-09-21 10:09:17 ] Completed replacing temp checkpoint with checkpoint 30.495 ms, 7.89 s total -Epoch [132] :: gen_loss: 0.415, disc_loss: 0.017 -[ 2023-09-21 10:09:17 ] Completed training generator for epoch 132 56.447 ms, 7.94 s total -[ 2023-09-21 10:09:17 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 132 from eval step 0 - -[ 2023-09-21 10:09:18 ] Completed eval batch 0 to device 1,131.609 ms, 1.13 s total -[ 2023-09-21 10:09:18 ] Completed eval batch 0 forward 42.609 ms, 1.17 s total -[ 2023-09-21 10:09:18 ] Completed eval batch 0 recons_loss 0.312 ms, 1.17 s total -[ 2023-09-21 10:09:18 ] Completed eval batch 0 metrics update 0.591 ms, 1.18 s total -Saving checkpoint at epoch 132 val batch 0 -[ 2023-09-21 10:09:18 ] Completed saving temp checkpoint 380.509 ms, 1.56 s total -[ 2023-09-21 10:09:18 ] Completed replacing temp checkpoint with checkpoint 29.447 ms, 1.59 s total -[ 2023-09-21 10:09:19 ] Completed eval batch 1 to device 447.357 ms, 2.03 s total -[ 2023-09-21 10:09:19 ] Completed eval batch 1 forward 40.024 ms, 2.07 s total -[ 2023-09-21 10:09:19 ] Completed eval batch 1 recons_loss 0.552 ms, 2.07 s total -[ 2023-09-21 10:09:19 ] Completed eval batch 1 metrics update 180.271 ms, 2.25 s total -Saving checkpoint at epoch 132 val batch 1 -Epoch 132 val loss: 0.0610 -[ 2023-09-21 10:09:20 ] Completed saving temp checkpoint 397.280 ms, 2.65 s total -[ 2023-09-21 10:09:20 ] Completed replacing temp checkpoint with checkpoint 20.420 ms, 2.67 s total -[ 2023-09-21 10:09:20 ] Completed evaluating generator for epoch 132 46.540 ms, 2.72 s total - - -EPOCH :: 133 - - -[ 2023-09-21 10:09:20 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 133 from training step 0 - -[ 2023-09-21 10:09:23 ] Completed train batch 0 to device 3,110.791 ms, 3.11 s total -[ 2023-09-21 10:09:23 ] Completed train batch 0 generator forward 42.951 ms, 3.15 s total -[ 2023-09-21 10:09:23 ] Completed train batch 0 generator loss: 1.276 64.889 ms, 3.22 s total -[ 2023-09-21 10:09:23 ] Completed train batch 0 generator backward 633.321 ms, 3.85 s total -[ 2023-09-21 10:09:24 ] Completed train batch 0 discriminator loss 0.048 56.410 ms, 3.91 s total -[ 2023-09-21 10:09:24 ] Completed train batch 0 discriminator backward 95.081 ms, 4.00 s total -Epoch [133] Step [0/2], gen_loss: 0.422, disc_loss: 0.016 -[ 2023-09-21 10:09:24 ] Completed train batch 0 metrics update 4.936 ms, 4.01 s total -Saving checkpoint at epoch 133 train batch 0 -[ 2023-09-21 10:09:24 ] Completed saving temp checkpoint 451.106 ms, 4.46 s total -[ 2023-09-21 10:09:24 ] Completed replacing temp checkpoint with checkpoint 19.597 ms, 4.48 s total -[ 2023-09-21 10:09:25 ] Completed train batch 1 to device 1,406.301 ms, 5.89 s total -[ 2023-09-21 10:09:26 ] Completed train batch 1 generator forward 22.485 ms, 5.91 s total -[ 2023-09-21 10:09:26 ] Completed train batch 1 generator loss: 1.256 63.816 ms, 5.97 s total -[ 2023-09-21 10:09:27 ] Completed train batch 1 generator backward 1,245.284 ms, 7.22 s total -[ 2023-09-21 10:09:27 ] Completed train batch 1 discriminator loss 0.049 56.724 ms, 7.27 s total -[ 2023-09-21 10:09:27 ] Completed train batch 1 discriminator backward 71.427 ms, 7.35 s total -Epoch [133] Step [1/2], gen_loss: 0.416, disc_loss: 0.016 -[ 2023-09-21 10:09:27 ] Completed train batch 1 metrics update 0.596 ms, 7.35 s total -Saving checkpoint at epoch 133 train batch 1 -[ 2023-09-21 10:09:27 ] Completed saving temp checkpoint 484.990 ms, 7.83 s total -[ 2023-09-21 10:09:27 ] Completed replacing temp checkpoint with checkpoint 17.338 ms, 7.85 s total -Epoch [133] :: gen_loss: 0.419, disc_loss: 0.016 -[ 2023-09-21 10:09:27 ] Completed training generator for epoch 133 52.161 ms, 7.90 s total -[ 2023-09-21 10:09:27 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 133 from eval step 0 - -[ 2023-09-21 10:09:29 ] Completed eval batch 0 to device 1,070.066 ms, 1.07 s total -[ 2023-09-21 10:09:29 ] Completed eval batch 0 forward 23.945 ms, 1.09 s total -[ 2023-09-21 10:09:29 ] Completed eval batch 0 recons_loss 0.226 ms, 1.09 s total -[ 2023-09-21 10:09:29 ] Completed eval batch 0 metrics update 0.514 ms, 1.09 s total -Saving checkpoint at epoch 133 val batch 0 -[ 2023-09-21 10:09:29 ] Completed saving temp checkpoint 418.723 ms, 1.51 s total -[ 2023-09-21 10:09:29 ] Completed replacing temp checkpoint with checkpoint 27.518 ms, 1.54 s total -[ 2023-09-21 10:09:29 ] Completed eval batch 1 to device 422.612 ms, 1.96 s total -[ 2023-09-21 10:09:29 ] Completed eval batch 1 forward 21.683 ms, 1.99 s total -[ 2023-09-21 10:09:29 ] Completed eval batch 1 recons_loss 0.347 ms, 1.99 s total -[ 2023-09-21 10:09:30 ] Completed eval batch 1 metrics update 256.914 ms, 2.24 s total -Saving checkpoint at epoch 133 val batch 1 -Epoch 133 val loss: 0.0566 -[ 2023-09-21 10:09:30 ] Completed saving temp checkpoint 417.784 ms, 2.66 s total -[ 2023-09-21 10:09:30 ] Completed replacing temp checkpoint with checkpoint 21.846 ms, 2.68 s total -[ 2023-09-21 10:09:30 ] Completed evaluating generator for epoch 133 46.285 ms, 2.73 s total - - -EPOCH :: 134 - - -[ 2023-09-21 10:09:30 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 134 from training step 0 - -[ 2023-09-21 10:09:33 ] Completed train batch 0 to device 2,803.099 ms, 2.80 s total -[ 2023-09-21 10:09:33 ] Completed train batch 0 generator forward 24.906 ms, 2.83 s total -[ 2023-09-21 10:09:33 ] Completed train batch 0 generator loss: 1.285 64.415 ms, 2.89 s total -[ 2023-09-21 10:09:34 ] Completed train batch 0 generator backward 913.025 ms, 3.81 s total -[ 2023-09-21 10:09:34 ] Completed train batch 0 discriminator loss 0.050 56.435 ms, 3.86 s total -[ 2023-09-21 10:09:34 ] Completed train batch 0 discriminator backward 109.698 ms, 3.97 s total -Epoch [134] Step [0/2], gen_loss: 0.415, disc_loss: 0.017 -[ 2023-09-21 10:09:34 ] Completed train batch 0 metrics update 8.834 ms, 3.98 s total -Saving checkpoint at epoch 134 train batch 0 -[ 2023-09-21 10:09:35 ] Completed saving temp checkpoint 442.805 ms, 4.42 s total -[ 2023-09-21 10:09:35 ] Completed replacing temp checkpoint with checkpoint 30.289 ms, 4.45 s total -[ 2023-09-21 10:09:36 ] Completed train batch 1 to device 1,018.144 ms, 5.47 s total -[ 2023-09-21 10:09:36 ] Completed train batch 1 generator forward 39.220 ms, 5.51 s total -[ 2023-09-21 10:09:36 ] Completed train batch 1 generator loss: 1.237 62.417 ms, 5.57 s total -[ 2023-09-21 10:09:37 ] Completed train batch 1 generator backward 1,646.945 ms, 7.22 s total -[ 2023-09-21 10:09:38 ] Completed train batch 1 discriminator loss 0.052 56.599 ms, 7.28 s total -[ 2023-09-21 10:09:38 ] Completed train batch 1 discriminator backward 72.885 ms, 7.35 s total -Epoch [134] Step [1/2], gen_loss: 0.414, disc_loss: 0.017 -[ 2023-09-21 10:09:38 ] Completed train batch 1 metrics update 0.664 ms, 7.35 s total -Saving checkpoint at epoch 134 train batch 1 -[ 2023-09-21 10:09:38 ] Completed saving temp checkpoint 516.944 ms, 7.87 s total -[ 2023-09-21 10:09:38 ] Completed replacing temp checkpoint with checkpoint 24.262 ms, 7.89 s total -Epoch [134] :: gen_loss: 0.414, disc_loss: 0.017 -[ 2023-09-21 10:09:38 ] Completed training generator for epoch 134 61.644 ms, 7.95 s total -[ 2023-09-21 10:09:38 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 134 from eval step 0 - -[ 2023-09-21 10:09:39 ] Completed eval batch 0 to device 1,055.533 ms, 1.06 s total -[ 2023-09-21 10:09:39 ] Completed eval batch 0 forward 25.016 ms, 1.08 s total -[ 2023-09-21 10:09:39 ] Completed eval batch 0 recons_loss 0.297 ms, 1.08 s total -[ 2023-09-21 10:09:39 ] Completed eval batch 0 metrics update 0.601 ms, 1.08 s total -Saving checkpoint at epoch 134 val batch 0 -[ 2023-09-21 10:09:40 ] Completed saving temp checkpoint 449.658 ms, 1.53 s total -[ 2023-09-21 10:09:40 ] Completed replacing temp checkpoint with checkpoint 36.129 ms, 1.57 s total -[ 2023-09-21 10:09:40 ] Completed eval batch 1 to device 333.659 ms, 1.90 s total -[ 2023-09-21 10:09:40 ] Completed eval batch 1 forward 22.269 ms, 1.92 s total -[ 2023-09-21 10:09:40 ] Completed eval batch 1 recons_loss 0.348 ms, 1.92 s total -[ 2023-09-21 10:09:40 ] Completed eval batch 1 metrics update 262.210 ms, 2.19 s total -Saving checkpoint at epoch 134 val batch 1 -Epoch 134 val loss: 0.0512 -[ 2023-09-21 10:09:41 ] Completed saving temp checkpoint 446.764 ms, 2.63 s total -[ 2023-09-21 10:09:41 ] Completed replacing temp checkpoint with checkpoint 26.619 ms, 2.66 s total -[ 2023-09-21 10:09:41 ] Completed evaluating generator for epoch 134 51.425 ms, 2.71 s total - - -EPOCH :: 135 - - -[ 2023-09-21 10:09:41 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 135 from training step 0 - -[ 2023-09-21 10:09:44 ] Completed train batch 0 to device 2,977.969 ms, 2.98 s total -[ 2023-09-21 10:09:44 ] Completed train batch 0 generator forward 34.116 ms, 3.01 s total -[ 2023-09-21 10:09:44 ] Completed train batch 0 generator loss: 1.282 76.628 ms, 3.09 s total -[ 2023-09-21 10:09:45 ] Completed train batch 0 generator backward 677.238 ms, 3.77 s total -[ 2023-09-21 10:09:45 ] Completed train batch 0 discriminator loss 0.053 57.183 ms, 3.82 s total -[ 2023-09-21 10:09:45 ] Completed train batch 0 discriminator backward 84.301 ms, 3.91 s total -Epoch [135] Step [0/2], gen_loss: 0.415, disc_loss: 0.017 -[ 2023-09-21 10:09:45 ] Completed train batch 0 metrics update 20.943 ms, 3.93 s total -Saving checkpoint at epoch 135 train batch 0 -[ 2023-09-21 10:09:45 ] Completed saving temp checkpoint 401.671 ms, 4.33 s total -[ 2023-09-21 10:09:45 ] Completed replacing temp checkpoint with checkpoint 20.985 ms, 4.35 s total -[ 2023-09-21 10:09:47 ] Completed train batch 1 to device 1,731.447 ms, 6.08 s total -[ 2023-09-21 10:09:47 ] Completed train batch 1 generator forward 40.359 ms, 6.12 s total -[ 2023-09-21 10:09:47 ] Completed train batch 1 generator loss: 1.133 63.374 ms, 6.19 s total -[ 2023-09-21 10:09:48 ] Completed train batch 1 generator backward 671.246 ms, 6.86 s total -[ 2023-09-21 10:09:48 ] Completed train batch 1 discriminator loss 0.050 56.583 ms, 6.91 s total -[ 2023-09-21 10:09:48 ] Completed train batch 1 discriminator backward 69.060 ms, 6.98 s total -Epoch [135] Step [1/2], gen_loss: 0.407, disc_loss: 0.017 -[ 2023-09-21 10:09:48 ] Completed train batch 1 metrics update 0.698 ms, 6.98 s total -Saving checkpoint at epoch 135 train batch 1 -[ 2023-09-21 10:09:48 ] Completed saving temp checkpoint 464.251 ms, 7.45 s total -[ 2023-09-21 10:09:48 ] Completed replacing temp checkpoint with checkpoint 28.430 ms, 7.48 s total -Epoch [135] :: gen_loss: 0.411, disc_loss: 0.017 -[ 2023-09-21 10:09:48 ] Completed training generator for epoch 135 59.232 ms, 7.54 s total -[ 2023-09-21 10:09:48 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 135 from eval step 0 - -[ 2023-09-21 10:09:49 ] Completed eval batch 0 to device 1,058.407 ms, 1.06 s total -[ 2023-09-21 10:09:50 ] Completed eval batch 0 forward 42.205 ms, 1.10 s total -[ 2023-09-21 10:09:50 ] Completed eval batch 0 recons_loss 0.304 ms, 1.10 s total -[ 2023-09-21 10:09:50 ] Completed eval batch 0 metrics update 0.562 ms, 1.10 s total -Saving checkpoint at epoch 135 val batch 0 -[ 2023-09-21 10:09:50 ] Completed saving temp checkpoint 415.183 ms, 1.52 s total -[ 2023-09-21 10:09:50 ] Completed replacing temp checkpoint with checkpoint 26.111 ms, 1.54 s total -[ 2023-09-21 10:09:50 ] Completed eval batch 1 to device 362.227 ms, 1.91 s total -[ 2023-09-21 10:09:50 ] Completed eval batch 1 forward 40.039 ms, 1.95 s total -[ 2023-09-21 10:09:50 ] Completed eval batch 1 recons_loss 0.570 ms, 1.95 s total -[ 2023-09-21 10:09:51 ] Completed eval batch 1 metrics update 284.821 ms, 2.23 s total -Saving checkpoint at epoch 135 val batch 1 -Epoch 135 val loss: 0.0554 -[ 2023-09-21 10:09:51 ] Completed saving temp checkpoint 439.152 ms, 2.67 s total -[ 2023-09-21 10:09:51 ] Completed replacing temp checkpoint with checkpoint 21.358 ms, 2.69 s total -[ 2023-09-21 10:09:51 ] Completed evaluating generator for epoch 135 51.862 ms, 2.74 s total - - -EPOCH :: 136 - - -[ 2023-09-21 10:09:51 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 136 from training step 0 - -[ 2023-09-21 10:09:54 ] Completed train batch 0 to device 2,975.918 ms, 2.98 s total -[ 2023-09-21 10:09:54 ] Completed train batch 0 generator forward 34.055 ms, 3.01 s total -[ 2023-09-21 10:09:54 ] Completed train batch 0 generator loss: 1.291 66.567 ms, 3.08 s total -[ 2023-09-21 10:09:55 ] Completed train batch 0 generator backward 653.738 ms, 3.73 s total -[ 2023-09-21 10:09:55 ] Completed train batch 0 discriminator loss 0.052 56.996 ms, 3.79 s total -[ 2023-09-21 10:09:55 ] Completed train batch 0 discriminator backward 106.783 ms, 3.89 s total -Epoch [136] Step [0/2], gen_loss: 0.408, disc_loss: 0.017 -[ 2023-09-21 10:09:55 ] Completed train batch 0 metrics update 3.372 ms, 3.90 s total -Saving checkpoint at epoch 136 train batch 0 -[ 2023-09-21 10:09:55 ] Completed saving temp checkpoint 346.642 ms, 4.24 s total -[ 2023-09-21 10:09:55 ] Completed replacing temp checkpoint with checkpoint 25.422 ms, 4.27 s total -[ 2023-09-21 10:09:57 ] Completed train batch 1 to device 1,439.920 ms, 5.71 s total -[ 2023-09-21 10:09:57 ] Completed train batch 1 generator forward 41.072 ms, 5.75 s total -[ 2023-09-21 10:09:57 ] Completed train batch 1 generator loss: 1.191 63.669 ms, 5.81 s total -[ 2023-09-21 10:09:58 ] Completed train batch 1 generator backward 1,082.039 ms, 6.90 s total -[ 2023-09-21 10:09:58 ] Completed train batch 1 discriminator loss 0.049 56.785 ms, 6.95 s total -[ 2023-09-21 10:09:58 ] Completed train batch 1 discriminator backward 68.972 ms, 7.02 s total -Epoch [136] Step [1/2], gen_loss: 0.409, disc_loss: 0.017 -[ 2023-09-21 10:09:58 ] Completed train batch 1 metrics update 5.333 ms, 7.03 s total -Saving checkpoint at epoch 136 train batch 1 -[ 2023-09-21 10:09:59 ] Completed saving temp checkpoint 397.639 ms, 7.42 s total -[ 2023-09-21 10:09:59 ] Completed replacing temp checkpoint with checkpoint 26.047 ms, 7.45 s total -Epoch [136] :: gen_loss: 0.409, disc_loss: 0.017 -[ 2023-09-21 10:09:59 ] Completed training generator for epoch 136 60.052 ms, 7.51 s total -[ 2023-09-21 10:09:59 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 136 from eval step 0 - -[ 2023-09-21 10:10:00 ] Completed eval batch 0 to device 1,158.785 ms, 1.16 s total -[ 2023-09-21 10:10:00 ] Completed eval batch 0 forward 42.824 ms, 1.20 s total -[ 2023-09-21 10:10:00 ] Completed eval batch 0 recons_loss 0.309 ms, 1.20 s total -[ 2023-09-21 10:10:00 ] Completed eval batch 0 metrics update 0.582 ms, 1.20 s total -Saving checkpoint at epoch 136 val batch 0 -[ 2023-09-21 10:10:00 ] Completed saving temp checkpoint 344.932 ms, 1.55 s total -[ 2023-09-21 10:10:00 ] Completed replacing temp checkpoint with checkpoint 35.329 ms, 1.58 s total -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 10:23:29 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 10:23:29 ] Completed importing Timer 0.023 ms, 0.00 s total -[ 2023-09-21 10:23:34 ] Completed importing everything else 4,515.867 ms, 4.52 s total -| distributed init (rank 0): env:// -| distributed init (rank 5): env:// -| distributed init (rank 1): env:// -| distributed init (rank 3): env:// -| distributed init (rank 2): env:// -| distributed init (rank 4): env:// -[ 2023-09-21 10:23:42 ] Completed preliminaries 7,829.495 ms, 12.35 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 10:23:42 ] Completed build datasets 12.511 ms, 12.36 s total -[ 2023-09-21 10:23:42 ] Completed build samplers 0.047 ms, 12.36 s total -[ 2023-09-21 10:23:42 ] Completed build dataloaders 0.146 ms, 12.36 s total -[ 2023-09-21 10:23:42 ] Completed generator to device 78.840 ms, 12.44 s total -[ 2023-09-21 10:23:42 ] Completed discriminator to device 6.999 ms, 12.44 s total -[ 2023-09-21 10:23:42 ] Completed loss functions 403.760 ms, 12.85 s total -[ 2023-09-21 10:23:42 ] Completed models prepped for distribution 91.855 ms, 12.94 s total -[ 2023-09-21 10:23:42 ] Completed optimizers 0.729 ms, 12.94 s total -[ 2023-09-21 10:23:42 ] Completed grad scalers 0.034 ms, 12.94 s total -[ 2023-09-21 10:23:43 ] Completed checkpoint retrieval 524.468 ms, 13.46 s total - - -EPOCH :: 136 - - -[ 2023-09-21 10:23:43 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 136 from training step 2 - -Epoch [136] :: gen_loss: 0.409, disc_loss: 0.017 -[ 2023-09-21 10:23:43 ] Completed training generator for epoch 136 142.907 ms, 0.14 s total -[ 2023-09-21 10:23:43 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 136 from eval step 1 - -[ 2023-09-21 10:23:45 ] Completed eval batch 1 to device 1,479.558 ms, 1.48 s total -[ 2023-09-21 10:23:46 ] Completed eval batch 1 forward 1,158.731 ms, 2.64 s total -[ 2023-09-21 10:23:46 ] Completed eval batch 1 recons_loss 0.311 ms, 2.64 s total -[ 2023-09-21 10:23:46 ] Completed eval batch 1 metrics update 229.434 ms, 2.87 s total -Saving checkpoint at epoch 136 val batch 1 -Epoch 136 val loss: 0.0584 -[ 2023-09-21 10:23:46 ] Completed saving temp checkpoint 461.046 ms, 3.33 s total -[ 2023-09-21 10:23:47 ] Completed replacing temp checkpoint with checkpoint 119.003 ms, 3.45 s total -[ 2023-09-21 10:23:47 ] Completed evaluating generator for epoch 136 48.346 ms, 3.50 s total - - -EPOCH :: 137 - - -[ 2023-09-21 10:23:47 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 137 from training step 0 - -[ 2023-09-21 10:23:50 ] Completed train batch 0 to device 3,163.110 ms, 3.16 s total -[ 2023-09-21 10:23:50 ] Completed train batch 0 generator forward 43.518 ms, 3.21 s total -[ 2023-09-21 10:23:50 ] Completed train batch 0 generator loss: 1.202 79.561 ms, 3.29 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 10:23:50 ] Completed train batch 0 generator backward 590.481 ms, 3.88 s total -[ 2023-09-21 10:23:50 ] Completed train batch 0 discriminator loss 0.058 58.248 ms, 3.93 s total -[ 2023-09-21 10:23:51 ] Completed train batch 0 discriminator backward 102.573 ms, 4.04 s total -Epoch [137] Step [0/2], gen_loss: 0.412, disc_loss: 0.019 -[ 2023-09-21 10:23:51 ] Completed train batch 0 metrics update 4.051 ms, 4.04 s total -Saving checkpoint at epoch 137 train batch 0 -[ 2023-09-21 10:23:51 ] Completed saving temp checkpoint 465.046 ms, 4.51 s total -[ 2023-09-21 10:23:51 ] Completed replacing temp checkpoint with checkpoint 24.310 ms, 4.53 s total -[ 2023-09-21 10:23:53 ] Completed train batch 1 to device 1,641.582 ms, 6.17 s total -[ 2023-09-21 10:23:53 ] Completed train batch 1 generator forward 28.458 ms, 6.20 s total -[ 2023-09-21 10:23:53 ] Completed train batch 1 generator loss: 1.235 63.335 ms, 6.26 s total -[ 2023-09-21 10:23:54 ] Completed train batch 1 generator backward 1,014.821 ms, 7.28 s total -[ 2023-09-21 10:23:54 ] Completed train batch 1 discriminator loss 0.079 56.528 ms, 7.34 s total -[ 2023-09-21 10:23:54 ] Completed train batch 1 discriminator backward 74.696 ms, 7.41 s total -Epoch [137] Step [1/2], gen_loss: 0.408, disc_loss: 0.026 -[ 2023-09-21 10:23:54 ] Completed train batch 1 metrics update 3.564 ms, 7.41 s total -Saving checkpoint at epoch 137 train batch 1 -[ 2023-09-21 10:23:55 ] Completed saving temp checkpoint 553.008 ms, 7.97 s total -[ 2023-09-21 10:23:55 ] Completed replacing temp checkpoint with checkpoint 24.387 ms, 7.99 s total -Epoch [137] :: gen_loss: 0.410, disc_loss: 0.023 -[ 2023-09-21 10:23:55 ] Completed training generator for epoch 137 68.990 ms, 8.06 s total -[ 2023-09-21 10:23:55 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 137 from eval step 0 - -[ 2023-09-21 10:23:56 ] Completed eval batch 0 to device 1,102.988 ms, 1.10 s total -[ 2023-09-21 10:23:56 ] Completed eval batch 0 forward 24.993 ms, 1.13 s total -[ 2023-09-21 10:23:56 ] Completed eval batch 0 recons_loss 0.314 ms, 1.13 s total -[ 2023-09-21 10:23:56 ] Completed eval batch 0 metrics update 0.581 ms, 1.13 s total -Saving checkpoint at epoch 137 val batch 0 -[ 2023-09-21 10:23:56 ] Completed saving temp checkpoint 673.598 ms, 1.80 s total -[ 2023-09-21 10:23:56 ] Completed replacing temp checkpoint with checkpoint 27.446 ms, 1.83 s total -[ 2023-09-21 10:23:57 ] Completed eval batch 1 to device 232.574 ms, 2.06 s total -[ 2023-09-21 10:23:57 ] Completed eval batch 1 forward 21.464 ms, 2.08 s total -[ 2023-09-21 10:23:57 ] Completed eval batch 1 recons_loss 0.358 ms, 2.08 s total -[ 2023-09-21 10:23:57 ] Completed eval batch 1 metrics update 48.838 ms, 2.13 s total -Saving checkpoint at epoch 137 val batch 1 -Epoch 137 val loss: 0.0627 -[ 2023-09-21 10:23:58 ] Completed saving temp checkpoint 762.302 ms, 2.90 s total -[ 2023-09-21 10:23:58 ] Completed replacing temp checkpoint with checkpoint 29.030 ms, 2.92 s total -[ 2023-09-21 10:23:58 ] Completed evaluating generator for epoch 137 48.991 ms, 2.97 s total - - -EPOCH :: 138 - - -[ 2023-09-21 10:23:58 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 138 from training step 0 - -[ 2023-09-21 10:24:01 ] Completed train batch 0 to device 3,036.774 ms, 3.04 s total -[ 2023-09-21 10:24:01 ] Completed train batch 0 generator forward 30.539 ms, 3.07 s total -[ 2023-09-21 10:24:01 ] Completed train batch 0 generator loss: 1.273 66.150 ms, 3.13 s total -[ 2023-09-21 10:24:01 ] Completed train batch 0 generator backward 407.581 ms, 3.54 s total -[ 2023-09-21 10:24:01 ] Completed train batch 0 discriminator loss 0.066 58.353 ms, 3.60 s total -[ 2023-09-21 10:24:01 ] Completed train batch 0 discriminator backward 107.181 ms, 3.71 s total -Epoch [138] Step [0/2], gen_loss: 0.406, disc_loss: 0.022 -[ 2023-09-21 10:24:01 ] Completed train batch 0 metrics update 23.927 ms, 3.73 s total -Saving checkpoint at epoch 138 train batch 0 -[ 2023-09-21 10:24:02 ] Completed saving temp checkpoint 482.197 ms, 4.21 s total -[ 2023-09-21 10:24:02 ] Completed replacing temp checkpoint with checkpoint 35.094 ms, 4.25 s total -[ 2023-09-21 10:24:04 ] Completed train batch 1 to device 1,960.523 ms, 6.21 s total -[ 2023-09-21 10:24:04 ] Completed train batch 1 generator forward 40.631 ms, 6.25 s total -[ 2023-09-21 10:24:04 ] Completed train batch 1 generator loss: 1.270 63.440 ms, 6.31 s total -[ 2023-09-21 10:24:05 ] Completed train batch 1 generator backward 922.469 ms, 7.23 s total -[ 2023-09-21 10:24:05 ] Completed train batch 1 discriminator loss 0.054 56.642 ms, 7.29 s total -[ 2023-09-21 10:24:05 ] Completed train batch 1 discriminator backward 78.110 ms, 7.37 s total -Epoch [138] Step [1/2], gen_loss: 0.414, disc_loss: 0.018 -[ 2023-09-21 10:24:05 ] Completed train batch 1 metrics update 1.203 ms, 7.37 s total -Saving checkpoint at epoch 138 train batch 1 -[ 2023-09-21 10:24:05 ] Completed saving temp checkpoint 498.521 ms, 7.87 s total -[ 2023-09-21 10:24:05 ] Completed replacing temp checkpoint with checkpoint 20.224 ms, 7.89 s total -Epoch [138] :: gen_loss: 0.410, disc_loss: 0.020 -[ 2023-09-21 10:24:06 ] Completed training generator for epoch 138 53.223 ms, 7.94 s total -[ 2023-09-21 10:24:06 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 138 from eval step 0 - -[ 2023-09-21 10:24:07 ] Completed eval batch 0 to device 1,143.013 ms, 1.14 s total -[ 2023-09-21 10:24:07 ] Completed eval batch 0 forward 25.307 ms, 1.17 s total -[ 2023-09-21 10:24:07 ] Completed eval batch 0 recons_loss 0.309 ms, 1.17 s total -[ 2023-09-21 10:24:07 ] Completed eval batch 0 metrics update 0.623 ms, 1.17 s total -Saving checkpoint at epoch 138 val batch 0 -[ 2023-09-21 10:24:07 ] Completed saving temp checkpoint 452.054 ms, 1.62 s total -[ 2023-09-21 10:24:07 ] Completed replacing temp checkpoint with checkpoint 23.590 ms, 1.64 s total -[ 2023-09-21 10:24:08 ] Completed eval batch 1 to device 404.125 ms, 2.05 s total -[ 2023-09-21 10:24:08 ] Completed eval batch 1 forward 40.113 ms, 2.09 s total -[ 2023-09-21 10:24:08 ] Completed eval batch 1 recons_loss 0.452 ms, 2.09 s total -[ 2023-09-21 10:24:08 ] Completed eval batch 1 metrics update 186.879 ms, 2.28 s total -Saving checkpoint at epoch 138 val batch 1 -Epoch 138 val loss: 0.0671 -[ 2023-09-21 10:24:08 ] Completed saving temp checkpoint 498.068 ms, 2.77 s total -[ 2023-09-21 10:24:08 ] Completed replacing temp checkpoint with checkpoint 23.522 ms, 2.80 s total -[ 2023-09-21 10:24:08 ] Completed evaluating generator for epoch 138 51.846 ms, 2.85 s total - - -EPOCH :: 139 - - -[ 2023-09-21 10:24:08 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 139 from training step 0 - -[ 2023-09-21 10:24:12 ] Completed train batch 0 to device 3,223.835 ms, 3.22 s total -[ 2023-09-21 10:24:12 ] Completed train batch 0 generator forward 35.526 ms, 3.26 s total -[ 2023-09-21 10:24:12 ] Completed train batch 0 generator loss: 1.204 65.554 ms, 3.32 s total -[ 2023-09-21 10:24:12 ] Completed train batch 0 generator backward 552.246 ms, 3.88 s total -[ 2023-09-21 10:24:12 ] Completed train batch 0 discriminator loss 0.052 62.744 ms, 3.94 s total -[ 2023-09-21 10:24:12 ] Completed train batch 0 discriminator backward 101.004 ms, 4.04 s total -Epoch [139] Step [0/2], gen_loss: 0.418, disc_loss: 0.018 -[ 2023-09-21 10:24:12 ] Completed train batch 0 metrics update 6.872 ms, 4.05 s total -Saving checkpoint at epoch 139 train batch 0 -[ 2023-09-21 10:24:13 ] Completed saving temp checkpoint 441.238 ms, 4.49 s total -[ 2023-09-21 10:24:13 ] Completed replacing temp checkpoint with checkpoint 23.390 ms, 4.51 s total -[ 2023-09-21 10:24:15 ] Completed train batch 1 to device 1,699.240 ms, 6.21 s total -[ 2023-09-21 10:24:15 ] Completed train batch 1 generator forward 39.389 ms, 6.25 s total -[ 2023-09-21 10:24:15 ] Completed train batch 1 generator loss: 1.225 65.172 ms, 6.32 s total -[ 2023-09-21 10:24:15 ] Completed train batch 1 generator backward 752.446 ms, 7.07 s total -[ 2023-09-21 10:24:16 ] Completed train batch 1 discriminator loss 0.051 57.517 ms, 7.13 s total -[ 2023-09-21 10:24:16 ] Completed train batch 1 discriminator backward 77.423 ms, 7.20 s total -Epoch [139] Step [1/2], gen_loss: 0.422, disc_loss: 0.017 -[ 2023-09-21 10:24:16 ] Completed train batch 1 metrics update 0.603 ms, 7.20 s total -Saving checkpoint at epoch 139 train batch 1 -[ 2023-09-21 10:24:16 ] Completed saving temp checkpoint 473.023 ms, 7.68 s total -[ 2023-09-21 10:24:16 ] Completed replacing temp checkpoint with checkpoint 23.186 ms, 7.70 s total -Epoch [139] :: gen_loss: 0.420, disc_loss: 0.017 -[ 2023-09-21 10:24:16 ] Completed training generator for epoch 139 53.885 ms, 7.75 s total -[ 2023-09-21 10:24:16 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 139 from eval step 0 - -[ 2023-09-21 10:24:17 ] Completed eval batch 0 to device 1,149.328 ms, 1.15 s total -[ 2023-09-21 10:24:17 ] Completed eval batch 0 forward 41.512 ms, 1.19 s total -[ 2023-09-21 10:24:17 ] Completed eval batch 0 recons_loss 0.359 ms, 1.19 s total -[ 2023-09-21 10:24:17 ] Completed eval batch 0 metrics update 0.694 ms, 1.19 s total -Saving checkpoint at epoch 139 val batch 0 -[ 2023-09-21 10:24:18 ] Completed saving temp checkpoint 434.954 ms, 1.63 s total -[ 2023-09-21 10:24:18 ] Completed replacing temp checkpoint with checkpoint 24.408 ms, 1.65 s total -[ 2023-09-21 10:24:18 ] Completed eval batch 1 to device 415.411 ms, 2.07 s total -[ 2023-09-21 10:24:18 ] Completed eval batch 1 forward 21.605 ms, 2.09 s total -[ 2023-09-21 10:24:18 ] Completed eval batch 1 recons_loss 0.367 ms, 2.09 s total -[ 2023-09-21 10:24:18 ] Completed eval batch 1 metrics update 193.887 ms, 2.28 s total -Saving checkpoint at epoch 139 val batch 1 -Epoch 139 val loss: 0.0599 -[ 2023-09-21 10:24:19 ] Completed saving temp checkpoint 476.679 ms, 2.76 s total -[ 2023-09-21 10:24:19 ] Completed replacing temp checkpoint with checkpoint 27.842 ms, 2.79 s total -[ 2023-09-21 10:24:19 ] Completed evaluating generator for epoch 139 49.559 ms, 2.84 s total - - -EPOCH :: 140 - - -[ 2023-09-21 10:24:19 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 140 from training step 0 - -[ 2023-09-21 10:24:22 ] Completed train batch 0 to device 2,988.764 ms, 2.99 s total -[ 2023-09-21 10:24:22 ] Completed train batch 0 generator forward 42.495 ms, 3.03 s total -[ 2023-09-21 10:24:22 ] Completed train batch 0 generator loss: 1.278 66.655 ms, 3.10 s total -[ 2023-09-21 10:24:23 ] Completed train batch 0 generator backward 788.418 ms, 3.89 s total -[ 2023-09-21 10:24:23 ] Completed train batch 0 discriminator loss 0.050 57.698 ms, 3.94 s total -[ 2023-09-21 10:24:23 ] Completed train batch 0 discriminator backward 106.500 ms, 4.05 s total -Epoch [140] Step [0/2], gen_loss: 0.413, disc_loss: 0.017 -[ 2023-09-21 10:24:23 ] Completed train batch 0 metrics update 6.077 ms, 4.06 s total -Saving checkpoint at epoch 140 train batch 0 -[ 2023-09-21 10:24:23 ] Completed saving temp checkpoint 459.635 ms, 4.52 s total -[ 2023-09-21 10:24:24 ] Completed replacing temp checkpoint with checkpoint 30.014 ms, 4.55 s total -[ 2023-09-21 10:24:25 ] Completed train batch 1 to device 1,199.730 ms, 5.75 s total -[ 2023-09-21 10:24:25 ] Completed train batch 1 generator forward 36.197 ms, 5.78 s total -[ 2023-09-21 10:24:25 ] Completed train batch 1 generator loss: 1.193 63.454 ms, 5.85 s total -[ 2023-09-21 10:24:27 ] Completed train batch 1 generator backward 1,697.378 ms, 7.54 s total -[ 2023-09-21 10:24:27 ] Completed train batch 1 discriminator loss 0.051 56.760 ms, 7.60 s total -[ 2023-09-21 10:24:27 ] Completed train batch 1 discriminator backward 79.412 ms, 7.68 s total -Epoch [140] Step [1/2], gen_loss: 0.411, disc_loss: 0.017 -[ 2023-09-21 10:24:27 ] Completed train batch 1 metrics update 5.405 ms, 7.68 s total -Saving checkpoint at epoch 140 train batch 1 -[ 2023-09-21 10:24:27 ] Completed saving temp checkpoint 446.228 ms, 8.13 s total -[ 2023-09-21 10:24:27 ] Completed replacing temp checkpoint with checkpoint 29.909 ms, 8.16 s total -Epoch [140] :: gen_loss: 0.412, disc_loss: 0.017 -[ 2023-09-21 10:24:27 ] Completed training generator for epoch 140 52.212 ms, 8.21 s total -[ 2023-09-21 10:24:27 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 140 from eval step 0 - -[ 2023-09-21 10:24:28 ] Completed eval batch 0 to device 1,170.899 ms, 1.17 s total -[ 2023-09-21 10:24:28 ] Completed eval batch 0 forward 43.275 ms, 1.21 s total -[ 2023-09-21 10:24:28 ] Completed eval batch 0 recons_loss 0.440 ms, 1.21 s total -[ 2023-09-21 10:24:28 ] Completed eval batch 0 metrics update 0.796 ms, 1.22 s total -Saving checkpoint at epoch 140 val batch 0 -[ 2023-09-21 10:24:29 ] Completed saving temp checkpoint 411.828 ms, 1.63 s total -[ 2023-09-21 10:24:29 ] Completed replacing temp checkpoint with checkpoint 29.904 ms, 1.66 s total -[ 2023-09-21 10:24:29 ] Completed eval batch 1 to device 423.048 ms, 2.08 s total -[ 2023-09-21 10:24:29 ] Completed eval batch 1 forward 39.963 ms, 2.12 s total -[ 2023-09-21 10:24:29 ] Completed eval batch 1 recons_loss 0.598 ms, 2.12 s total -[ 2023-09-21 10:24:30 ] Completed eval batch 1 metrics update 214.965 ms, 2.34 s total -Saving checkpoint at epoch 140 val batch 1 -Epoch 140 val loss: 0.0569 -[ 2023-09-21 10:24:30 ] Completed saving temp checkpoint 448.146 ms, 2.78 s total -[ 2023-09-21 10:24:30 ] Completed replacing temp checkpoint with checkpoint 26.566 ms, 2.81 s total -[ 2023-09-21 10:24:30 ] Completed evaluating generator for epoch 140 47.807 ms, 2.86 s total - - -EPOCH :: 141 - - -[ 2023-09-21 10:24:30 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 141 from training step 0 - -[ 2023-09-21 10:24:33 ] Completed train batch 0 to device 3,047.083 ms, 3.05 s total -[ 2023-09-21 10:24:33 ] Completed train batch 0 generator forward 27.354 ms, 3.07 s total -[ 2023-09-21 10:24:33 ] Completed train batch 0 generator loss: 1.209 66.364 ms, 3.14 s total -[ 2023-09-21 10:24:34 ] Completed train batch 0 generator backward 754.758 ms, 3.90 s total -[ 2023-09-21 10:24:34 ] Completed train batch 0 discriminator loss 0.051 59.444 ms, 3.96 s total -[ 2023-09-21 10:24:34 ] Completed train batch 0 discriminator backward 84.337 ms, 4.04 s total -Epoch [141] Step [0/2], gen_loss: 0.407, disc_loss: 0.017 -[ 2023-09-21 10:24:34 ] Completed train batch 0 metrics update 21.526 ms, 4.06 s total -Saving checkpoint at epoch 141 train batch 0 -[ 2023-09-21 10:24:35 ] Completed saving temp checkpoint 425.043 ms, 4.49 s total -[ 2023-09-21 10:24:35 ] Completed replacing temp checkpoint with checkpoint 25.907 ms, 4.51 s total -[ 2023-09-21 10:24:36 ] Completed train batch 1 to device 1,285.156 ms, 5.80 s total -[ 2023-09-21 10:24:36 ] Completed train batch 1 generator forward 41.056 ms, 5.84 s total -[ 2023-09-21 10:24:36 ] Completed train batch 1 generator loss: 1.219 62.446 ms, 5.90 s total -[ 2023-09-21 10:24:38 ] Completed train batch 1 generator backward 1,673.577 ms, 7.57 s total -[ 2023-09-21 10:24:38 ] Completed train batch 1 discriminator loss 0.050 56.573 ms, 7.63 s total -[ 2023-09-21 10:24:38 ] Completed train batch 1 discriminator backward 78.419 ms, 7.71 s total -Epoch [141] Step [1/2], gen_loss: 0.408, disc_loss: 0.017 -[ 2023-09-21 10:24:38 ] Completed train batch 1 metrics update 4.797 ms, 7.71 s total -Saving checkpoint at epoch 141 train batch 1 -[ 2023-09-21 10:24:38 ] Completed saving temp checkpoint 606.448 ms, 8.32 s total -[ 2023-09-21 10:24:38 ] Completed replacing temp checkpoint with checkpoint 27.935 ms, 8.35 s total -Epoch [141] :: gen_loss: 0.408, disc_loss: 0.017 -[ 2023-09-21 10:24:38 ] Completed training generator for epoch 141 49.213 ms, 8.40 s total -[ 2023-09-21 10:24:38 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 141 from eval step 0 - -[ 2023-09-21 10:24:39 ] Completed eval batch 0 to device 1,043.172 ms, 1.04 s total -[ 2023-09-21 10:24:40 ] Completed eval batch 0 forward 41.070 ms, 1.08 s total -[ 2023-09-21 10:24:40 ] Completed eval batch 0 recons_loss 0.338 ms, 1.08 s total -[ 2023-09-21 10:24:40 ] Completed eval batch 0 metrics update 0.599 ms, 1.09 s total -Saving checkpoint at epoch 141 val batch 0 -[ 2023-09-21 10:24:40 ] Completed saving temp checkpoint 507.136 ms, 1.59 s total -[ 2023-09-21 10:24:40 ] Completed replacing temp checkpoint with checkpoint 22.735 ms, 1.62 s total -[ 2023-09-21 10:24:40 ] Completed eval batch 1 to device 294.415 ms, 1.91 s total -[ 2023-09-21 10:24:40 ] Completed eval batch 1 forward 39.570 ms, 1.95 s total -[ 2023-09-21 10:24:40 ] Completed eval batch 1 recons_loss 0.371 ms, 1.95 s total -[ 2023-09-21 10:24:41 ] Completed eval batch 1 metrics update 267.569 ms, 2.22 s total -Saving checkpoint at epoch 141 val batch 1 -Epoch 141 val loss: 0.0621 -[ 2023-09-21 10:24:41 ] Completed saving temp checkpoint 551.478 ms, 2.77 s total -[ 2023-09-21 10:24:41 ] Completed replacing temp checkpoint with checkpoint 30.127 ms, 2.80 s total -[ 2023-09-21 10:24:41 ] Completed evaluating generator for epoch 141 63.638 ms, 2.86 s total - - -EPOCH :: 142 - - -[ 2023-09-21 10:24:41 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 142 from training step 0 - -[ 2023-09-21 10:24:44 ] Completed train batch 0 to device 3,074.391 ms, 3.07 s total -[ 2023-09-21 10:24:44 ] Completed train batch 0 generator forward 34.710 ms, 3.11 s total -[ 2023-09-21 10:24:44 ] Completed train batch 0 generator loss: 1.220 77.270 ms, 3.19 s total -[ 2023-09-21 10:24:45 ] Completed train batch 0 generator backward 553.616 ms, 3.74 s total -[ 2023-09-21 10:24:45 ] Completed train batch 0 discriminator loss 0.049 59.881 ms, 3.80 s total -[ 2023-09-21 10:24:45 ] Completed train batch 0 discriminator backward 116.821 ms, 3.92 s total -Epoch [142] Step [0/2], gen_loss: 0.413, disc_loss: 0.016 -[ 2023-09-21 10:24:45 ] Completed train batch 0 metrics update 5.405 ms, 3.92 s total -Saving checkpoint at epoch 142 train batch 0 -[ 2023-09-21 10:24:46 ] Completed saving temp checkpoint 542.465 ms, 4.46 s total -[ 2023-09-21 10:24:46 ] Completed replacing temp checkpoint with checkpoint 32.838 ms, 4.50 s total -[ 2023-09-21 10:24:48 ] Completed train batch 1 to device 1,873.873 ms, 6.37 s total -[ 2023-09-21 10:24:48 ] Completed train batch 1 generator forward 23.035 ms, 6.39 s total -[ 2023-09-21 10:24:48 ] Completed train batch 1 generator loss: 1.268 64.852 ms, 6.46 s total -[ 2023-09-21 10:24:48 ] Completed train batch 1 generator backward 736.655 ms, 7.20 s total -[ 2023-09-21 10:24:49 ] Completed train batch 1 discriminator loss 0.047 57.429 ms, 7.25 s total -[ 2023-09-21 10:24:49 ] Completed train batch 1 discriminator backward 73.512 ms, 7.33 s total -Epoch [142] Step [1/2], gen_loss: 0.413, disc_loss: 0.016 -[ 2023-09-21 10:24:49 ] Completed train batch 1 metrics update 0.481 ms, 7.33 s total -Saving checkpoint at epoch 142 train batch 1 -[ 2023-09-21 10:24:49 ] Completed saving temp checkpoint 537.766 ms, 7.86 s total -[ 2023-09-21 10:24:49 ] Completed replacing temp checkpoint with checkpoint 24.443 ms, 7.89 s total -Epoch [142] :: gen_loss: 0.413, disc_loss: 0.016 -[ 2023-09-21 10:24:49 ] Completed training generator for epoch 142 78.071 ms, 7.97 s total -[ 2023-09-21 10:24:49 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 142 from eval step 0 - -[ 2023-09-21 10:24:50 ] Completed eval batch 0 to device 1,111.343 ms, 1.11 s total -[ 2023-09-21 10:24:50 ] Completed eval batch 0 forward 25.370 ms, 1.14 s total -[ 2023-09-21 10:24:50 ] Completed eval batch 0 recons_loss 0.314 ms, 1.14 s total -[ 2023-09-21 10:24:50 ] Completed eval batch 0 metrics update 0.620 ms, 1.14 s total -Saving checkpoint at epoch 142 val batch 0 -[ 2023-09-21 10:24:51 ] Completed saving temp checkpoint 493.889 ms, 1.63 s total -[ 2023-09-21 10:24:51 ] Completed replacing temp checkpoint with checkpoint 30.930 ms, 1.66 s total -[ 2023-09-21 10:24:51 ] Completed eval batch 1 to device 315.539 ms, 1.98 s total -[ 2023-09-21 10:24:51 ] Completed eval batch 1 forward 21.540 ms, 2.00 s total -[ 2023-09-21 10:24:51 ] Completed eval batch 1 recons_loss 0.346 ms, 2.00 s total -[ 2023-09-21 10:24:51 ] Completed eval batch 1 metrics update 213.833 ms, 2.21 s total -Saving checkpoint at epoch 142 val batch 1 -Epoch 142 val loss: 0.0629 -[ 2023-09-21 10:24:52 ] Completed saving temp checkpoint 531.291 ms, 2.75 s total -[ 2023-09-21 10:24:52 ] Completed replacing temp checkpoint with checkpoint 23.161 ms, 2.77 s total -[ 2023-09-21 10:24:52 ] Completed evaluating generator for epoch 142 60.419 ms, 2.83 s total - - -EPOCH :: 143 - - -[ 2023-09-21 10:24:52 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 143 from training step 0 - -[ 2023-09-21 10:24:55 ] Completed train batch 0 to device 3,116.593 ms, 3.12 s total -[ 2023-09-21 10:24:55 ] Completed train batch 0 generator forward 33.263 ms, 3.15 s total -[ 2023-09-21 10:24:55 ] Completed train batch 0 generator loss: 1.271 66.432 ms, 3.22 s total -[ 2023-09-21 10:24:56 ] Completed train batch 0 generator backward 515.528 ms, 3.73 s total -[ 2023-09-21 10:24:56 ] Completed train batch 0 discriminator loss 0.047 60.127 ms, 3.79 s total -[ 2023-09-21 10:24:56 ] Completed train batch 0 discriminator backward 84.593 ms, 3.88 s total -Epoch [143] Step [0/2], gen_loss: 0.412, disc_loss: 0.015 -[ 2023-09-21 10:24:56 ] Completed train batch 0 metrics update 6.183 ms, 3.88 s total -Saving checkpoint at epoch 143 train batch 0 -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 10:42:32 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 10:42:32 ] Completed importing Timer 0.021 ms, 0.00 s total -| distributed init (rank 5): env:// -| distributed init (rank 3): env:// -| distributed init (rank 4): env:// -| distributed init (rank 2): env:// -| distributed init (rank 1): env:// -[ 2023-09-21 10:42:40 ] Completed importing everything else 8,772.280 ms, 8.77 s total -| distributed init (rank 0): env:// -[ 2023-09-21 10:42:44 ] Completed preliminaries 4,121.876 ms, 12.89 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 10:42:44 ] Completed build datasets 10.792 ms, 12.90 s total -[ 2023-09-21 10:42:44 ] Completed build samplers 0.066 ms, 12.91 s total -[ 2023-09-21 10:42:44 ] Completed build dataloaders 0.156 ms, 12.91 s total -[ 2023-09-21 10:42:44 ] Completed generator to device 75.073 ms, 12.98 s total -[ 2023-09-21 10:42:44 ] Completed discriminator to device 7.087 ms, 12.99 s total -[ 2023-09-21 10:42:45 ] Completed loss functions 382.795 ms, 13.37 s total -[ 2023-09-21 10:42:45 ] Completed models prepped for distribution 125.037 ms, 13.50 s total -[ 2023-09-21 10:42:45 ] Completed optimizers 0.650 ms, 13.50 s total -[ 2023-09-21 10:42:45 ] Completed grad scalers 0.033 ms, 13.50 s total -[ 2023-09-21 10:42:45 ] Completed checkpoint retrieval 483.028 ms, 13.98 s total - - -EPOCH :: 142 - - -[ 2023-09-21 10:42:45 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 142 from training step 2 - -Epoch [142] :: gen_loss: 0.413, disc_loss: 0.016 -[ 2023-09-21 10:42:46 ] Completed training generator for epoch 142 138.134 ms, 0.14 s total -[ 2023-09-21 10:42:46 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 142 from eval step 2 - -[ 2023-09-21 10:42:46 ] Completed evaluating generator for epoch 142 125.944 ms, 0.13 s total - - -EPOCH :: 143 - - -[ 2023-09-21 10:42:46 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 143 from training step 0 - -[ 2023-09-21 10:42:49 ] Completed train batch 0 to device 3,561.110 ms, 3.56 s total -[ 2023-09-21 10:42:51 ] Completed train batch 0 generator forward 1,416.851 ms, 4.98 s total -[ 2023-09-21 10:42:51 ] Completed train batch 0 generator loss: 1.289 85.950 ms, 5.06 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 10:42:52 ] Completed train batch 0 generator backward 1,309.943 ms, 6.37 s total -[ 2023-09-21 10:42:52 ] Completed train batch 0 discriminator loss 0.046 56.159 ms, 6.43 s total -[ 2023-09-21 10:42:52 ] Completed train batch 0 discriminator backward 109.364 ms, 6.54 s total -Epoch [143] Step [0/2], gen_loss: 0.414, disc_loss: 0.015 -[ 2023-09-21 10:42:52 ] Completed train batch 0 metrics update 6.102 ms, 6.55 s total -Saving checkpoint at epoch 143 train batch 0 -[ 2023-09-21 10:42:53 ] Completed saving temp checkpoint 417.668 ms, 6.96 s total -[ 2023-09-21 10:42:53 ] Completed replacing temp checkpoint with checkpoint 122.121 ms, 7.09 s total -[ 2023-09-21 10:42:53 ] Completed train batch 1 to device 21.261 ms, 7.11 s total -[ 2023-09-21 10:42:53 ] Completed train batch 1 generator forward 36.468 ms, 7.14 s total -[ 2023-09-21 10:42:53 ] Completed train batch 1 generator loss: 1.170 61.987 ms, 7.20 s total -[ 2023-09-21 10:42:55 ] Completed train batch 1 generator backward 1,555.447 ms, 8.76 s total -[ 2023-09-21 10:42:55 ] Completed train batch 1 discriminator loss 0.045 55.824 ms, 8.82 s total -[ 2023-09-21 10:42:55 ] Completed train batch 1 discriminator backward 81.921 ms, 8.90 s total -Epoch [143] Step [1/2], gen_loss: 0.403, disc_loss: 0.015 -[ 2023-09-21 10:42:55 ] Completed train batch 1 metrics update 2.155 ms, 8.90 s total -Saving checkpoint at epoch 143 train batch 1 -[ 2023-09-21 10:42:55 ] Completed saving temp checkpoint 457.607 ms, 9.36 s total -[ 2023-09-21 10:42:55 ] Completed replacing temp checkpoint with checkpoint 31.119 ms, 9.39 s total -Epoch [143] :: gen_loss: 0.409, disc_loss: 0.015 -[ 2023-09-21 10:42:55 ] Completed training generator for epoch 143 54.076 ms, 9.44 s total -[ 2023-09-21 10:42:55 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 143 from eval step 0 - -[ 2023-09-21 10:42:56 ] Completed eval batch 0 to device 1,059.862 ms, 1.06 s total -[ 2023-09-21 10:42:56 ] Completed eval batch 0 forward 29.835 ms, 1.09 s total -[ 2023-09-21 10:42:56 ] Completed eval batch 0 recons_loss 0.266 ms, 1.09 s total -[ 2023-09-21 10:42:56 ] Completed eval batch 0 metrics update 0.539 ms, 1.09 s total -Saving checkpoint at epoch 143 val batch 0 -[ 2023-09-21 10:42:57 ] Completed saving temp checkpoint 432.654 ms, 1.52 s total -[ 2023-09-21 10:42:57 ] Completed replacing temp checkpoint with checkpoint 25.877 ms, 1.55 s total -[ 2023-09-21 10:42:57 ] Completed eval batch 1 to device 401.085 ms, 1.95 s total -[ 2023-09-21 10:42:57 ] Completed eval batch 1 forward 39.608 ms, 1.99 s total -[ 2023-09-21 10:42:57 ] Completed eval batch 1 recons_loss 0.551 ms, 1.99 s total -[ 2023-09-21 10:42:57 ] Completed eval batch 1 metrics update 271.050 ms, 2.26 s total -Saving checkpoint at epoch 143 val batch 1 -Epoch 143 val loss: 0.0686 -[ 2023-09-21 10:42:58 ] Completed saving temp checkpoint 441.700 ms, 2.70 s total -[ 2023-09-21 10:42:58 ] Completed replacing temp checkpoint with checkpoint 33.883 ms, 2.74 s total -[ 2023-09-21 10:42:58 ] Completed evaluating generator for epoch 143 69.174 ms, 2.81 s total - - -EPOCH :: 144 - - -[ 2023-09-21 10:42:58 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 144 from training step 0 - -[ 2023-09-21 10:43:01 ] Completed train batch 0 to device 3,023.098 ms, 3.02 s total -[ 2023-09-21 10:43:01 ] Completed train batch 0 generator forward 24.729 ms, 3.05 s total -[ 2023-09-21 10:43:01 ] Completed train batch 0 generator loss: 1.255 63.634 ms, 3.11 s total -[ 2023-09-21 10:43:02 ] Completed train batch 0 generator backward 691.951 ms, 3.80 s total -[ 2023-09-21 10:43:02 ] Completed train batch 0 discriminator loss 0.047 56.217 ms, 3.86 s total -[ 2023-09-21 10:43:02 ] Completed train batch 0 discriminator backward 94.420 ms, 3.95 s total -Epoch [144] Step [0/2], gen_loss: 0.412, disc_loss: 0.016 -[ 2023-09-21 10:43:02 ] Completed train batch 0 metrics update 6.544 ms, 3.96 s total -Saving checkpoint at epoch 144 train batch 0 -[ 2023-09-21 10:43:02 ] Completed saving temp checkpoint 502.617 ms, 4.46 s total -[ 2023-09-21 10:43:02 ] Completed replacing temp checkpoint with checkpoint 19.565 ms, 4.48 s total -[ 2023-09-21 10:43:04 ] Completed train batch 1 to device 1,288.468 ms, 5.77 s total -[ 2023-09-21 10:43:04 ] Completed train batch 1 generator forward 40.509 ms, 5.81 s total -[ 2023-09-21 10:43:04 ] Completed train batch 1 generator loss: 1.249 62.298 ms, 5.87 s total -[ 2023-09-21 10:43:05 ] Completed train batch 1 generator backward 1,550.041 ms, 7.42 s total -[ 2023-09-21 10:43:05 ] Completed train batch 1 discriminator loss 0.046 55.914 ms, 7.48 s total -[ 2023-09-21 10:43:06 ] Completed train batch 1 discriminator backward 73.898 ms, 7.55 s total -Epoch [144] Step [1/2], gen_loss: 0.415, disc_loss: 0.015 -[ 2023-09-21 10:43:06 ] Completed train batch 1 metrics update 5.344 ms, 7.56 s total -Saving checkpoint at epoch 144 train batch 1 -[ 2023-09-21 10:43:06 ] Completed saving temp checkpoint 700.197 ms, 8.26 s total -[ 2023-09-21 10:43:06 ] Completed replacing temp checkpoint with checkpoint 38.364 ms, 8.30 s total -Epoch [144] :: gen_loss: 0.414, disc_loss: 0.016 -[ 2023-09-21 10:43:06 ] Completed training generator for epoch 144 70.147 ms, 8.37 s total -[ 2023-09-21 10:43:06 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 144 from eval step 0 - -[ 2023-09-21 10:43:08 ] Completed eval batch 0 to device 1,145.354 ms, 1.15 s total -[ 2023-09-21 10:43:08 ] Completed eval batch 0 forward 42.188 ms, 1.19 s total -[ 2023-09-21 10:43:08 ] Completed eval batch 0 recons_loss 0.305 ms, 1.19 s total -[ 2023-09-21 10:43:08 ] Completed eval batch 0 metrics update 0.635 ms, 1.19 s total -Saving checkpoint at epoch 144 val batch 0 -[ 2023-09-21 10:43:08 ] Completed saving temp checkpoint 500.049 ms, 1.69 s total -[ 2023-09-21 10:43:08 ] Completed replacing temp checkpoint with checkpoint 38.193 ms, 1.73 s total -[ 2023-09-21 10:43:08 ] Completed eval batch 1 to device 219.799 ms, 1.95 s total -[ 2023-09-21 10:43:08 ] Completed eval batch 1 forward 38.100 ms, 1.98 s total -[ 2023-09-21 10:43:08 ] Completed eval batch 1 recons_loss 0.333 ms, 1.98 s total -[ 2023-09-21 10:43:08 ] Completed eval batch 1 metrics update 0.307 ms, 1.99 s total -Saving checkpoint at epoch 144 val batch 1 -Epoch 144 val loss: 0.0699 -[ 2023-09-21 10:43:09 ] Completed saving temp checkpoint 525.356 ms, 2.51 s total -[ 2023-09-21 10:43:09 ] Completed replacing temp checkpoint with checkpoint 26.581 ms, 2.54 s total -[ 2023-09-21 10:43:09 ] Completed evaluating generator for epoch 144 71.446 ms, 2.61 s total - - -EPOCH :: 145 - - -[ 2023-09-21 10:43:09 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 145 from training step 0 - -[ 2023-09-21 10:43:12 ] Completed train batch 0 to device 2,902.633 ms, 2.90 s total -[ 2023-09-21 10:43:12 ] Completed train batch 0 generator forward 25.954 ms, 2.93 s total -[ 2023-09-21 10:43:12 ] Completed train batch 0 generator loss: 1.240 65.788 ms, 2.99 s total -[ 2023-09-21 10:43:13 ] Completed train batch 0 generator backward 731.515 ms, 3.73 s total -[ 2023-09-21 10:43:13 ] Completed train batch 0 discriminator loss 0.046 57.425 ms, 3.78 s total -[ 2023-09-21 10:43:13 ] Completed train batch 0 discriminator backward 106.792 ms, 3.89 s total -Epoch [145] Step [0/2], gen_loss: 0.412, disc_loss: 0.015 -[ 2023-09-21 10:43:13 ] Completed train batch 0 metrics update 7.017 ms, 3.90 s total -Saving checkpoint at epoch 145 train batch 0 -[ 2023-09-21 10:43:13 ] Completed saving temp checkpoint 521.419 ms, 4.42 s total -[ 2023-09-21 10:43:13 ] Completed replacing temp checkpoint with checkpoint 30.298 ms, 4.45 s total -[ 2023-09-21 10:43:15 ] Completed train batch 1 to device 1,156.304 ms, 5.61 s total -[ 2023-09-21 10:43:15 ] Completed train batch 1 generator forward 41.118 ms, 5.65 s total -[ 2023-09-21 10:43:15 ] Completed train batch 1 generator loss: 1.202 63.048 ms, 5.71 s total -[ 2023-09-21 10:43:16 ] Completed train batch 1 generator backward 1,419.335 ms, 7.13 s total -[ 2023-09-21 10:43:16 ] Completed train batch 1 discriminator loss 0.048 55.868 ms, 7.18 s total -[ 2023-09-21 10:43:16 ] Completed train batch 1 discriminator backward 75.940 ms, 7.26 s total -Epoch [145] Step [1/2], gen_loss: 0.408, disc_loss: 0.016 -[ 2023-09-21 10:43:16 ] Completed train batch 1 metrics update 1.055 ms, 7.26 s total -Saving checkpoint at epoch 145 train batch 1 -[ 2023-09-21 10:43:17 ] Completed saving temp checkpoint 546.738 ms, 7.81 s total -[ 2023-09-21 10:43:17 ] Completed replacing temp checkpoint with checkpoint 23.977 ms, 7.83 s total -Epoch [145] :: gen_loss: 0.410, disc_loss: 0.016 -[ 2023-09-21 10:43:17 ] Completed training generator for epoch 145 72.989 ms, 7.91 s total -[ 2023-09-21 10:43:17 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 145 from eval step 0 - -[ 2023-09-21 10:43:18 ] Completed eval batch 0 to device 1,074.765 ms, 1.07 s total -[ 2023-09-21 10:43:18 ] Completed eval batch 0 forward 24.918 ms, 1.10 s total -[ 2023-09-21 10:43:18 ] Completed eval batch 0 recons_loss 0.364 ms, 1.10 s total -[ 2023-09-21 10:43:18 ] Completed eval batch 0 metrics update 0.719 ms, 1.10 s total -Saving checkpoint at epoch 145 val batch 0 -[ 2023-09-21 10:43:19 ] Completed saving temp checkpoint 517.401 ms, 1.62 s total -[ 2023-09-21 10:43:19 ] Completed replacing temp checkpoint with checkpoint 38.024 ms, 1.66 s total -[ 2023-09-21 10:43:19 ] Completed eval batch 1 to device 271.178 ms, 1.93 s total -[ 2023-09-21 10:43:19 ] Completed eval batch 1 forward 39.082 ms, 1.97 s total -[ 2023-09-21 10:43:19 ] Completed eval batch 1 recons_loss 0.337 ms, 1.97 s total -[ 2023-09-21 10:43:19 ] Completed eval batch 1 metrics update 115.254 ms, 2.08 s total -Saving checkpoint at epoch 145 val batch 1 -Epoch 145 val loss: 0.0603 -[ 2023-09-21 10:43:20 ] Completed saving temp checkpoint 557.422 ms, 2.64 s total -[ 2023-09-21 10:43:20 ] Completed replacing temp checkpoint with checkpoint 27.651 ms, 2.67 s total -[ 2023-09-21 10:43:20 ] Completed evaluating generator for epoch 145 55.608 ms, 2.72 s total - - -EPOCH :: 146 - - -[ 2023-09-21 10:43:20 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 146 from training step 0 - -[ 2023-09-21 10:43:22 ] Completed train batch 0 to device 2,846.205 ms, 2.85 s total -[ 2023-09-21 10:43:22 ] Completed train batch 0 generator forward 25.253 ms, 2.87 s total -[ 2023-09-21 10:43:23 ] Completed train batch 0 generator loss: 1.243 63.843 ms, 2.94 s total -[ 2023-09-21 10:43:23 ] Completed train batch 0 generator backward 830.507 ms, 3.77 s total -[ 2023-09-21 10:43:23 ] Completed train batch 0 discriminator loss 0.050 55.861 ms, 3.82 s total -[ 2023-09-21 10:43:24 ] Completed train batch 0 discriminator backward 103.338 ms, 3.93 s total -Epoch [146] Step [0/2], gen_loss: 0.410, disc_loss: 0.017 -[ 2023-09-21 10:43:24 ] Completed train batch 0 metrics update 9.692 ms, 3.93 s total -Saving checkpoint at epoch 146 train batch 0 -[ 2023-09-21 10:43:24 ] Completed saving temp checkpoint 521.883 ms, 4.46 s total -[ 2023-09-21 10:43:24 ] Completed replacing temp checkpoint with checkpoint 35.482 ms, 4.49 s total -[ 2023-09-21 10:43:25 ] Completed train batch 1 to device 928.141 ms, 5.42 s total -[ 2023-09-21 10:43:25 ] Completed train batch 1 generator forward 41.860 ms, 5.46 s total -[ 2023-09-21 10:43:25 ] Completed train batch 1 generator loss: 1.250 74.961 ms, 5.54 s total -[ 2023-09-21 10:43:27 ] Completed train batch 1 generator backward 1,750.573 ms, 7.29 s total -[ 2023-09-21 10:43:27 ] Completed train batch 1 discriminator loss 0.048 55.773 ms, 7.34 s total -[ 2023-09-21 10:43:27 ] Completed train batch 1 discriminator backward 76.781 ms, 7.42 s total -Epoch [146] Step [1/2], gen_loss: 0.411, disc_loss: 0.016 -[ 2023-09-21 10:43:27 ] Completed train batch 1 metrics update 5.492 ms, 7.43 s total -Saving checkpoint at epoch 146 train batch 1 -[ 2023-09-21 10:43:28 ] Completed saving temp checkpoint 548.950 ms, 7.97 s total -[ 2023-09-21 10:43:28 ] Completed replacing temp checkpoint with checkpoint 25.355 ms, 8.00 s total -Epoch [146] :: gen_loss: 0.410, disc_loss: 0.016 -[ 2023-09-21 10:43:28 ] Completed training generator for epoch 146 70.477 ms, 8.07 s total -[ 2023-09-21 10:43:28 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 146 from eval step 0 - -[ 2023-09-21 10:43:29 ] Completed eval batch 0 to device 1,032.994 ms, 1.03 s total -[ 2023-09-21 10:43:29 ] Completed eval batch 0 forward 42.508 ms, 1.08 s total -[ 2023-09-21 10:43:29 ] Completed eval batch 0 recons_loss 0.342 ms, 1.08 s total -[ 2023-09-21 10:43:29 ] Completed eval batch 0 metrics update 0.730 ms, 1.08 s total -Saving checkpoint at epoch 146 val batch 0 -[ 2023-09-21 10:43:29 ] Completed saving temp checkpoint 501.648 ms, 1.58 s total -[ 2023-09-21 10:43:29 ] Completed replacing temp checkpoint with checkpoint 40.272 ms, 1.62 s total -[ 2023-09-21 10:43:30 ] Completed eval batch 1 to device 295.863 ms, 1.91 s total -[ 2023-09-21 10:43:30 ] Completed eval batch 1 forward 39.556 ms, 1.95 s total -[ 2023-09-21 10:43:30 ] Completed eval batch 1 recons_loss 0.550 ms, 1.95 s total -[ 2023-09-21 10:43:30 ] Completed eval batch 1 metrics update 159.921 ms, 2.11 s total -Saving checkpoint at epoch 146 val batch 1 -Epoch 146 val loss: 0.0583 -[ 2023-09-21 10:43:30 ] Completed saving temp checkpoint 410.149 ms, 2.52 s total -[ 2023-09-21 10:43:30 ] Completed replacing temp checkpoint with checkpoint 28.198 ms, 2.55 s total -[ 2023-09-21 10:43:30 ] Completed evaluating generator for epoch 146 65.376 ms, 2.62 s total - - -EPOCH :: 147 - - -[ 2023-09-21 10:43:30 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 147 from training step 0 - -[ 2023-09-21 10:43:33 ] Completed train batch 0 to device 2,976.152 ms, 2.98 s total -[ 2023-09-21 10:43:33 ] Completed train batch 0 generator forward 26.329 ms, 3.00 s total -[ 2023-09-21 10:43:33 ] Completed train batch 0 generator loss: 1.264 65.016 ms, 3.07 s total -[ 2023-09-21 10:43:34 ] Completed train batch 0 generator backward 733.242 ms, 3.80 s total -[ 2023-09-21 10:43:34 ] Completed train batch 0 discriminator loss 0.048 56.448 ms, 3.86 s total -[ 2023-09-21 10:43:34 ] Completed train batch 0 discriminator backward 112.700 ms, 3.97 s total -Epoch [147] Step [0/2], gen_loss: 0.410, disc_loss: 0.016 -[ 2023-09-21 10:43:34 ] Completed train batch 0 metrics update 4.522 ms, 3.97 s total -Saving checkpoint at epoch 147 train batch 0 -[ 2023-09-21 10:43:35 ] Completed saving temp checkpoint 432.278 ms, 4.41 s total -[ 2023-09-21 10:43:35 ] Completed replacing temp checkpoint with checkpoint 33.685 ms, 4.44 s total -[ 2023-09-21 10:43:36 ] Completed train batch 1 to device 1,226.178 ms, 5.67 s total -[ 2023-09-21 10:43:36 ] Completed train batch 1 generator forward 22.381 ms, 5.69 s total -[ 2023-09-21 10:43:36 ] Completed train batch 1 generator loss: 1.410 63.359 ms, 5.75 s total -[ 2023-09-21 10:43:37 ] Completed train batch 1 generator backward 1,257.427 ms, 7.01 s total -[ 2023-09-21 10:43:37 ] Completed train batch 1 discriminator loss 0.058 55.626 ms, 7.07 s total -[ 2023-09-21 10:43:37 ] Completed train batch 1 discriminator backward 79.292 ms, 7.14 s total -Epoch [147] Step [1/2], gen_loss: 0.431, disc_loss: 0.020 -[ 2023-09-21 10:43:37 ] Completed train batch 1 metrics update 1.551 ms, 7.15 s total -Saving checkpoint at epoch 147 train batch 1 -[ 2023-09-21 10:43:38 ] Completed saving temp checkpoint 497.113 ms, 7.64 s total -[ 2023-09-21 10:43:38 ] Completed replacing temp checkpoint with checkpoint 30.985 ms, 7.67 s total -Epoch [147] :: gen_loss: 0.421, disc_loss: 0.018 -[ 2023-09-21 10:43:38 ] Completed training generator for epoch 147 82.212 ms, 7.76 s total -[ 2023-09-21 10:43:38 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 147 from eval step 0 - -[ 2023-09-21 10:43:39 ] Completed eval batch 0 to device 1,045.995 ms, 1.05 s total -[ 2023-09-21 10:43:39 ] Completed eval batch 0 forward 42.478 ms, 1.09 s total -[ 2023-09-21 10:43:39 ] Completed eval batch 0 recons_loss 0.314 ms, 1.09 s total -[ 2023-09-21 10:43:39 ] Completed eval batch 0 metrics update 0.593 ms, 1.09 s total -Saving checkpoint at epoch 147 val batch 0 -[ 2023-09-21 10:43:40 ] Completed saving temp checkpoint 409.008 ms, 1.50 s total -[ 2023-09-21 10:43:40 ] Completed replacing temp checkpoint with checkpoint 24.394 ms, 1.52 s total -[ 2023-09-21 10:43:40 ] Completed eval batch 1 to device 392.426 ms, 1.92 s total -[ 2023-09-21 10:43:40 ] Completed eval batch 1 forward 38.755 ms, 1.95 s total -[ 2023-09-21 10:43:40 ] Completed eval batch 1 recons_loss 0.329 ms, 1.95 s total -[ 2023-09-21 10:43:40 ] Completed eval batch 1 metrics update 206.840 ms, 2.16 s total -Saving checkpoint at epoch 147 val batch 1 -Epoch 147 val loss: 0.0547 -[ 2023-09-21 10:43:41 ] Completed saving temp checkpoint 463.453 ms, 2.62 s total -[ 2023-09-21 10:43:41 ] Completed replacing temp checkpoint with checkpoint 31.275 ms, 2.66 s total -[ 2023-09-21 10:43:41 ] Completed evaluating generator for epoch 147 53.828 ms, 2.71 s total - - -EPOCH :: 148 - - -[ 2023-09-21 10:43:41 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 148 from training step 0 - -[ 2023-09-21 10:43:44 ] Completed train batch 0 to device 3,051.711 ms, 3.05 s total -[ 2023-09-21 10:43:44 ] Completed train batch 0 generator forward 30.010 ms, 3.08 s total -[ 2023-09-21 10:43:44 ] Completed train batch 0 generator loss: 1.244 63.977 ms, 3.15 s total -[ 2023-09-21 10:43:45 ] Completed train batch 0 generator backward 678.881 ms, 3.82 s total -[ 2023-09-21 10:43:45 ] Completed train batch 0 discriminator loss 0.065 57.858 ms, 3.88 s total -[ 2023-09-21 10:43:45 ] Completed train batch 0 discriminator backward 104.584 ms, 3.99 s total -Epoch [148] Step [0/2], gen_loss: 0.416, disc_loss: 0.022 -[ 2023-09-21 10:43:45 ] Completed train batch 0 metrics update 4.926 ms, 3.99 s total -Saving checkpoint at epoch 148 train batch 0 -[ 2023-09-21 10:43:45 ] Completed saving temp checkpoint 521.848 ms, 4.51 s total -[ 2023-09-21 10:43:45 ] Completed replacing temp checkpoint with checkpoint 34.304 ms, 4.55 s total -[ 2023-09-21 10:43:47 ] Completed train batch 1 to device 1,304.602 ms, 5.85 s total -[ 2023-09-21 10:43:47 ] Completed train batch 1 generator forward 22.134 ms, 5.87 s total -[ 2023-09-21 10:43:47 ] Completed train batch 1 generator loss: 1.296 63.266 ms, 5.94 s total -[ 2023-09-21 10:43:48 ] Completed train batch 1 generator backward 1,258.418 ms, 7.20 s total -[ 2023-09-21 10:43:48 ] Completed train batch 1 discriminator loss 0.063 56.103 ms, 7.25 s total -[ 2023-09-21 10:43:48 ] Completed train batch 1 discriminator backward 80.426 ms, 7.33 s total -Epoch [148] Step [1/2], gen_loss: 0.428, disc_loss: 0.020 -[ 2023-09-21 10:43:48 ] Completed train batch 1 metrics update 1.536 ms, 7.33 s total -Saving checkpoint at epoch 148 train batch 1 -[ 2023-09-21 10:43:49 ] Completed saving temp checkpoint 631.191 ms, 7.97 s total -[ 2023-09-21 10:43:49 ] Completed replacing temp checkpoint with checkpoint 32.902 ms, 8.00 s total -Epoch [148] :: gen_loss: 0.422, disc_loss: 0.021 -[ 2023-09-21 10:43:49 ] Completed training generator for epoch 148 73.655 ms, 8.07 s total -[ 2023-09-21 10:43:49 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 148 from eval step 0 - -[ 2023-09-21 10:43:50 ] Completed eval batch 0 to device 1,084.074 ms, 1.08 s total -[ 2023-09-21 10:43:50 ] Completed eval batch 0 forward 25.775 ms, 1.11 s total -[ 2023-09-21 10:43:50 ] Completed eval batch 0 recons_loss 0.332 ms, 1.11 s total -[ 2023-09-21 10:43:50 ] Completed eval batch 0 metrics update 0.640 ms, 1.11 s total -Saving checkpoint at epoch 148 val batch 0 -[ 2023-09-21 10:43:51 ] Completed saving temp checkpoint 563.330 ms, 1.67 s total -[ 2023-09-21 10:43:51 ] Completed replacing temp checkpoint with checkpoint 25.021 ms, 1.70 s total -[ 2023-09-21 10:43:51 ] Completed eval batch 1 to device 372.865 ms, 2.07 s total -[ 2023-09-21 10:43:51 ] Completed eval batch 1 forward 21.382 ms, 2.09 s total -[ 2023-09-21 10:43:51 ] Completed eval batch 1 recons_loss 0.400 ms, 2.09 s total -[ 2023-09-21 10:43:51 ] Completed eval batch 1 metrics update 0.328 ms, 2.09 s total -Saving checkpoint at epoch 148 val batch 1 -Epoch 148 val loss: 0.0554 -[ 2023-09-21 10:43:52 ] Completed saving temp checkpoint 612.725 ms, 2.71 s total -[ 2023-09-21 10:43:52 ] Completed replacing temp checkpoint with checkpoint 34.388 ms, 2.74 s total -[ 2023-09-21 10:43:52 ] Completed evaluating generator for epoch 148 57.573 ms, 2.80 s total - - -EPOCH :: 149 - - -[ 2023-09-21 10:43:52 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 149 from training step 0 - -[ 2023-09-21 10:43:55 ] Completed train batch 0 to device 2,906.653 ms, 2.91 s total -[ 2023-09-21 10:43:55 ] Completed train batch 0 generator forward 44.198 ms, 2.95 s total -[ 2023-09-21 10:43:55 ] Completed train batch 0 generator loss: 1.253 64.538 ms, 3.02 s total -[ 2023-09-21 10:43:55 ] Completed train batch 0 generator backward 558.542 ms, 3.57 s total -[ 2023-09-21 10:43:55 ] Completed train batch 0 discriminator loss 0.057 56.515 ms, 3.63 s total -[ 2023-09-21 10:43:55 ] Completed train batch 0 discriminator backward 116.663 ms, 3.75 s total -Epoch [149] Step [0/2], gen_loss: 0.425, disc_loss: 0.019 -[ 2023-09-21 10:43:55 ] Completed train batch 0 metrics update 13.104 ms, 3.76 s total -Saving checkpoint at epoch 149 train batch 0 -[ 2023-09-21 10:43:56 ] Completed saving temp checkpoint 597.449 ms, 4.36 s total -[ 2023-09-21 10:43:56 ] Completed replacing temp checkpoint with checkpoint 34.696 ms, 4.39 s total -[ 2023-09-21 10:43:57 ] Completed train batch 1 to device 1,217.597 ms, 5.61 s total -[ 2023-09-21 10:43:57 ] Completed train batch 1 generator forward 39.660 ms, 5.65 s total -[ 2023-09-21 10:43:57 ] Completed train batch 1 generator loss: 1.286 62.222 ms, 5.71 s total -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 10:57:25 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 10:57:25 ] Completed importing Timer 0.022 ms, 0.00 s total -[ 2023-09-21 10:57:30 ] Completed importing everything else 4,588.223 ms, 4.59 s total -| distributed init (rank 0): env:// -| distributed init (rank 2): env:// -| distributed init (rank 4): env:// -| distributed init (rank 5): env:// -| distributed init (rank 3): env:// -| distributed init (rank 1): env:// -[ 2023-09-21 10:57:37 ] Completed preliminaries 7,529.043 ms, 12.12 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 10:57:37 ] Completed build datasets 11.615 ms, 12.13 s total -[ 2023-09-21 10:57:37 ] Completed build samplers 0.043 ms, 12.13 s total -[ 2023-09-21 10:57:37 ] Completed build dataloaders 0.127 ms, 12.13 s total -[ 2023-09-21 10:57:37 ] Completed generator to device 92.414 ms, 12.22 s total -[ 2023-09-21 10:57:37 ] Completed discriminator to device 6.997 ms, 12.23 s total -[ 2023-09-21 10:57:38 ] Completed loss functions 403.100 ms, 12.63 s total -[ 2023-09-21 10:57:38 ] Completed models prepped for distribution 105.794 ms, 12.74 s total -[ 2023-09-21 10:57:38 ] Completed optimizers 0.694 ms, 12.74 s total -[ 2023-09-21 10:57:38 ] Completed grad scalers 0.030 ms, 12.74 s total -[ 2023-09-21 10:57:38 ] Completed checkpoint retrieval 460.245 ms, 13.20 s total - - -EPOCH :: 149 - - -[ 2023-09-21 10:57:38 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 149 from training step 1 - -[ 2023-09-21 10:57:42 ] Completed train batch 1 to device 3,258.652 ms, 3.26 s total -[ 2023-09-21 10:57:43 ] Completed train batch 1 generator forward 1,106.073 ms, 4.36 s total -[ 2023-09-21 10:57:43 ] Completed train batch 1 generator loss: 1.210 79.032 ms, 4.44 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 10:57:45 ] Completed train batch 1 generator backward 1,941.900 ms, 6.39 s total -[ 2023-09-21 10:57:45 ] Completed train batch 1 discriminator loss 0.053 56.121 ms, 6.44 s total -[ 2023-09-21 10:57:45 ] Completed train batch 1 discriminator backward 85.566 ms, 6.53 s total -Epoch [149] Step [1/2], gen_loss: 0.418, disc_loss: 0.018 -[ 2023-09-21 10:57:45 ] Completed train batch 1 metrics update 5.338 ms, 6.53 s total -Saving checkpoint at epoch 149 train batch 1 -[ 2023-09-21 10:57:45 ] Completed saving temp checkpoint 462.524 ms, 7.00 s total -[ 2023-09-21 10:57:45 ] Completed replacing temp checkpoint with checkpoint 118.984 ms, 7.11 s total -Epoch [149] :: gen_loss: 0.422, disc_loss: 0.019 -[ 2023-09-21 10:57:45 ] Completed training generator for epoch 149 56.589 ms, 7.17 s total -[ 2023-09-21 10:57:45 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 149 from eval step 0 - -[ 2023-09-21 10:57:47 ] Completed eval batch 0 to device 1,103.730 ms, 1.10 s total -[ 2023-09-21 10:57:47 ] Completed eval batch 0 forward 33.267 ms, 1.14 s total -[ 2023-09-21 10:57:47 ] Completed eval batch 0 recons_loss 0.426 ms, 1.14 s total -[ 2023-09-21 10:57:47 ] Completed eval batch 0 metrics update 0.746 ms, 1.14 s total -Saving checkpoint at epoch 149 val batch 0 -[ 2023-09-21 10:57:47 ] Completed saving temp checkpoint 439.329 ms, 1.58 s total -[ 2023-09-21 10:57:47 ] Completed replacing temp checkpoint with checkpoint 31.009 ms, 1.61 s total -[ 2023-09-21 10:57:47 ] Completed eval batch 1 to device 405.169 ms, 2.01 s total -[ 2023-09-21 10:57:47 ] Completed eval batch 1 forward 21.624 ms, 2.04 s total -[ 2023-09-21 10:57:47 ] Completed eval batch 1 recons_loss 0.349 ms, 2.04 s total -[ 2023-09-21 10:57:48 ] Completed eval batch 1 metrics update 171.412 ms, 2.21 s total -Saving checkpoint at epoch 149 val batch 1 -Epoch 149 val loss: 0.0621 -[ 2023-09-21 10:57:48 ] Completed saving temp checkpoint 453.253 ms, 2.66 s total -[ 2023-09-21 10:57:48 ] Completed replacing temp checkpoint with checkpoint 19.542 ms, 2.68 s total -[ 2023-09-21 10:57:48 ] Completed evaluating generator for epoch 149 60.507 ms, 2.74 s total - - -EPOCH :: 150 - - -[ 2023-09-21 10:57:48 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 150 from training step 0 - -[ 2023-09-21 10:57:51 ] Completed train batch 0 to device 2,883.296 ms, 2.88 s total -[ 2023-09-21 10:57:51 ] Completed train batch 0 generator forward 25.535 ms, 2.91 s total -[ 2023-09-21 10:57:51 ] Completed train batch 0 generator loss: 1.257 65.573 ms, 2.97 s total -[ 2023-09-21 10:57:52 ] Completed train batch 0 generator backward 930.514 ms, 3.90 s total -[ 2023-09-21 10:57:52 ] Completed train batch 0 discriminator loss 0.055 59.089 ms, 3.96 s total -[ 2023-09-21 10:57:52 ] Completed train batch 0 discriminator backward 104.074 ms, 4.07 s total -Epoch [150] Step [0/2], gen_loss: 0.418, disc_loss: 0.018 -[ 2023-09-21 10:57:52 ] Completed train batch 0 metrics update 5.988 ms, 4.07 s total -Saving checkpoint at epoch 150 train batch 0 -[ 2023-09-21 10:57:53 ] Completed saving temp checkpoint 458.210 ms, 4.53 s total -[ 2023-09-21 10:57:53 ] Completed replacing temp checkpoint with checkpoint 19.872 ms, 4.55 s total -[ 2023-09-21 10:57:54 ] Completed train batch 1 to device 1,534.312 ms, 6.09 s total -[ 2023-09-21 10:57:54 ] Completed train batch 1 generator forward 26.624 ms, 6.11 s total -[ 2023-09-21 10:57:54 ] Completed train batch 1 generator loss: 1.207 64.620 ms, 6.18 s total -[ 2023-09-21 10:57:56 ] Completed train batch 1 generator backward 1,307.292 ms, 7.49 s total -[ 2023-09-21 10:57:56 ] Completed train batch 1 discriminator loss 0.053 57.185 ms, 7.54 s total -[ 2023-09-21 10:57:56 ] Completed train batch 1 discriminator backward 68.702 ms, 7.61 s total -Epoch [150] Step [1/2], gen_loss: 0.407, disc_loss: 0.018 -[ 2023-09-21 10:57:56 ] Completed train batch 1 metrics update 1.620 ms, 7.61 s total -Saving checkpoint at epoch 150 train batch 1 -[ 2023-09-21 10:57:56 ] Completed saving temp checkpoint 464.388 ms, 8.08 s total -[ 2023-09-21 10:57:56 ] Completed replacing temp checkpoint with checkpoint 24.051 ms, 8.10 s total -Epoch [150] :: gen_loss: 0.413, disc_loss: 0.018 -[ 2023-09-21 10:57:56 ] Completed training generator for epoch 150 50.487 ms, 8.15 s total -[ 2023-09-21 10:57:56 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 150 from eval step 0 - -[ 2023-09-21 10:57:57 ] Completed eval batch 0 to device 1,165.420 ms, 1.17 s total -[ 2023-09-21 10:57:58 ] Completed eval batch 0 forward 28.490 ms, 1.19 s total -[ 2023-09-21 10:57:58 ] Completed eval batch 0 recons_loss 0.318 ms, 1.19 s total -[ 2023-09-21 10:57:58 ] Completed eval batch 0 metrics update 0.579 ms, 1.19 s total -Saving checkpoint at epoch 150 val batch 0 -[ 2023-09-21 10:57:58 ] Completed saving temp checkpoint 440.180 ms, 1.63 s total -[ 2023-09-21 10:57:58 ] Completed replacing temp checkpoint with checkpoint 23.681 ms, 1.66 s total -[ 2023-09-21 10:57:58 ] Completed eval batch 1 to device 471.810 ms, 2.13 s total -[ 2023-09-21 10:57:58 ] Completed eval batch 1 forward 33.146 ms, 2.16 s total -[ 2023-09-21 10:57:58 ] Completed eval batch 1 recons_loss 0.408 ms, 2.16 s total -[ 2023-09-21 10:57:59 ] Completed eval batch 1 metrics update 162.725 ms, 2.33 s total -Saving checkpoint at epoch 150 val batch 1 -Epoch 150 val loss: 0.0683 -[ 2023-09-21 10:57:59 ] Completed saving temp checkpoint 449.929 ms, 2.78 s total -[ 2023-09-21 10:57:59 ] Completed replacing temp checkpoint with checkpoint 26.987 ms, 2.80 s total -[ 2023-09-21 10:57:59 ] Completed evaluating generator for epoch 150 51.532 ms, 2.86 s total - - -EPOCH :: 151 - - -[ 2023-09-21 10:57:59 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 151 from training step 0 - -[ 2023-09-21 10:58:02 ] Completed train batch 0 to device 3,186.571 ms, 3.19 s total -[ 2023-09-21 10:58:02 ] Completed train batch 0 generator forward 25.311 ms, 3.21 s total -[ 2023-09-21 10:58:02 ] Completed train batch 0 generator loss: 1.242 64.836 ms, 3.28 s total -[ 2023-09-21 10:58:03 ] Completed train batch 0 generator backward 591.204 ms, 3.87 s total -[ 2023-09-21 10:58:03 ] Completed train batch 0 discriminator loss 0.053 60.925 ms, 3.93 s total -[ 2023-09-21 10:58:03 ] Completed train batch 0 discriminator backward 75.221 ms, 4.00 s total -Epoch [151] Step [0/2], gen_loss: 0.425, disc_loss: 0.018 -[ 2023-09-21 10:58:03 ] Completed train batch 0 metrics update 5.284 ms, 4.01 s total -Saving checkpoint at epoch 151 train batch 0 -[ 2023-09-21 10:58:04 ] Completed saving temp checkpoint 493.921 ms, 4.50 s total -[ 2023-09-21 10:58:04 ] Completed replacing temp checkpoint with checkpoint 18.737 ms, 4.52 s total -[ 2023-09-21 10:58:05 ] Completed train batch 1 to device 1,675.619 ms, 6.20 s total -[ 2023-09-21 10:58:05 ] Completed train batch 1 generator forward 23.564 ms, 6.22 s total -[ 2023-09-21 10:58:05 ] Completed train batch 1 generator loss: 1.238 64.709 ms, 6.29 s total -[ 2023-09-21 10:58:07 ] Completed train batch 1 generator backward 1,188.705 ms, 7.47 s total -[ 2023-09-21 10:58:07 ] Completed train batch 1 discriminator loss 0.051 56.443 ms, 7.53 s total -[ 2023-09-21 10:58:07 ] Completed train batch 1 discriminator backward 72.627 ms, 7.60 s total -Epoch [151] Step [1/2], gen_loss: 0.415, disc_loss: 0.017 -[ 2023-09-21 10:58:07 ] Completed train batch 1 metrics update 2.375 ms, 7.61 s total -Saving checkpoint at epoch 151 train batch 1 -[ 2023-09-21 10:58:07 ] Completed saving temp checkpoint 487.261 ms, 8.09 s total -[ 2023-09-21 10:58:07 ] Completed replacing temp checkpoint with checkpoint 23.964 ms, 8.12 s total -Epoch [151] :: gen_loss: 0.420, disc_loss: 0.017 -[ 2023-09-21 10:58:07 ] Completed training generator for epoch 151 48.526 ms, 8.17 s total -[ 2023-09-21 10:58:07 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 151 from eval step 0 - -[ 2023-09-21 10:58:08 ] Completed eval batch 0 to device 1,120.697 ms, 1.12 s total -[ 2023-09-21 10:58:08 ] Completed eval batch 0 forward 25.043 ms, 1.15 s total -[ 2023-09-21 10:58:08 ] Completed eval batch 0 recons_loss 0.304 ms, 1.15 s total -[ 2023-09-21 10:58:08 ] Completed eval batch 0 metrics update 0.703 ms, 1.15 s total -Saving checkpoint at epoch 151 val batch 0 -[ 2023-09-21 10:58:09 ] Completed saving temp checkpoint 426.425 ms, 1.57 s total -[ 2023-09-21 10:58:09 ] Completed replacing temp checkpoint with checkpoint 30.617 ms, 1.60 s total -[ 2023-09-21 10:58:09 ] Completed eval batch 1 to device 542.062 ms, 2.15 s total -[ 2023-09-21 10:58:10 ] Completed eval batch 1 forward 39.661 ms, 2.19 s total -[ 2023-09-21 10:58:10 ] Completed eval batch 1 recons_loss 0.340 ms, 2.19 s total -[ 2023-09-21 10:58:10 ] Completed eval batch 1 metrics update 189.996 ms, 2.38 s total -Saving checkpoint at epoch 151 val batch 1 -Epoch 151 val loss: 0.0636 -[ 2023-09-21 10:58:10 ] Completed saving temp checkpoint 430.006 ms, 2.81 s total -[ 2023-09-21 10:58:10 ] Completed replacing temp checkpoint with checkpoint 24.000 ms, 2.83 s total -[ 2023-09-21 10:58:10 ] Completed evaluating generator for epoch 151 47.262 ms, 2.88 s total - - -EPOCH :: 152 - - -[ 2023-09-21 10:58:10 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 152 from training step 0 - -[ 2023-09-21 10:58:13 ] Completed train batch 0 to device 2,811.174 ms, 2.81 s total -[ 2023-09-21 10:58:13 ] Completed train batch 0 generator forward 39.923 ms, 2.85 s total -[ 2023-09-21 10:58:13 ] Completed train batch 0 generator loss: 1.244 64.236 ms, 2.92 s total -[ 2023-09-21 10:58:14 ] Completed train batch 0 generator backward 886.731 ms, 3.80 s total -[ 2023-09-21 10:58:14 ] Completed train batch 0 discriminator loss 0.051 57.405 ms, 3.86 s total -[ 2023-09-21 10:58:14 ] Completed train batch 0 discriminator backward 117.558 ms, 3.98 s total -Epoch [152] Step [0/2], gen_loss: 0.420, disc_loss: 0.017 -[ 2023-09-21 10:58:14 ] Completed train batch 0 metrics update 15.209 ms, 3.99 s total -Saving checkpoint at epoch 152 train batch 0 -[ 2023-09-21 10:58:15 ] Completed saving temp checkpoint 423.455 ms, 4.42 s total -[ 2023-09-21 10:58:15 ] Completed replacing temp checkpoint with checkpoint 16.239 ms, 4.43 s total -[ 2023-09-21 10:58:16 ] Completed train batch 1 to device 1,163.977 ms, 5.60 s total -[ 2023-09-21 10:58:16 ] Completed train batch 1 generator forward 29.892 ms, 5.63 s total -[ 2023-09-21 10:58:16 ] Completed train batch 1 generator loss: 1.347 61.972 ms, 5.69 s total -[ 2023-09-21 10:58:17 ] Completed train batch 1 generator backward 1,444.819 ms, 7.13 s total -[ 2023-09-21 10:58:17 ] Completed train batch 1 discriminator loss 0.050 56.557 ms, 7.19 s total -[ 2023-09-21 10:58:17 ] Completed train batch 1 discriminator backward 65.544 ms, 7.25 s total -Epoch [152] Step [1/2], gen_loss: 0.415, disc_loss: 0.017 -[ 2023-09-21 10:58:17 ] Completed train batch 1 metrics update 5.849 ms, 7.26 s total -Saving checkpoint at epoch 152 train batch 1 -[ 2023-09-21 10:58:18 ] Completed saving temp checkpoint 438.705 ms, 7.70 s total -[ 2023-09-21 10:58:18 ] Completed replacing temp checkpoint with checkpoint 20.777 ms, 7.72 s total -Epoch [152] :: gen_loss: 0.418, disc_loss: 0.017 -[ 2023-09-21 10:58:18 ] Completed training generator for epoch 152 58.139 ms, 7.78 s total -[ 2023-09-21 10:58:18 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 152 from eval step 0 - -[ 2023-09-21 10:58:19 ] Completed eval batch 0 to device 1,130.713 ms, 1.13 s total -[ 2023-09-21 10:58:19 ] Completed eval batch 0 forward 27.057 ms, 1.16 s total -[ 2023-09-21 10:58:19 ] Completed eval batch 0 recons_loss 0.325 ms, 1.16 s total -[ 2023-09-21 10:58:19 ] Completed eval batch 0 metrics update 0.632 ms, 1.16 s total -Saving checkpoint at epoch 152 val batch 0 -[ 2023-09-21 10:58:20 ] Completed saving temp checkpoint 433.626 ms, 1.59 s total -[ 2023-09-21 10:58:20 ] Completed replacing temp checkpoint with checkpoint 24.051 ms, 1.62 s total -[ 2023-09-21 10:58:20 ] Completed eval batch 1 to device 411.947 ms, 2.03 s total -[ 2023-09-21 10:58:20 ] Completed eval batch 1 forward 40.621 ms, 2.07 s total -[ 2023-09-21 10:58:20 ] Completed eval batch 1 recons_loss 0.541 ms, 2.07 s total -[ 2023-09-21 10:58:20 ] Completed eval batch 1 metrics update 197.105 ms, 2.27 s total -Saving checkpoint at epoch 152 val batch 1 -Epoch 152 val loss: 0.0581 -[ 2023-09-21 10:58:21 ] Completed saving temp checkpoint 623.217 ms, 2.89 s total -[ 2023-09-21 10:58:21 ] Completed replacing temp checkpoint with checkpoint 46.093 ms, 2.94 s total -[ 2023-09-21 10:58:21 ] Completed evaluating generator for epoch 152 48.824 ms, 2.98 s total - - -EPOCH :: 153 - - -[ 2023-09-21 10:58:21 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 153 from training step 0 - -[ 2023-09-21 10:58:24 ] Completed train batch 0 to device 3,117.366 ms, 3.12 s total -[ 2023-09-21 10:58:24 ] Completed train batch 0 generator forward 25.600 ms, 3.14 s total -[ 2023-09-21 10:58:24 ] Completed train batch 0 generator loss: 1.202 65.402 ms, 3.21 s total -[ 2023-09-21 10:58:25 ] Completed train batch 0 generator backward 507.061 ms, 3.72 s total -[ 2023-09-21 10:58:25 ] Completed train batch 0 discriminator loss 0.052 64.424 ms, 3.78 s total -[ 2023-09-21 10:58:25 ] Completed train batch 0 discriminator backward 96.414 ms, 3.88 s total -Epoch [153] Step [0/2], gen_loss: 0.411, disc_loss: 0.017 -[ 2023-09-21 10:58:25 ] Completed train batch 0 metrics update 6.155 ms, 3.88 s total -Saving checkpoint at epoch 153 train batch 0 -[ 2023-09-21 10:58:25 ] Completed saving temp checkpoint 452.706 ms, 4.34 s total -[ 2023-09-21 10:58:25 ] Completed replacing temp checkpoint with checkpoint 16.782 ms, 4.35 s total -[ 2023-09-21 10:58:27 ] Completed train batch 1 to device 2,148.747 ms, 6.50 s total -[ 2023-09-21 10:58:28 ] Completed train batch 1 generator forward 41.265 ms, 6.54 s total -[ 2023-09-21 10:58:28 ] Completed train batch 1 generator loss: 1.250 64.441 ms, 6.61 s total -[ 2023-09-21 10:58:28 ] Completed train batch 1 generator backward 716.992 ms, 7.32 s total -[ 2023-09-21 10:58:28 ] Completed train batch 1 discriminator loss 0.052 56.474 ms, 7.38 s total -[ 2023-09-21 10:58:28 ] Completed train batch 1 discriminator backward 74.228 ms, 7.45 s total -Epoch [153] Step [1/2], gen_loss: 0.406, disc_loss: 0.017 -[ 2023-09-21 10:58:28 ] Completed train batch 1 metrics update 5.365 ms, 7.46 s total -Saving checkpoint at epoch 153 train batch 1 -[ 2023-09-21 10:58:29 ] Completed saving temp checkpoint 484.147 ms, 7.94 s total -[ 2023-09-21 10:58:29 ] Completed replacing temp checkpoint with checkpoint 23.783 ms, 7.97 s total -Epoch [153] :: gen_loss: 0.409, disc_loss: 0.017 -[ 2023-09-21 10:58:29 ] Completed training generator for epoch 153 55.202 ms, 8.02 s total -[ 2023-09-21 10:58:29 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 153 from eval step 0 - -[ 2023-09-21 10:58:30 ] Completed eval batch 0 to device 1,164.895 ms, 1.16 s total -[ 2023-09-21 10:58:30 ] Completed eval batch 0 forward 32.862 ms, 1.20 s total -[ 2023-09-21 10:58:30 ] Completed eval batch 0 recons_loss 0.382 ms, 1.20 s total -[ 2023-09-21 10:58:30 ] Completed eval batch 0 metrics update 0.687 ms, 1.20 s total -Saving checkpoint at epoch 153 val batch 0 -[ 2023-09-21 10:58:31 ] Completed saving temp checkpoint 447.940 ms, 1.65 s total -[ 2023-09-21 10:58:31 ] Completed replacing temp checkpoint with checkpoint 21.371 ms, 1.67 s total -[ 2023-09-21 10:58:31 ] Completed eval batch 1 to device 368.625 ms, 2.04 s total -[ 2023-09-21 10:58:31 ] Completed eval batch 1 forward 21.795 ms, 2.06 s total -[ 2023-09-21 10:58:31 ] Completed eval batch 1 recons_loss 0.360 ms, 2.06 s total -[ 2023-09-21 10:58:31 ] Completed eval batch 1 metrics update 224.402 ms, 2.28 s total -Saving checkpoint at epoch 153 val batch 1 -Epoch 153 val loss: 0.0592 -[ 2023-09-21 10:58:32 ] Completed saving temp checkpoint 469.256 ms, 2.75 s total -[ 2023-09-21 10:58:32 ] Completed replacing temp checkpoint with checkpoint 26.219 ms, 2.78 s total -[ 2023-09-21 10:58:32 ] Completed evaluating generator for epoch 153 46.816 ms, 2.83 s total - - -EPOCH :: 154 - - -[ 2023-09-21 10:58:32 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 154 from training step 0 - -[ 2023-09-21 10:58:35 ] Completed train batch 0 to device 3,182.660 ms, 3.18 s total -[ 2023-09-21 10:58:35 ] Completed train batch 0 generator forward 32.745 ms, 3.22 s total -[ 2023-09-21 10:58:35 ] Completed train batch 0 generator loss: 1.237 75.573 ms, 3.29 s total -[ 2023-09-21 10:58:36 ] Completed train batch 0 generator backward 527.516 ms, 3.82 s total -[ 2023-09-21 10:58:36 ] Completed train batch 0 discriminator loss 0.053 57.814 ms, 3.88 s total -[ 2023-09-21 10:58:36 ] Completed train batch 0 discriminator backward 106.554 ms, 3.98 s total -Epoch [154] Step [0/2], gen_loss: 0.411, disc_loss: 0.018 -[ 2023-09-21 10:58:36 ] Completed train batch 0 metrics update 5.708 ms, 3.99 s total -Saving checkpoint at epoch 154 train batch 0 -[ 2023-09-21 10:58:36 ] Completed saving temp checkpoint 546.757 ms, 4.54 s total -[ 2023-09-21 10:58:36 ] Completed replacing temp checkpoint with checkpoint 26.505 ms, 4.56 s total -[ 2023-09-21 10:58:38 ] Completed train batch 1 to device 1,693.154 ms, 6.25 s total -[ 2023-09-21 10:58:38 ] Completed train batch 1 generator forward 22.711 ms, 6.28 s total -[ 2023-09-21 10:58:38 ] Completed train batch 1 generator loss: 1.182 63.680 ms, 6.34 s total -[ 2023-09-21 10:58:39 ] Completed train batch 1 generator backward 1,082.971 ms, 7.42 s total -[ 2023-09-21 10:58:39 ] Completed train batch 1 discriminator loss 0.059 56.555 ms, 7.48 s total -[ 2023-09-21 10:58:39 ] Completed train batch 1 discriminator backward 69.227 ms, 7.55 s total -Epoch [154] Step [1/2], gen_loss: 0.401, disc_loss: 0.020 -[ 2023-09-21 10:58:39 ] Completed train batch 1 metrics update 5.767 ms, 7.56 s total -Saving checkpoint at epoch 154 train batch 1 -[ 2023-09-21 10:58:40 ] Completed saving temp checkpoint 566.742 ms, 8.12 s total -[ 2023-09-21 10:58:40 ] Completed replacing temp checkpoint with checkpoint 35.727 ms, 8.16 s total -Epoch [154] :: gen_loss: 0.406, disc_loss: 0.019 -[ 2023-09-21 10:58:40 ] Completed training generator for epoch 154 74.848 ms, 8.23 s total -[ 2023-09-21 10:58:40 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 154 from eval step 0 - -[ 2023-09-21 10:58:41 ] Completed eval batch 0 to device 1,184.157 ms, 1.18 s total -[ 2023-09-21 10:58:41 ] Completed eval batch 0 forward 32.066 ms, 1.22 s total -[ 2023-09-21 10:58:41 ] Completed eval batch 0 recons_loss 0.353 ms, 1.22 s total -[ 2023-09-21 10:58:41 ] Completed eval batch 0 metrics update 0.671 ms, 1.22 s total -Saving checkpoint at epoch 154 val batch 0 -[ 2023-09-21 10:58:42 ] Completed saving temp checkpoint 441.991 ms, 1.66 s total -[ 2023-09-21 10:58:42 ] Completed replacing temp checkpoint with checkpoint 23.425 ms, 1.68 s total -[ 2023-09-21 10:58:42 ] Completed eval batch 1 to device 427.953 ms, 2.11 s total -[ 2023-09-21 10:58:42 ] Completed eval batch 1 forward 22.208 ms, 2.13 s total -[ 2023-09-21 10:58:42 ] Completed eval batch 1 recons_loss 0.407 ms, 2.13 s total -[ 2023-09-21 10:58:42 ] Completed eval batch 1 metrics update 0.383 ms, 2.13 s total -Saving checkpoint at epoch 154 val batch 1 -Epoch 154 val loss: 0.0558 -[ 2023-09-21 10:58:43 ] Completed saving temp checkpoint 455.896 ms, 2.59 s total -[ 2023-09-21 10:58:43 ] Completed replacing temp checkpoint with checkpoint 20.323 ms, 2.61 s total -[ 2023-09-21 10:58:43 ] Completed evaluating generator for epoch 154 49.392 ms, 2.66 s total - - -EPOCH :: 155 - - -[ 2023-09-21 10:58:43 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 155 from training step 0 - -[ 2023-09-21 10:58:46 ] Completed train batch 0 to device 3,057.070 ms, 3.06 s total -[ 2023-09-21 10:58:46 ] Completed train batch 0 generator forward 26.935 ms, 3.08 s total -[ 2023-09-21 10:58:46 ] Completed train batch 0 generator loss: 1.203 66.341 ms, 3.15 s total -[ 2023-09-21 10:58:46 ] Completed train batch 0 generator backward 620.963 ms, 3.77 s total -[ 2023-09-21 10:58:47 ] Completed train batch 0 discriminator loss 0.071 56.967 ms, 3.83 s total -[ 2023-09-21 10:58:47 ] Completed train batch 0 discriminator backward 101.940 ms, 3.93 s total -Epoch [155] Step [0/2], gen_loss: 0.401, disc_loss: 0.023 -[ 2023-09-21 10:58:47 ] Completed train batch 0 metrics update 12.638 ms, 3.94 s total -Saving checkpoint at epoch 155 train batch 0 -[ 2023-09-21 10:58:47 ] Completed saving temp checkpoint 440.228 ms, 4.38 s total -[ 2023-09-21 10:58:47 ] Completed replacing temp checkpoint with checkpoint 24.667 ms, 4.41 s total -[ 2023-09-21 10:58:49 ] Completed train batch 1 to device 1,461.780 ms, 5.87 s total -[ 2023-09-21 10:58:49 ] Completed train batch 1 generator forward 39.912 ms, 5.91 s total -[ 2023-09-21 10:58:49 ] Completed train batch 1 generator loss: 1.242 63.450 ms, 5.97 s total -[ 2023-09-21 10:58:50 ] Completed train batch 1 generator backward 1,287.871 ms, 7.26 s total -[ 2023-09-21 10:58:50 ] Completed train batch 1 discriminator loss 0.063 56.587 ms, 7.32 s total -[ 2023-09-21 10:58:50 ] Completed train batch 1 discriminator backward 73.250 ms, 7.39 s total -Epoch [155] Step [1/2], gen_loss: 0.415, disc_loss: 0.022 -[ 2023-09-21 10:58:50 ] Completed train batch 1 metrics update 3.680 ms, 7.39 s total -Saving checkpoint at epoch 155 train batch 1 -[ 2023-09-21 10:58:51 ] Completed saving temp checkpoint 491.284 ms, 7.89 s total -[ 2023-09-21 10:58:51 ] Completed replacing temp checkpoint with checkpoint 22.432 ms, 7.91 s total -Epoch [155] :: gen_loss: 0.408, disc_loss: 0.023 -[ 2023-09-21 10:58:51 ] Completed training generator for epoch 155 56.008 ms, 7.96 s total -[ 2023-09-21 10:58:51 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 155 from eval step 0 - -[ 2023-09-21 10:58:52 ] Completed eval batch 0 to device 1,151.396 ms, 1.15 s total -[ 2023-09-21 10:58:52 ] Completed eval batch 0 forward 43.389 ms, 1.19 s total -[ 2023-09-21 10:58:52 ] Completed eval batch 0 recons_loss 0.333 ms, 1.20 s total -[ 2023-09-21 10:58:52 ] Completed eval batch 0 metrics update 0.613 ms, 1.20 s total -Saving checkpoint at epoch 155 val batch 0 -[ 2023-09-21 10:58:52 ] Completed saving temp checkpoint 455.429 ms, 1.65 s total -[ 2023-09-21 10:58:52 ] Completed replacing temp checkpoint with checkpoint 19.801 ms, 1.67 s total -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 11:13:00 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 11:13:00 ] Completed importing Timer 0.030 ms, 0.00 s total -[ 2023-09-21 11:13:05 ] Completed importing everything else 4,714.783 ms, 4.71 s total -| distributed init (rank 1): env:// -| distributed init (rank 3): env:// -| distributed init (rank 2): env:// -| distributed init (rank 4): env:// -| distributed init (rank 0): env:// -| distributed init (rank 5): env:// -[ 2023-09-21 11:13:13 ] Completed preliminaries 7,377.462 ms, 12.09 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 11:13:13 ] Completed build datasets 10.896 ms, 12.10 s total -[ 2023-09-21 11:13:13 ] Completed build samplers 0.049 ms, 12.10 s total -[ 2023-09-21 11:13:13 ] Completed build dataloaders 0.137 ms, 12.10 s total -[ 2023-09-21 11:13:13 ] Completed generator to device 78.193 ms, 12.18 s total -[ 2023-09-21 11:13:13 ] Completed discriminator to device 7.235 ms, 12.19 s total -[ 2023-09-21 11:13:13 ] Completed loss functions 414.811 ms, 12.60 s total -[ 2023-09-21 11:13:13 ] Completed models prepped for distribution 52.126 ms, 12.66 s total -[ 2023-09-21 11:13:13 ] Completed optimizers 0.668 ms, 12.66 s total -[ 2023-09-21 11:13:13 ] Completed grad scalers 0.022 ms, 12.66 s total -[ 2023-09-21 11:13:14 ] Completed checkpoint retrieval 460.843 ms, 13.12 s total - - -EPOCH :: 155 - - -[ 2023-09-21 11:13:14 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 155 from training step 2 - -Epoch [155] :: gen_loss: 0.408, disc_loss: 0.023 -[ 2023-09-21 11:13:14 ] Completed training generator for epoch 155 138.954 ms, 0.14 s total -[ 2023-09-21 11:13:14 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 155 from eval step 1 - -[ 2023-09-21 11:13:15 ] Completed eval batch 1 to device 1,431.437 ms, 1.43 s total -[ 2023-09-21 11:13:16 ] Completed eval batch 1 forward 1,177.239 ms, 2.61 s total -[ 2023-09-21 11:13:16 ] Completed eval batch 1 recons_loss 0.268 ms, 2.61 s total -[ 2023-09-21 11:13:16 ] Completed eval batch 1 metrics update 196.947 ms, 2.81 s total -Saving checkpoint at epoch 155 val batch 1 -Epoch 155 val loss: 0.0614 -[ 2023-09-21 11:13:17 ] Completed saving temp checkpoint 440.641 ms, 3.25 s total -[ 2023-09-21 11:13:17 ] Completed replacing temp checkpoint with checkpoint 141.088 ms, 3.39 s total -[ 2023-09-21 11:13:17 ] Completed evaluating generator for epoch 155 37.005 ms, 3.42 s total - - -EPOCH :: 156 - - -[ 2023-09-21 11:13:17 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 156 from training step 0 - -[ 2023-09-21 11:13:20 ] Completed train batch 0 to device 2,967.470 ms, 2.97 s total -[ 2023-09-21 11:13:20 ] Completed train batch 0 generator forward 39.117 ms, 3.01 s total -[ 2023-09-21 11:13:20 ] Completed train batch 0 generator loss: 1.256 95.929 ms, 3.10 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 11:13:21 ] Completed train batch 0 generator backward 674.508 ms, 3.78 s total -[ 2023-09-21 11:13:21 ] Completed train batch 0 discriminator loss 0.049 57.286 ms, 3.83 s total -[ 2023-09-21 11:13:21 ] Completed train batch 0 discriminator backward 115.254 ms, 3.95 s total -Epoch [156] Step [0/2], gen_loss: 0.414, disc_loss: 0.017 -[ 2023-09-21 11:13:21 ] Completed train batch 0 metrics update 5.312 ms, 3.95 s total -Saving checkpoint at epoch 156 train batch 0 -[ 2023-09-21 11:13:21 ] Completed saving temp checkpoint 417.611 ms, 4.37 s total -[ 2023-09-21 11:13:22 ] Completed replacing temp checkpoint with checkpoint 25.522 ms, 4.40 s total -[ 2023-09-21 11:13:23 ] Completed train batch 1 to device 1,352.741 ms, 5.75 s total -[ 2023-09-21 11:13:23 ] Completed train batch 1 generator forward 22.218 ms, 5.77 s total -[ 2023-09-21 11:13:23 ] Completed train batch 1 generator loss: 1.251 62.264 ms, 5.84 s total -[ 2023-09-21 11:13:24 ] Completed train batch 1 generator backward 1,385.862 ms, 7.22 s total -[ 2023-09-21 11:13:24 ] Completed train batch 1 discriminator loss 0.045 60.205 ms, 7.28 s total -[ 2023-09-21 11:13:24 ] Completed train batch 1 discriminator backward 61.517 ms, 7.34 s total -Epoch [156] Step [1/2], gen_loss: 0.426, disc_loss: 0.015 -[ 2023-09-21 11:13:24 ] Completed train batch 1 metrics update 1.956 ms, 7.34 s total -Saving checkpoint at epoch 156 train batch 1 -[ 2023-09-21 11:13:25 ] Completed saving temp checkpoint 436.508 ms, 7.78 s total -[ 2023-09-21 11:13:25 ] Completed replacing temp checkpoint with checkpoint 20.627 ms, 7.80 s total -Epoch [156] :: gen_loss: 0.420, disc_loss: 0.016 -[ 2023-09-21 11:13:25 ] Completed training generator for epoch 156 51.867 ms, 7.85 s total -[ 2023-09-21 11:13:25 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 156 from eval step 0 - -[ 2023-09-21 11:13:26 ] Completed eval batch 0 to device 1,145.493 ms, 1.15 s total -[ 2023-09-21 11:13:26 ] Completed eval batch 0 forward 31.748 ms, 1.18 s total -[ 2023-09-21 11:13:26 ] Completed eval batch 0 recons_loss 0.402 ms, 1.18 s total -[ 2023-09-21 11:13:26 ] Completed eval batch 0 metrics update 0.738 ms, 1.18 s total -Saving checkpoint at epoch 156 val batch 0 -[ 2023-09-21 11:13:27 ] Completed saving temp checkpoint 367.484 ms, 1.55 s total -[ 2023-09-21 11:13:27 ] Completed replacing temp checkpoint with checkpoint 22.758 ms, 1.57 s total -[ 2023-09-21 11:13:27 ] Completed eval batch 1 to device 464.944 ms, 2.03 s total -[ 2023-09-21 11:13:27 ] Completed eval batch 1 forward 23.428 ms, 2.06 s total -[ 2023-09-21 11:13:27 ] Completed eval batch 1 recons_loss 0.329 ms, 2.06 s total -[ 2023-09-21 11:13:27 ] Completed eval batch 1 metrics update 273.920 ms, 2.33 s total -Saving checkpoint at epoch 156 val batch 1 -Epoch 156 val loss: 0.0636 -[ 2023-09-21 11:13:28 ] Completed saving temp checkpoint 420.426 ms, 2.75 s total -[ 2023-09-21 11:13:28 ] Completed replacing temp checkpoint with checkpoint 16.221 ms, 2.77 s total -[ 2023-09-21 11:13:28 ] Completed evaluating generator for epoch 156 65.548 ms, 2.83 s total - - -EPOCH :: 157 - - -[ 2023-09-21 11:13:28 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 157 from training step 0 - -[ 2023-09-21 11:13:31 ] Completed train batch 0 to device 3,065.257 ms, 3.07 s total -[ 2023-09-21 11:13:31 ] Completed train batch 0 generator forward 43.861 ms, 3.11 s total -[ 2023-09-21 11:13:31 ] Completed train batch 0 generator loss: 1.225 67.641 ms, 3.18 s total -[ 2023-09-21 11:13:32 ] Completed train batch 0 generator backward 749.137 ms, 3.93 s total -[ 2023-09-21 11:13:32 ] Completed train batch 0 discriminator loss 0.046 57.843 ms, 3.98 s total -[ 2023-09-21 11:13:32 ] Completed train batch 0 discriminator backward 97.532 ms, 4.08 s total -Epoch [157] Step [0/2], gen_loss: 0.417, disc_loss: 0.016 -[ 2023-09-21 11:13:32 ] Completed train batch 0 metrics update 6.089 ms, 4.09 s total -Saving checkpoint at epoch 157 train batch 0 -[ 2023-09-21 11:13:32 ] Completed saving temp checkpoint 492.651 ms, 4.58 s total -[ 2023-09-21 11:13:32 ] Completed replacing temp checkpoint with checkpoint 21.601 ms, 4.60 s total -[ 2023-09-21 11:13:34 ] Completed train batch 1 to device 1,245.935 ms, 5.85 s total -[ 2023-09-21 11:13:34 ] Completed train batch 1 generator forward 40.384 ms, 5.89 s total -[ 2023-09-21 11:13:34 ] Completed train batch 1 generator loss: 1.240 68.391 ms, 5.96 s total -[ 2023-09-21 11:13:35 ] Completed train batch 1 generator backward 1,642.776 ms, 7.60 s total -[ 2023-09-21 11:13:35 ] Completed train batch 1 discriminator loss 0.046 56.871 ms, 7.66 s total -[ 2023-09-21 11:13:36 ] Completed train batch 1 discriminator backward 66.908 ms, 7.72 s total -Epoch [157] Step [1/2], gen_loss: 0.418, disc_loss: 0.016 -[ 2023-09-21 11:13:36 ] Completed train batch 1 metrics update 0.775 ms, 7.72 s total -Saving checkpoint at epoch 157 train batch 1 -[ 2023-09-21 11:13:36 ] Completed saving temp checkpoint 416.610 ms, 8.14 s total -[ 2023-09-21 11:13:36 ] Completed replacing temp checkpoint with checkpoint 22.190 ms, 8.16 s total -Epoch [157] :: gen_loss: 0.418, disc_loss: 0.016 -[ 2023-09-21 11:13:36 ] Completed training generator for epoch 157 51.849 ms, 8.21 s total -[ 2023-09-21 11:13:36 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 157 from eval step 0 - -[ 2023-09-21 11:13:37 ] Completed eval batch 0 to device 1,145.786 ms, 1.15 s total -[ 2023-09-21 11:13:37 ] Completed eval batch 0 forward 27.276 ms, 1.17 s total -[ 2023-09-21 11:13:37 ] Completed eval batch 0 recons_loss 0.278 ms, 1.17 s total -[ 2023-09-21 11:13:37 ] Completed eval batch 0 metrics update 0.567 ms, 1.17 s total -Saving checkpoint at epoch 157 val batch 0 -[ 2023-09-21 11:13:38 ] Completed saving temp checkpoint 429.322 ms, 1.60 s total -[ 2023-09-21 11:13:38 ] Completed replacing temp checkpoint with checkpoint 30.365 ms, 1.63 s total -[ 2023-09-21 11:13:38 ] Completed eval batch 1 to device 545.660 ms, 2.18 s total -[ 2023-09-21 11:13:38 ] Completed eval batch 1 forward 22.180 ms, 2.20 s total -[ 2023-09-21 11:13:38 ] Completed eval batch 1 recons_loss 0.414 ms, 2.20 s total -[ 2023-09-21 11:13:38 ] Completed eval batch 1 metrics update 164.753 ms, 2.37 s total -Saving checkpoint at epoch 157 val batch 1 -Epoch 157 val loss: 0.0581 -[ 2023-09-21 11:13:39 ] Completed saving temp checkpoint 534.373 ms, 2.90 s total -[ 2023-09-21 11:13:39 ] Completed replacing temp checkpoint with checkpoint 31.559 ms, 2.93 s total -[ 2023-09-21 11:13:39 ] Completed evaluating generator for epoch 157 74.742 ms, 3.01 s total - - -EPOCH :: 158 - - -[ 2023-09-21 11:13:39 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 158 from training step 0 - -[ 2023-09-21 11:13:42 ] Completed train batch 0 to device 3,127.317 ms, 3.13 s total -[ 2023-09-21 11:13:42 ] Completed train batch 0 generator forward 26.959 ms, 3.15 s total -[ 2023-09-21 11:13:42 ] Completed train batch 0 generator loss: 1.222 66.715 ms, 3.22 s total -[ 2023-09-21 11:13:43 ] Completed train batch 0 generator backward 645.169 ms, 3.87 s total -[ 2023-09-21 11:13:43 ] Completed train batch 0 discriminator loss 0.048 61.805 ms, 3.93 s total -[ 2023-09-21 11:13:43 ] Completed train batch 0 discriminator backward 99.637 ms, 4.03 s total -Epoch [158] Step [0/2], gen_loss: 0.411, disc_loss: 0.016 -[ 2023-09-21 11:13:43 ] Completed train batch 0 metrics update 4.100 ms, 4.03 s total -Saving checkpoint at epoch 158 train batch 0 -[ 2023-09-21 11:13:43 ] Completed saving temp checkpoint 452.702 ms, 4.48 s total -[ 2023-09-21 11:13:44 ] Completed replacing temp checkpoint with checkpoint 23.390 ms, 4.51 s total -[ 2023-09-21 11:13:45 ] Completed train batch 1 to device 1,547.690 ms, 6.06 s total -[ 2023-09-21 11:13:45 ] Completed train batch 1 generator forward 22.277 ms, 6.08 s total -[ 2023-09-21 11:13:45 ] Completed train batch 1 generator loss: 1.224 63.822 ms, 6.14 s total -[ 2023-09-21 11:13:47 ] Completed train batch 1 generator backward 1,405.915 ms, 7.55 s total -[ 2023-09-21 11:13:47 ] Completed train batch 1 discriminator loss 0.048 56.513 ms, 7.60 s total -[ 2023-09-21 11:13:47 ] Completed train batch 1 discriminator backward 73.274 ms, 7.68 s total -Epoch [158] Step [1/2], gen_loss: 0.418, disc_loss: 0.016 -[ 2023-09-21 11:13:47 ] Completed train batch 1 metrics update 5.079 ms, 7.68 s total -Saving checkpoint at epoch 158 train batch 1 -[ 2023-09-21 11:13:47 ] Completed saving temp checkpoint 476.783 ms, 8.16 s total -[ 2023-09-21 11:13:47 ] Completed replacing temp checkpoint with checkpoint 22.559 ms, 8.18 s total -Epoch [158] :: gen_loss: 0.415, disc_loss: 0.016 -[ 2023-09-21 11:13:47 ] Completed training generator for epoch 158 60.523 ms, 8.24 s total -[ 2023-09-21 11:13:47 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 158 from eval step 0 - -[ 2023-09-21 11:13:48 ] Completed eval batch 0 to device 1,150.040 ms, 1.15 s total -[ 2023-09-21 11:13:48 ] Completed eval batch 0 forward 29.131 ms, 1.18 s total -[ 2023-09-21 11:13:48 ] Completed eval batch 0 recons_loss 0.376 ms, 1.18 s total -[ 2023-09-21 11:13:48 ] Completed eval batch 0 metrics update 0.644 ms, 1.18 s total -Saving checkpoint at epoch 158 val batch 0 -[ 2023-09-21 11:13:49 ] Completed saving temp checkpoint 423.280 ms, 1.60 s total -[ 2023-09-21 11:13:49 ] Completed replacing temp checkpoint with checkpoint 31.289 ms, 1.63 s total -[ 2023-09-21 11:13:49 ] Completed eval batch 1 to device 464.634 ms, 2.10 s total -[ 2023-09-21 11:13:49 ] Completed eval batch 1 forward 21.772 ms, 2.12 s total -[ 2023-09-21 11:13:49 ] Completed eval batch 1 recons_loss 0.348 ms, 2.12 s total -[ 2023-09-21 11:13:50 ] Completed eval batch 1 metrics update 147.495 ms, 2.27 s total -Saving checkpoint at epoch 158 val batch 1 -Epoch 158 val loss: 0.0565 -[ 2023-09-21 11:13:50 ] Completed saving temp checkpoint 499.477 ms, 2.77 s total -[ 2023-09-21 11:13:50 ] Completed replacing temp checkpoint with checkpoint 25.554 ms, 2.79 s total -[ 2023-09-21 11:13:50 ] Completed evaluating generator for epoch 158 50.411 ms, 2.84 s total - - -EPOCH :: 159 - - -[ 2023-09-21 11:13:50 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 159 from training step 0 - -[ 2023-09-21 11:13:53 ] Completed train batch 0 to device 3,180.885 ms, 3.18 s total -[ 2023-09-21 11:13:53 ] Completed train batch 0 generator forward 32.338 ms, 3.21 s total -[ 2023-09-21 11:13:53 ] Completed train batch 0 generator loss: 1.182 70.856 ms, 3.28 s total -[ 2023-09-21 11:13:54 ] Completed train batch 0 generator backward 523.307 ms, 3.81 s total -[ 2023-09-21 11:13:54 ] Completed train batch 0 discriminator loss 0.046 56.068 ms, 3.86 s total -[ 2023-09-21 11:13:54 ] Completed train batch 0 discriminator backward 94.783 ms, 3.96 s total -Epoch [159] Step [0/2], gen_loss: 0.416, disc_loss: 0.015 -[ 2023-09-21 11:13:54 ] Completed train batch 0 metrics update 21.699 ms, 3.98 s total -Saving checkpoint at epoch 159 train batch 0 -[ 2023-09-21 11:13:55 ] Completed saving temp checkpoint 460.045 ms, 4.44 s total -[ 2023-09-21 11:13:55 ] Completed replacing temp checkpoint with checkpoint 31.156 ms, 4.47 s total -[ 2023-09-21 11:13:56 ] Completed train batch 1 to device 1,678.937 ms, 6.15 s total -[ 2023-09-21 11:13:56 ] Completed train batch 1 generator forward 40.275 ms, 6.19 s total -[ 2023-09-21 11:13:56 ] Completed train batch 1 generator loss: 1.306 63.576 ms, 6.25 s total -[ 2023-09-21 11:13:57 ] Completed train batch 1 generator backward 1,045.973 ms, 7.30 s total -[ 2023-09-21 11:13:57 ] Completed train batch 1 discriminator loss 0.048 57.006 ms, 7.36 s total -[ 2023-09-21 11:13:58 ] Completed train batch 1 discriminator backward 75.803 ms, 7.43 s total -Epoch [159] Step [1/2], gen_loss: 0.424, disc_loss: 0.016 -[ 2023-09-21 11:13:58 ] Completed train batch 1 metrics update 5.617 ms, 7.44 s total -Saving checkpoint at epoch 159 train batch 1 -[ 2023-09-21 11:13:58 ] Completed saving temp checkpoint 464.381 ms, 7.90 s total -[ 2023-09-21 11:13:58 ] Completed replacing temp checkpoint with checkpoint 21.634 ms, 7.92 s total -Epoch [159] :: gen_loss: 0.420, disc_loss: 0.015 -[ 2023-09-21 11:13:58 ] Completed training generator for epoch 159 52.964 ms, 7.98 s total -[ 2023-09-21 11:13:58 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 159 from eval step 0 - -[ 2023-09-21 11:13:59 ] Completed eval batch 0 to device 1,108.262 ms, 1.11 s total -[ 2023-09-21 11:13:59 ] Completed eval batch 0 forward 32.259 ms, 1.14 s total -[ 2023-09-21 11:13:59 ] Completed eval batch 0 recons_loss 0.368 ms, 1.14 s total -[ 2023-09-21 11:13:59 ] Completed eval batch 0 metrics update 0.692 ms, 1.14 s total -Saving checkpoint at epoch 159 val batch 0 -[ 2023-09-21 11:14:00 ] Completed saving temp checkpoint 423.684 ms, 1.57 s total -[ 2023-09-21 11:14:00 ] Completed replacing temp checkpoint with checkpoint 24.927 ms, 1.59 s total -[ 2023-09-21 11:14:00 ] Completed eval batch 1 to device 382.337 ms, 1.97 s total -[ 2023-09-21 11:14:00 ] Completed eval batch 1 forward 21.565 ms, 1.99 s total -[ 2023-09-21 11:14:00 ] Completed eval batch 1 recons_loss 0.332 ms, 1.99 s total -[ 2023-09-21 11:14:00 ] Completed eval batch 1 metrics update 251.493 ms, 2.25 s total -Saving checkpoint at epoch 159 val batch 1 -Epoch 159 val loss: 0.0622 -[ 2023-09-21 11:14:01 ] Completed saving temp checkpoint 474.632 ms, 2.72 s total -[ 2023-09-21 11:14:01 ] Completed replacing temp checkpoint with checkpoint 23.104 ms, 2.74 s total -[ 2023-09-21 11:14:01 ] Completed evaluating generator for epoch 159 55.994 ms, 2.80 s total - - -EPOCH :: 160 - - -[ 2023-09-21 11:14:01 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 160 from training step 0 - -[ 2023-09-21 11:14:04 ] Completed train batch 0 to device 3,060.703 ms, 3.06 s total -[ 2023-09-21 11:14:04 ] Completed train batch 0 generator forward 37.316 ms, 3.10 s total -[ 2023-09-21 11:14:04 ] Completed train batch 0 generator loss: 1.248 66.920 ms, 3.16 s total -[ 2023-09-21 11:14:05 ] Completed train batch 0 generator backward 632.178 ms, 3.80 s total -[ 2023-09-21 11:14:05 ] Completed train batch 0 discriminator loss 0.048 64.488 ms, 3.86 s total -[ 2023-09-21 11:14:05 ] Completed train batch 0 discriminator backward 78.980 ms, 3.94 s total -Epoch [160] Step [0/2], gen_loss: 0.408, disc_loss: 0.016 -[ 2023-09-21 11:14:05 ] Completed train batch 0 metrics update 17.878 ms, 3.96 s total -Saving checkpoint at epoch 160 train batch 0 -[ 2023-09-21 11:14:05 ] Completed saving temp checkpoint 464.164 ms, 4.42 s total -[ 2023-09-21 11:14:05 ] Completed replacing temp checkpoint with checkpoint 34.739 ms, 4.46 s total -[ 2023-09-21 11:14:07 ] Completed train batch 1 to device 1,437.045 ms, 5.89 s total -[ 2023-09-21 11:14:07 ] Completed train batch 1 generator forward 22.302 ms, 5.92 s total -[ 2023-09-21 11:14:07 ] Completed train batch 1 generator loss: 1.218 63.208 ms, 5.98 s total -[ 2023-09-21 11:14:08 ] Completed train batch 1 generator backward 1,318.264 ms, 7.30 s total -[ 2023-09-21 11:14:08 ] Completed train batch 1 discriminator loss 0.048 56.191 ms, 7.35 s total -[ 2023-09-21 11:14:08 ] Completed train batch 1 discriminator backward 73.177 ms, 7.43 s total -Epoch [160] Step [1/2], gen_loss: 0.419, disc_loss: 0.016 -[ 2023-09-21 11:14:08 ] Completed train batch 1 metrics update 0.561 ms, 7.43 s total -Saving checkpoint at epoch 160 train batch 1 -[ 2023-09-21 11:14:09 ] Completed saving temp checkpoint 477.885 ms, 7.91 s total -[ 2023-09-21 11:14:09 ] Completed replacing temp checkpoint with checkpoint 23.729 ms, 7.93 s total -Epoch [160] :: gen_loss: 0.414, disc_loss: 0.016 -[ 2023-09-21 11:14:09 ] Completed training generator for epoch 160 50.791 ms, 7.98 s total -[ 2023-09-21 11:14:09 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 160 from eval step 0 - -[ 2023-09-21 11:14:10 ] Completed eval batch 0 to device 1,117.640 ms, 1.12 s total -[ 2023-09-21 11:14:10 ] Completed eval batch 0 forward 38.242 ms, 1.16 s total -[ 2023-09-21 11:14:10 ] Completed eval batch 0 recons_loss 0.330 ms, 1.16 s total -[ 2023-09-21 11:14:10 ] Completed eval batch 0 metrics update 0.689 ms, 1.16 s total -Saving checkpoint at epoch 160 val batch 0 -[ 2023-09-21 11:14:10 ] Completed saving temp checkpoint 398.881 ms, 1.56 s total -[ 2023-09-21 11:14:10 ] Completed replacing temp checkpoint with checkpoint 31.499 ms, 1.59 s total -[ 2023-09-21 11:14:11 ] Completed eval batch 1 to device 441.087 ms, 2.03 s total -[ 2023-09-21 11:14:11 ] Completed eval batch 1 forward 29.332 ms, 2.06 s total -[ 2023-09-21 11:14:11 ] Completed eval batch 1 recons_loss 0.328 ms, 2.06 s total -[ 2023-09-21 11:14:11 ] Completed eval batch 1 metrics update 176.758 ms, 2.23 s total -Saving checkpoint at epoch 160 val batch 1 -Epoch 160 val loss: 0.0651 -[ 2023-09-21 11:14:12 ] Completed saving temp checkpoint 447.384 ms, 2.68 s total -[ 2023-09-21 11:14:12 ] Completed replacing temp checkpoint with checkpoint 29.461 ms, 2.71 s total -[ 2023-09-21 11:14:12 ] Completed evaluating generator for epoch 160 64.309 ms, 2.78 s total - - -EPOCH :: 161 - - -[ 2023-09-21 11:14:12 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 161 from training step 0 - -[ 2023-09-21 11:14:15 ] Completed train batch 0 to device 2,987.119 ms, 2.99 s total -[ 2023-09-21 11:14:15 ] Completed train batch 0 generator forward 44.226 ms, 3.03 s total -[ 2023-09-21 11:14:15 ] Completed train batch 0 generator loss: 1.235 66.261 ms, 3.10 s total -[ 2023-09-21 11:14:15 ] Completed train batch 0 generator backward 703.737 ms, 3.80 s total -[ 2023-09-21 11:14:15 ] Completed train batch 0 discriminator loss 0.046 57.034 ms, 3.86 s total -[ 2023-09-21 11:14:16 ] Completed train batch 0 discriminator backward 90.117 ms, 3.95 s total -Epoch [161] Step [0/2], gen_loss: 0.416, disc_loss: 0.015 -[ 2023-09-21 11:14:16 ] Completed train batch 0 metrics update 4.990 ms, 3.95 s total -Saving checkpoint at epoch 161 train batch 0 -[ 2023-09-21 11:14:16 ] Completed saving temp checkpoint 463.270 ms, 4.42 s total -[ 2023-09-21 11:14:16 ] Completed replacing temp checkpoint with checkpoint 32.384 ms, 4.45 s total -[ 2023-09-21 11:14:17 ] Completed train batch 1 to device 1,415.060 ms, 5.86 s total -[ 2023-09-21 11:14:18 ] Completed train batch 1 generator forward 22.308 ms, 5.89 s total -[ 2023-09-21 11:14:18 ] Completed train batch 1 generator loss: 1.281 64.160 ms, 5.95 s total -[ 2023-09-21 11:14:19 ] Completed train batch 1 generator backward 1,317.523 ms, 7.27 s total -[ 2023-09-21 11:14:19 ] Completed train batch 1 discriminator loss 0.043 56.366 ms, 7.32 s total -[ 2023-09-21 11:14:19 ] Completed train batch 1 discriminator backward 71.954 ms, 7.40 s total -Epoch [161] Step [1/2], gen_loss: 0.413, disc_loss: 0.015 -[ 2023-09-21 11:14:19 ] Completed train batch 1 metrics update 2.673 ms, 7.40 s total -Saving checkpoint at epoch 161 train batch 1 -[ 2023-09-21 11:14:19 ] Completed saving temp checkpoint 461.537 ms, 7.86 s total -[ 2023-09-21 11:14:20 ] Completed replacing temp checkpoint with checkpoint 28.838 ms, 7.89 s total -Epoch [161] :: gen_loss: 0.414, disc_loss: 0.015 -[ 2023-09-21 11:14:20 ] Completed training generator for epoch 161 61.098 ms, 7.95 s total -[ 2023-09-21 11:14:20 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 161 from eval step 0 - -[ 2023-09-21 11:14:21 ] Completed eval batch 0 to device 1,060.488 ms, 1.06 s total -[ 2023-09-21 11:14:21 ] Completed eval batch 0 forward 24.724 ms, 1.09 s total -[ 2023-09-21 11:14:21 ] Completed eval batch 0 recons_loss 0.299 ms, 1.09 s total -[ 2023-09-21 11:14:21 ] Completed eval batch 0 metrics update 0.567 ms, 1.09 s total -Saving checkpoint at epoch 161 val batch 0 -[ 2023-09-21 11:14:21 ] Completed saving temp checkpoint 428.982 ms, 1.52 s total -[ 2023-09-21 11:14:21 ] Completed replacing temp checkpoint with checkpoint 23.686 ms, 1.54 s total -[ 2023-09-21 11:14:22 ] Completed eval batch 1 to device 524.977 ms, 2.06 s total -[ 2023-09-21 11:14:22 ] Completed eval batch 1 forward 21.602 ms, 2.09 s total -[ 2023-09-21 11:14:22 ] Completed eval batch 1 recons_loss 0.336 ms, 2.09 s total -[ 2023-09-21 11:14:22 ] Completed eval batch 1 metrics update 136.166 ms, 2.22 s total -Saving checkpoint at epoch 161 val batch 1 -Epoch 161 val loss: 0.0633 -[ 2023-09-21 11:14:22 ] Completed saving temp checkpoint 474.875 ms, 2.70 s total -[ 2023-09-21 11:14:22 ] Completed replacing temp checkpoint with checkpoint 25.641 ms, 2.72 s total -[ 2023-09-21 11:14:22 ] Completed evaluating generator for epoch 161 51.258 ms, 2.77 s total - - -EPOCH :: 162 - - -[ 2023-09-21 11:14:22 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 162 from training step 0 - -[ 2023-09-21 11:14:25 ] Completed train batch 0 to device 3,020.641 ms, 3.02 s total -[ 2023-09-21 11:14:25 ] Completed train batch 0 generator forward 29.925 ms, 3.05 s total -[ 2023-09-21 11:14:25 ] Completed train batch 0 generator loss: 1.230 66.006 ms, 3.12 s total -[ 2023-09-21 11:14:26 ] Completed train batch 0 generator backward 616.488 ms, 3.73 s total -[ 2023-09-21 11:14:26 ] Completed train batch 0 discriminator loss 0.043 57.633 ms, 3.79 s total -[ 2023-09-21 11:14:26 ] Completed train batch 0 discriminator backward 103.355 ms, 3.89 s total -Epoch [162] Step [0/2], gen_loss: 0.423, disc_loss: 0.015 -[ 2023-09-21 11:14:26 ] Completed train batch 0 metrics update 3.208 ms, 3.90 s total -Saving checkpoint at epoch 162 train batch 0 -[ 2023-09-21 11:14:27 ] Completed saving temp checkpoint 423.170 ms, 4.32 s total -[ 2023-09-21 11:14:27 ] Completed replacing temp checkpoint with checkpoint 24.386 ms, 4.34 s total -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 11:25:14 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 11:25:14 ] Completed importing Timer 0.024 ms, 0.00 s total -[ 2023-09-21 11:25:18 ] Completed importing everything else 4,521.837 ms, 4.52 s total -| distributed init (rank 1): env:// -| distributed init (rank 2): env:// -| distributed init (rank 3): env:// -| distributed init (rank 5): env:// -| distributed init (rank 4): env:// -| distributed init (rank 0): env:// -[ 2023-09-21 11:25:27 ] Completed preliminaries 8,082.521 ms, 12.60 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 11:25:27 ] Completed build datasets 10.598 ms, 12.61 s total -[ 2023-09-21 11:25:27 ] Completed build samplers 0.043 ms, 12.62 s total -[ 2023-09-21 11:25:27 ] Completed build dataloaders 0.135 ms, 12.62 s total -[ 2023-09-21 11:25:27 ] Completed generator to device 78.406 ms, 12.69 s total -[ 2023-09-21 11:25:27 ] Completed discriminator to device 7.022 ms, 12.70 s total -[ 2023-09-21 11:25:27 ] Completed loss functions 392.663 ms, 13.09 s total -[ 2023-09-21 11:25:27 ] Completed models prepped for distribution 83.490 ms, 13.18 s total -[ 2023-09-21 11:25:27 ] Completed optimizers 0.673 ms, 13.18 s total -[ 2023-09-21 11:25:27 ] Completed grad scalers 0.027 ms, 13.18 s total -[ 2023-09-21 11:25:28 ] Completed checkpoint retrieval 476.859 ms, 13.65 s total - - -EPOCH :: 162 - - -[ 2023-09-21 11:25:28 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 162 from training step 1 - -[ 2023-09-21 11:25:30 ] Completed train batch 1 to device 2,893.362 ms, 2.89 s total -[ 2023-09-21 11:25:32 ] Completed train batch 1 generator forward 1,178.150 ms, 4.07 s total -[ 2023-09-21 11:25:32 ] Completed train batch 1 generator loss: 1.224 76.550 ms, 4.15 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 11:25:33 ] Completed train batch 1 generator backward 1,781.445 ms, 5.93 s total -[ 2023-09-21 11:25:34 ] Completed train batch 1 discriminator loss 0.045 56.100 ms, 5.99 s total -[ 2023-09-21 11:25:34 ] Completed train batch 1 discriminator backward 79.203 ms, 6.06 s total -Epoch [162] Step [1/2], gen_loss: 0.424, disc_loss: 0.015 -[ 2023-09-21 11:25:34 ] Completed train batch 1 metrics update 1.753 ms, 6.07 s total -Saving checkpoint at epoch 162 train batch 1 -[ 2023-09-21 11:25:34 ] Completed saving temp checkpoint 398.223 ms, 6.46 s total -[ 2023-09-21 11:25:34 ] Completed replacing temp checkpoint with checkpoint 114.188 ms, 6.58 s total -Epoch [162] :: gen_loss: 0.423, disc_loss: 0.015 -[ 2023-09-21 11:25:34 ] Completed training generator for epoch 162 44.197 ms, 6.62 s total -[ 2023-09-21 11:25:34 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 162 from eval step 0 - -[ 2023-09-21 11:25:35 ] Completed eval batch 0 to device 1,129.918 ms, 1.13 s total -[ 2023-09-21 11:25:35 ] Completed eval batch 0 forward 31.245 ms, 1.16 s total -[ 2023-09-21 11:25:35 ] Completed eval batch 0 recons_loss 0.390 ms, 1.16 s total -[ 2023-09-21 11:25:35 ] Completed eval batch 0 metrics update 0.646 ms, 1.16 s total -Saving checkpoint at epoch 162 val batch 0 -[ 2023-09-21 11:25:36 ] Completed saving temp checkpoint 444.320 ms, 1.61 s total -[ 2023-09-21 11:25:36 ] Completed replacing temp checkpoint with checkpoint 31.759 ms, 1.64 s total -[ 2023-09-21 11:25:36 ] Completed eval batch 1 to device 410.344 ms, 2.05 s total -[ 2023-09-21 11:25:36 ] Completed eval batch 1 forward 28.498 ms, 2.08 s total -[ 2023-09-21 11:25:36 ] Completed eval batch 1 recons_loss 0.622 ms, 2.08 s total -[ 2023-09-21 11:25:36 ] Completed eval batch 1 metrics update 150.208 ms, 2.23 s total -Saving checkpoint at epoch 162 val batch 1 -Epoch 162 val loss: 0.0615 -[ 2023-09-21 11:25:37 ] Completed saving temp checkpoint 421.220 ms, 2.65 s total -[ 2023-09-21 11:25:37 ] Completed replacing temp checkpoint with checkpoint 25.333 ms, 2.67 s total -[ 2023-09-21 11:25:37 ] Completed evaluating generator for epoch 162 57.587 ms, 2.73 s total - - -EPOCH :: 163 - - -[ 2023-09-21 11:25:37 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 163 from training step 0 - -[ 2023-09-21 11:25:40 ] Completed train batch 0 to device 3,079.723 ms, 3.08 s total -[ 2023-09-21 11:25:40 ] Completed train batch 0 generator forward 28.932 ms, 3.11 s total -[ 2023-09-21 11:25:40 ] Completed train batch 0 generator loss: 1.358 65.699 ms, 3.17 s total -[ 2023-09-21 11:25:41 ] Completed train batch 0 generator backward 689.187 ms, 3.86 s total -[ 2023-09-21 11:25:41 ] Completed train batch 0 discriminator loss 0.049 57.758 ms, 3.92 s total -[ 2023-09-21 11:25:41 ] Completed train batch 0 discriminator backward 95.933 ms, 4.02 s total -Epoch [163] Step [0/2], gen_loss: 0.422, disc_loss: 0.016 -[ 2023-09-21 11:25:41 ] Completed train batch 0 metrics update 7.342 ms, 4.02 s total -Saving checkpoint at epoch 163 train batch 0 -[ 2023-09-21 11:25:41 ] Completed saving temp checkpoint 420.361 ms, 4.44 s total -[ 2023-09-21 11:25:41 ] Completed replacing temp checkpoint with checkpoint 31.643 ms, 4.48 s total -[ 2023-09-21 11:25:43 ] Completed train batch 1 to device 1,516.230 ms, 5.99 s total -[ 2023-09-21 11:25:43 ] Completed train batch 1 generator forward 41.037 ms, 6.03 s total -[ 2023-09-21 11:25:43 ] Completed train batch 1 generator loss: 1.251 65.003 ms, 6.10 s total -[ 2023-09-21 11:25:44 ] Completed train batch 1 generator backward 925.214 ms, 7.02 s total -[ 2023-09-21 11:25:44 ] Completed train batch 1 discriminator loss 0.045 56.650 ms, 7.08 s total -[ 2023-09-21 11:25:44 ] Completed train batch 1 discriminator backward 73.526 ms, 7.15 s total -Epoch [163] Step [1/2], gen_loss: 0.408, disc_loss: 0.015 -[ 2023-09-21 11:25:44 ] Completed train batch 1 metrics update 1.049 ms, 7.16 s total -Saving checkpoint at epoch 163 train batch 1 -[ 2023-09-21 11:25:45 ] Completed saving temp checkpoint 449.425 ms, 7.60 s total -[ 2023-09-21 11:25:45 ] Completed replacing temp checkpoint with checkpoint 21.801 ms, 7.63 s total -Epoch [163] :: gen_loss: 0.415, disc_loss: 0.016 -[ 2023-09-21 11:25:45 ] Completed training generator for epoch 163 53.046 ms, 7.68 s total -[ 2023-09-21 11:25:45 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 163 from eval step 0 - -[ 2023-09-21 11:25:46 ] Completed eval batch 0 to device 1,152.409 ms, 1.15 s total -[ 2023-09-21 11:25:46 ] Completed eval batch 0 forward 31.038 ms, 1.18 s total -[ 2023-09-21 11:25:46 ] Completed eval batch 0 recons_loss 0.398 ms, 1.18 s total -[ 2023-09-21 11:25:46 ] Completed eval batch 0 metrics update 0.740 ms, 1.18 s total -Saving checkpoint at epoch 163 val batch 0 -[ 2023-09-21 11:25:46 ] Completed saving temp checkpoint 445.603 ms, 1.63 s total -[ 2023-09-21 11:25:46 ] Completed replacing temp checkpoint with checkpoint 31.427 ms, 1.66 s total -[ 2023-09-21 11:25:47 ] Completed eval batch 1 to device 366.416 ms, 2.03 s total -[ 2023-09-21 11:25:47 ] Completed eval batch 1 forward 23.067 ms, 2.05 s total -[ 2023-09-21 11:25:47 ] Completed eval batch 1 recons_loss 0.340 ms, 2.05 s total -[ 2023-09-21 11:25:47 ] Completed eval batch 1 metrics update 223.272 ms, 2.27 s total -Saving checkpoint at epoch 163 val batch 1 -Epoch 163 val loss: 0.0557 -[ 2023-09-21 11:25:47 ] Completed saving temp checkpoint 454.558 ms, 2.73 s total -[ 2023-09-21 11:25:47 ] Completed replacing temp checkpoint with checkpoint 25.107 ms, 2.75 s total -[ 2023-09-21 11:25:47 ] Completed evaluating generator for epoch 163 56.003 ms, 2.81 s total - - -EPOCH :: 164 - - -[ 2023-09-21 11:25:47 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 164 from training step 0 - -[ 2023-09-21 11:25:51 ] Completed train batch 0 to device 3,156.906 ms, 3.16 s total -[ 2023-09-21 11:25:51 ] Completed train batch 0 generator forward 26.668 ms, 3.18 s total -[ 2023-09-21 11:25:51 ] Completed train batch 0 generator loss: 1.236 66.851 ms, 3.25 s total -[ 2023-09-21 11:25:51 ] Completed train batch 0 generator backward 558.601 ms, 3.81 s total -[ 2023-09-21 11:25:51 ] Completed train batch 0 discriminator loss 0.047 61.505 ms, 3.87 s total -[ 2023-09-21 11:25:51 ] Completed train batch 0 discriminator backward 96.898 ms, 3.97 s total -Epoch [164] Step [0/2], gen_loss: 0.418, disc_loss: 0.015 -[ 2023-09-21 11:25:51 ] Completed train batch 0 metrics update 7.771 ms, 3.98 s total -Saving checkpoint at epoch 164 train batch 0 -[ 2023-09-21 11:25:52 ] Completed saving temp checkpoint 442.070 ms, 4.42 s total -[ 2023-09-21 11:25:52 ] Completed replacing temp checkpoint with checkpoint 22.905 ms, 4.44 s total -[ 2023-09-21 11:25:53 ] Completed train batch 1 to device 1,620.735 ms, 6.06 s total -[ 2023-09-21 11:25:53 ] Completed train batch 1 generator forward 22.947 ms, 6.08 s total -[ 2023-09-21 11:25:54 ] Completed train batch 1 generator loss: 1.196 65.168 ms, 6.15 s total -[ 2023-09-21 11:25:55 ] Completed train batch 1 generator backward 1,123.995 ms, 7.27 s total -[ 2023-09-21 11:25:55 ] Completed train batch 1 discriminator loss 0.045 56.500 ms, 7.33 s total -[ 2023-09-21 11:25:55 ] Completed train batch 1 discriminator backward 72.897 ms, 7.40 s total -Epoch [164] Step [1/2], gen_loss: 0.411, disc_loss: 0.015 -[ 2023-09-21 11:25:55 ] Completed train batch 1 metrics update 5.189 ms, 7.41 s total -Saving checkpoint at epoch 164 train batch 1 -[ 2023-09-21 11:25:55 ] Completed saving temp checkpoint 473.408 ms, 7.88 s total -[ 2023-09-21 11:25:55 ] Completed replacing temp checkpoint with checkpoint 28.004 ms, 7.91 s total -Epoch [164] :: gen_loss: 0.415, disc_loss: 0.015 -[ 2023-09-21 11:25:55 ] Completed training generator for epoch 164 60.606 ms, 7.97 s total -[ 2023-09-21 11:25:55 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 164 from eval step 0 - -[ 2023-09-21 11:25:56 ] Completed eval batch 0 to device 1,075.665 ms, 1.08 s total -[ 2023-09-21 11:25:56 ] Completed eval batch 0 forward 25.060 ms, 1.10 s total -[ 2023-09-21 11:25:56 ] Completed eval batch 0 recons_loss 0.346 ms, 1.10 s total -[ 2023-09-21 11:25:56 ] Completed eval batch 0 metrics update 0.628 ms, 1.10 s total -Saving checkpoint at epoch 164 val batch 0 -[ 2023-09-21 11:25:57 ] Completed saving temp checkpoint 413.336 ms, 1.52 s total -[ 2023-09-21 11:25:57 ] Completed replacing temp checkpoint with checkpoint 34.011 ms, 1.55 s total -[ 2023-09-21 11:25:57 ] Completed eval batch 1 to device 343.413 ms, 1.89 s total -[ 2023-09-21 11:25:57 ] Completed eval batch 1 forward 28.974 ms, 1.92 s total -[ 2023-09-21 11:25:57 ] Completed eval batch 1 recons_loss 0.334 ms, 1.92 s total -[ 2023-09-21 11:25:58 ] Completed eval batch 1 metrics update 316.485 ms, 2.24 s total -Saving checkpoint at epoch 164 val batch 1 -Epoch 164 val loss: 0.0610 -[ 2023-09-21 11:25:58 ] Completed saving temp checkpoint 395.672 ms, 2.63 s total -[ 2023-09-21 11:25:58 ] Completed replacing temp checkpoint with checkpoint 21.931 ms, 2.66 s total -[ 2023-09-21 11:25:58 ] Completed evaluating generator for epoch 164 48.680 ms, 2.70 s total - - -EPOCH :: 165 - - -[ 2023-09-21 11:25:58 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 165 from training step 0 - -[ 2023-09-21 11:26:01 ] Completed train batch 0 to device 3,125.896 ms, 3.13 s total -[ 2023-09-21 11:26:01 ] Completed train batch 0 generator forward 31.986 ms, 3.16 s total -[ 2023-09-21 11:26:01 ] Completed train batch 0 generator loss: 1.224 66.639 ms, 3.22 s total -[ 2023-09-21 11:26:02 ] Completed train batch 0 generator backward 682.725 ms, 3.91 s total -[ 2023-09-21 11:26:02 ] Completed train batch 0 discriminator loss 0.044 57.713 ms, 3.96 s total -[ 2023-09-21 11:26:02 ] Completed train batch 0 discriminator backward 106.534 ms, 4.07 s total -Epoch [165] Step [0/2], gen_loss: 0.420, disc_loss: 0.015 -[ 2023-09-21 11:26:02 ] Completed train batch 0 metrics update 4.476 ms, 4.08 s total -Saving checkpoint at epoch 165 train batch 0 -[ 2023-09-21 11:26:03 ] Completed saving temp checkpoint 450.911 ms, 4.53 s total -[ 2023-09-21 11:26:03 ] Completed replacing temp checkpoint with checkpoint 33.096 ms, 4.56 s total -[ 2023-09-21 11:26:04 ] Completed train batch 1 to device 1,376.057 ms, 5.94 s total -[ 2023-09-21 11:26:04 ] Completed train batch 1 generator forward 41.296 ms, 5.98 s total -[ 2023-09-21 11:26:04 ] Completed train batch 1 generator loss: 1.241 64.550 ms, 6.04 s total -[ 2023-09-21 11:26:05 ] Completed train batch 1 generator backward 1,183.704 ms, 7.23 s total -[ 2023-09-21 11:26:05 ] Completed train batch 1 discriminator loss 0.046 56.457 ms, 7.28 s total -[ 2023-09-21 11:26:05 ] Completed train batch 1 discriminator backward 75.709 ms, 7.36 s total -Epoch [165] Step [1/2], gen_loss: 0.420, disc_loss: 0.015 -[ 2023-09-21 11:26:05 ] Completed train batch 1 metrics update 2.259 ms, 7.36 s total -Saving checkpoint at epoch 165 train batch 1 -[ 2023-09-21 11:26:06 ] Completed saving temp checkpoint 680.522 ms, 8.04 s total -[ 2023-09-21 11:26:06 ] Completed replacing temp checkpoint with checkpoint 35.514 ms, 8.08 s total -Epoch [165] :: gen_loss: 0.420, disc_loss: 0.015 -[ 2023-09-21 11:26:06 ] Completed training generator for epoch 165 60.895 ms, 8.14 s total -[ 2023-09-21 11:26:06 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 165 from eval step 0 - -[ 2023-09-21 11:26:07 ] Completed eval batch 0 to device 1,076.073 ms, 1.08 s total -[ 2023-09-21 11:26:07 ] Completed eval batch 0 forward 40.099 ms, 1.12 s total -[ 2023-09-21 11:26:07 ] Completed eval batch 0 recons_loss 0.333 ms, 1.12 s total -[ 2023-09-21 11:26:07 ] Completed eval batch 0 metrics update 0.715 ms, 1.12 s total -Saving checkpoint at epoch 165 val batch 0 -[ 2023-09-21 11:26:08 ] Completed saving temp checkpoint 627.038 ms, 1.74 s total -[ 2023-09-21 11:26:08 ] Completed replacing temp checkpoint with checkpoint 41.649 ms, 1.79 s total -[ 2023-09-21 11:26:08 ] Completed eval batch 1 to device 96.195 ms, 1.88 s total -[ 2023-09-21 11:26:08 ] Completed eval batch 1 forward 38.081 ms, 1.92 s total -[ 2023-09-21 11:26:08 ] Completed eval batch 1 recons_loss 0.340 ms, 1.92 s total -[ 2023-09-21 11:26:08 ] Completed eval batch 1 metrics update 98.824 ms, 2.02 s total -Saving checkpoint at epoch 165 val batch 1 -Epoch 165 val loss: 0.0547 -[ 2023-09-21 11:26:09 ] Completed saving temp checkpoint 665.220 ms, 2.68 s total -[ 2023-09-21 11:26:09 ] Completed replacing temp checkpoint with checkpoint 35.386 ms, 2.72 s total -[ 2023-09-21 11:26:09 ] Completed evaluating generator for epoch 165 55.245 ms, 2.78 s total - - -EPOCH :: 166 - - -[ 2023-09-21 11:26:09 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 166 from training step 0 - -[ 2023-09-21 11:26:12 ] Completed train batch 0 to device 2,840.637 ms, 2.84 s total -[ 2023-09-21 11:26:12 ] Completed train batch 0 generator forward 26.157 ms, 2.87 s total -[ 2023-09-21 11:26:12 ] Completed train batch 0 generator loss: 1.191 66.406 ms, 2.93 s total -[ 2023-09-21 11:26:13 ] Completed train batch 0 generator backward 646.970 ms, 3.58 s total -[ 2023-09-21 11:26:13 ] Completed train batch 0 discriminator loss 0.048 57.232 ms, 3.64 s total -[ 2023-09-21 11:26:13 ] Completed train batch 0 discriminator backward 105.040 ms, 3.74 s total -Epoch [166] Step [0/2], gen_loss: 0.405, disc_loss: 0.016 -[ 2023-09-21 11:26:13 ] Completed train batch 0 metrics update 4.225 ms, 3.75 s total -Saving checkpoint at epoch 166 train batch 0 -[ 2023-09-21 11:26:13 ] Completed saving temp checkpoint 670.999 ms, 4.42 s total -[ 2023-09-21 11:26:13 ] Completed replacing temp checkpoint with checkpoint 36.478 ms, 4.45 s total -[ 2023-09-21 11:26:15 ] Completed train batch 1 to device 1,120.810 ms, 5.57 s total -[ 2023-09-21 11:26:15 ] Completed train batch 1 generator forward 22.445 ms, 5.60 s total -[ 2023-09-21 11:26:15 ] Completed train batch 1 generator loss: 1.260 63.742 ms, 5.66 s total -[ 2023-09-21 11:26:16 ] Completed train batch 1 generator backward 1,525.919 ms, 7.19 s total -[ 2023-09-21 11:26:16 ] Completed train batch 1 discriminator loss 0.055 56.548 ms, 7.24 s total -[ 2023-09-21 11:26:16 ] Completed train batch 1 discriminator backward 74.737 ms, 7.32 s total -Epoch [166] Step [1/2], gen_loss: 0.411, disc_loss: 0.016 -[ 2023-09-21 11:26:16 ] Completed train batch 1 metrics update 2.162 ms, 7.32 s total -Saving checkpoint at epoch 166 train batch 1 -[ 2023-09-21 11:26:17 ] Completed saving temp checkpoint 694.514 ms, 8.02 s total -[ 2023-09-21 11:26:17 ] Completed replacing temp checkpoint with checkpoint 40.289 ms, 8.06 s total -Epoch [166] :: gen_loss: 0.408, disc_loss: 0.016 -[ 2023-09-21 11:26:17 ] Completed training generator for epoch 166 56.289 ms, 8.11 s total -[ 2023-09-21 11:26:17 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 166 from eval step 0 - -[ 2023-09-21 11:26:18 ] Completed eval batch 0 to device 1,100.985 ms, 1.10 s total -[ 2023-09-21 11:26:18 ] Completed eval batch 0 forward 28.571 ms, 1.13 s total -[ 2023-09-21 11:26:18 ] Completed eval batch 0 recons_loss 0.325 ms, 1.13 s total -[ 2023-09-21 11:26:18 ] Completed eval batch 0 metrics update 0.660 ms, 1.13 s total -Saving checkpoint at epoch 166 val batch 0 -[ 2023-09-21 11:26:19 ] Completed saving temp checkpoint 687.935 ms, 1.82 s total -[ 2023-09-21 11:26:19 ] Completed replacing temp checkpoint with checkpoint 43.884 ms, 1.86 s total -[ 2023-09-21 11:26:19 ] Completed eval batch 1 to device 61.283 ms, 1.92 s total -[ 2023-09-21 11:26:19 ] Completed eval batch 1 forward 35.905 ms, 1.96 s total -[ 2023-09-21 11:26:19 ] Completed eval batch 1 recons_loss 0.349 ms, 1.96 s total -[ 2023-09-21 11:26:19 ] Completed eval batch 1 metrics update 68.720 ms, 2.03 s total -Saving checkpoint at epoch 166 val batch 1 -Epoch 166 val loss: 0.0544 -[ 2023-09-21 11:26:20 ] Completed saving temp checkpoint 671.195 ms, 2.70 s total -[ 2023-09-21 11:26:20 ] Completed replacing temp checkpoint with checkpoint 29.609 ms, 2.73 s total -[ 2023-09-21 11:26:20 ] Completed evaluating generator for epoch 166 59.050 ms, 2.79 s total - - -EPOCH :: 167 - - -[ 2023-09-21 11:26:20 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 167 from training step 0 - -[ 2023-09-21 11:26:23 ] Completed train batch 0 to device 2,869.917 ms, 2.87 s total -[ 2023-09-21 11:26:23 ] Completed train batch 0 generator forward 32.227 ms, 2.90 s total -[ 2023-09-21 11:26:23 ] Completed train batch 0 generator loss: 1.225 66.510 ms, 2.97 s total -[ 2023-09-21 11:26:23 ] Completed train batch 0 generator backward 632.250 ms, 3.60 s total -[ 2023-09-21 11:26:24 ] Completed train batch 0 discriminator loss 0.046 57.407 ms, 3.66 s total -[ 2023-09-21 11:26:24 ] Completed train batch 0 discriminator backward 120.961 ms, 3.78 s total -Epoch [167] Step [0/2], gen_loss: 0.412, disc_loss: 0.016 -[ 2023-09-21 11:26:24 ] Completed train batch 0 metrics update 6.131 ms, 3.79 s total -Saving checkpoint at epoch 167 train batch 0 -[ 2023-09-21 11:26:24 ] Completed saving temp checkpoint 633.592 ms, 4.42 s total -[ 2023-09-21 11:26:24 ] Completed replacing temp checkpoint with checkpoint 34.985 ms, 4.45 s total -[ 2023-09-21 11:26:25 ] Completed train batch 1 to device 1,128.124 ms, 5.58 s total -[ 2023-09-21 11:26:26 ] Completed train batch 1 generator forward 32.490 ms, 5.61 s total -[ 2023-09-21 11:26:26 ] Completed train batch 1 generator loss: 1.203 64.510 ms, 5.68 s total -[ 2023-09-21 11:26:27 ] Completed train batch 1 generator backward 1,371.315 ms, 7.05 s total -[ 2023-09-21 11:26:27 ] Completed train batch 1 discriminator loss 0.045 56.699 ms, 7.11 s total -[ 2023-09-21 11:26:27 ] Completed train batch 1 discriminator backward 90.698 ms, 7.20 s total -Epoch [167] Step [1/2], gen_loss: 0.411, disc_loss: 0.015 -[ 2023-09-21 11:26:27 ] Completed train batch 1 metrics update 4.066 ms, 7.20 s total -Saving checkpoint at epoch 167 train batch 1 -[ 2023-09-21 11:26:28 ] Completed saving temp checkpoint 685.223 ms, 7.89 s total -[ 2023-09-21 11:26:28 ] Completed replacing temp checkpoint with checkpoint 28.059 ms, 7.92 s total -Epoch [167] :: gen_loss: 0.411, disc_loss: 0.016 -[ 2023-09-21 11:26:28 ] Completed training generator for epoch 167 54.872 ms, 7.97 s total -[ 2023-09-21 11:26:28 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 167 from eval step 0 - -[ 2023-09-21 11:26:29 ] Completed eval batch 0 to device 1,048.089 ms, 1.05 s total -[ 2023-09-21 11:26:29 ] Completed eval batch 0 forward 24.836 ms, 1.07 s total -[ 2023-09-21 11:26:29 ] Completed eval batch 0 recons_loss 0.320 ms, 1.07 s total -[ 2023-09-21 11:26:29 ] Completed eval batch 0 metrics update 0.565 ms, 1.07 s total -Saving checkpoint at epoch 167 val batch 0 -[ 2023-09-21 11:26:29 ] Completed saving temp checkpoint 478.496 ms, 1.55 s total -[ 2023-09-21 11:26:29 ] Completed replacing temp checkpoint with checkpoint 32.605 ms, 1.58 s total -[ 2023-09-21 11:26:30 ] Completed eval batch 1 to device 267.063 ms, 1.85 s total -[ 2023-09-21 11:26:30 ] Completed eval batch 1 forward 40.766 ms, 1.89 s total -[ 2023-09-21 11:26:30 ] Completed eval batch 1 recons_loss 0.511 ms, 1.89 s total -[ 2023-09-21 11:26:30 ] Completed eval batch 1 metrics update 144.054 ms, 2.04 s total -Saving checkpoint at epoch 167 val batch 1 -Epoch 167 val loss: 0.0567 -[ 2023-09-21 11:26:30 ] Completed saving temp checkpoint 417.111 ms, 2.45 s total -[ 2023-09-21 11:26:30 ] Completed replacing temp checkpoint with checkpoint 21.899 ms, 2.48 s total -[ 2023-09-21 11:26:30 ] Completed evaluating generator for epoch 167 51.508 ms, 2.53 s total - - -EPOCH :: 168 - - -[ 2023-09-21 11:26:30 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 168 from training step 0 - -[ 2023-09-21 11:26:34 ] Completed train batch 0 to device 3,164.026 ms, 3.16 s total -[ 2023-09-21 11:26:34 ] Completed train batch 0 generator forward 37.776 ms, 3.20 s total -[ 2023-09-21 11:26:34 ] Completed train batch 0 generator loss: 1.252 71.726 ms, 3.27 s total -[ 2023-09-21 11:26:34 ] Completed train batch 0 generator backward 570.394 ms, 3.84 s total -[ 2023-09-21 11:26:34 ] Completed train batch 0 discriminator loss 0.046 72.625 ms, 3.92 s total -[ 2023-09-21 11:26:34 ] Completed train batch 0 discriminator backward 81.835 ms, 4.00 s total -Epoch [168] Step [0/2], gen_loss: 0.411, disc_loss: 0.015 -[ 2023-09-21 11:26:34 ] Completed train batch 0 metrics update 4.206 ms, 4.00 s total -Saving checkpoint at epoch 168 train batch 0 -[ 2023-09-21 11:26:35 ] Completed saving temp checkpoint 404.835 ms, 4.41 s total -[ 2023-09-21 11:26:35 ] Completed replacing temp checkpoint with checkpoint 26.809 ms, 4.43 s total -[ 2023-09-21 11:26:37 ] Completed train batch 1 to device 1,734.552 ms, 6.17 s total -[ 2023-09-21 11:26:37 ] Completed train batch 1 generator forward 34.731 ms, 6.20 s total -[ 2023-09-21 11:26:37 ] Completed train batch 1 generator loss: 1.201 72.537 ms, 6.28 s total -[ 2023-09-21 11:26:38 ] Completed train batch 1 generator backward 1,111.451 ms, 7.39 s total -[ 2023-09-21 11:26:38 ] Completed train batch 1 discriminator loss 0.046 56.927 ms, 7.44 s total -[ 2023-09-21 11:26:38 ] Completed train batch 1 discriminator backward 77.429 ms, 7.52 s total -Epoch [168] Step [1/2], gen_loss: 0.402, disc_loss: 0.015 -[ 2023-09-21 11:26:38 ] Completed train batch 1 metrics update 1.550 ms, 7.52 s total -Saving checkpoint at epoch 168 train batch 1 -[ 2023-09-21 11:26:38 ] Completed saving temp checkpoint 440.375 ms, 7.96 s total -[ 2023-09-21 11:26:38 ] Completed replacing temp checkpoint with checkpoint 23.686 ms, 7.99 s total -Epoch [168] :: gen_loss: 0.406, disc_loss: 0.015 -[ 2023-09-21 11:26:38 ] Completed training generator for epoch 168 58.996 ms, 8.05 s total -[ 2023-09-21 11:26:38 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 168 from eval step 0 - -[ 2023-09-21 11:26:40 ] Completed eval batch 0 to device 1,128.753 ms, 1.13 s total -[ 2023-09-21 11:26:40 ] Completed eval batch 0 forward 29.081 ms, 1.16 s total -[ 2023-09-21 11:26:40 ] Completed eval batch 0 recons_loss 0.267 ms, 1.16 s total -[ 2023-09-21 11:26:40 ] Completed eval batch 0 metrics update 0.535 ms, 1.16 s total -Saving checkpoint at epoch 168 val batch 0 -[ 2023-09-21 11:26:40 ] Completed saving temp checkpoint 586.971 ms, 1.75 s total -[ 2023-09-21 11:26:40 ] Completed replacing temp checkpoint with checkpoint 17.180 ms, 1.76 s total -[ 2023-09-21 11:26:40 ] Completed eval batch 1 to device 262.178 ms, 2.02 s total -[ 2023-09-21 11:26:40 ] Completed eval batch 1 forward 22.001 ms, 2.05 s total -[ 2023-09-21 11:26:40 ] Completed eval batch 1 recons_loss 0.345 ms, 2.05 s total -[ 2023-09-21 11:26:41 ] Completed eval batch 1 metrics update 187.725 ms, 2.24 s total -Saving checkpoint at epoch 168 val batch 1 -Epoch 168 val loss: 0.0548 -[ 2023-09-21 11:26:41 ] Completed saving temp checkpoint 410.879 ms, 2.65 s total -[ 2023-09-21 11:26:41 ] Completed replacing temp checkpoint with checkpoint 24.957 ms, 2.67 s total -[ 2023-09-21 11:26:41 ] Completed evaluating generator for epoch 168 62.101 ms, 2.73 s total - - -EPOCH :: 169 - - -[ 2023-09-21 11:26:41 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 169 from training step 0 - -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 11:40:59 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 11:40:59 ] Completed importing Timer 0.024 ms, 0.00 s total -[ 2023-09-21 11:41:04 ] Completed importing everything else 4,648.372 ms, 4.65 s total -| distributed init (rank 2): env:// -| distributed init (rank 5): env:// -| distributed init (rank 1): env:// -| distributed init (rank 3): env:// -| distributed init (rank 4): env:// -| distributed init (rank 0): env:// -[ 2023-09-21 11:41:11 ] Completed preliminaries 7,856.384 ms, 12.50 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 11:41:12 ] Completed build datasets 11.608 ms, 12.52 s total -[ 2023-09-21 11:41:12 ] Completed build samplers 0.044 ms, 12.52 s total -[ 2023-09-21 11:41:12 ] Completed build dataloaders 0.139 ms, 12.52 s total -[ 2023-09-21 11:41:12 ] Completed generator to device 78.638 ms, 12.60 s total -[ 2023-09-21 11:41:12 ] Completed discriminator to device 7.244 ms, 12.60 s total -[ 2023-09-21 11:41:12 ] Completed loss functions 382.366 ms, 12.98 s total -[ 2023-09-21 11:41:12 ] Completed models prepped for distribution 102.065 ms, 13.09 s total -[ 2023-09-21 11:41:12 ] Completed optimizers 0.680 ms, 13.09 s total -[ 2023-09-21 11:41:12 ] Completed grad scalers 0.029 ms, 13.09 s total -[ 2023-09-21 11:41:13 ] Completed checkpoint retrieval 519.796 ms, 13.61 s total - - -EPOCH :: 168 - - -[ 2023-09-21 11:41:13 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 168 from training step 2 - -Epoch [168] :: gen_loss: 0.406, disc_loss: 0.015 -[ 2023-09-21 11:41:13 ] Completed training generator for epoch 168 109.812 ms, 0.11 s total -[ 2023-09-21 11:41:13 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 168 from eval step 2 - -[ 2023-09-21 11:41:13 ] Completed evaluating generator for epoch 168 121.936 ms, 0.12 s total - - -EPOCH :: 169 - - -[ 2023-09-21 11:41:13 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 169 from training step 0 - -[ 2023-09-21 11:41:16 ] Completed train batch 0 to device 3,485.993 ms, 3.49 s total -[ 2023-09-21 11:41:18 ] Completed train batch 0 generator forward 1,536.519 ms, 5.02 s total -[ 2023-09-21 11:41:18 ] Completed train batch 0 generator loss: 1.239 92.599 ms, 5.12 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 11:41:20 ] Completed train batch 0 generator backward 1,555.021 ms, 6.67 s total -[ 2023-09-21 11:41:20 ] Completed train batch 0 discriminator loss 0.046 56.930 ms, 6.73 s total -[ 2023-09-21 11:41:20 ] Completed train batch 0 discriminator backward 84.578 ms, 6.81 s total -Epoch [169] Step [0/2], gen_loss: 0.404, disc_loss: 0.016 -[ 2023-09-21 11:41:20 ] Completed train batch 0 metrics update 19.441 ms, 6.83 s total -Saving checkpoint at epoch 169 train batch 0 -[ 2023-09-21 11:41:20 ] Completed saving temp checkpoint 454.433 ms, 7.29 s total -[ 2023-09-21 11:41:20 ] Completed replacing temp checkpoint with checkpoint 120.329 ms, 7.41 s total -[ 2023-09-21 11:41:20 ] Completed train batch 1 to device 20.052 ms, 7.43 s total -[ 2023-09-21 11:41:20 ] Completed train batch 1 generator forward 25.062 ms, 7.45 s total -[ 2023-09-21 11:41:20 ] Completed train batch 1 generator loss: 1.180 62.879 ms, 7.51 s total -[ 2023-09-21 11:41:22 ] Completed train batch 1 generator backward 1,325.656 ms, 8.84 s total -[ 2023-09-21 11:41:22 ] Completed train batch 1 discriminator loss 0.047 56.547 ms, 8.90 s total -[ 2023-09-21 11:41:22 ] Completed train batch 1 discriminator backward 80.041 ms, 8.98 s total -Epoch [169] Step [1/2], gen_loss: 0.404, disc_loss: 0.016 -[ 2023-09-21 11:41:22 ] Completed train batch 1 metrics update 1.553 ms, 8.98 s total -Saving checkpoint at epoch 169 train batch 1 -[ 2023-09-21 11:41:22 ] Completed saving temp checkpoint 447.335 ms, 9.42 s total -[ 2023-09-21 11:41:22 ] Completed replacing temp checkpoint with checkpoint 22.635 ms, 9.45 s total -Epoch [169] :: gen_loss: 0.404, disc_loss: 0.016 -[ 2023-09-21 11:41:22 ] Completed training generator for epoch 169 36.249 ms, 9.48 s total -[ 2023-09-21 11:41:22 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 169 from eval step 0 - -[ 2023-09-21 11:41:23 ] Completed eval batch 0 to device 1,160.101 ms, 1.16 s total -[ 2023-09-21 11:41:24 ] Completed eval batch 0 forward 25.846 ms, 1.19 s total -[ 2023-09-21 11:41:24 ] Completed eval batch 0 recons_loss 0.368 ms, 1.19 s total -[ 2023-09-21 11:41:24 ] Completed eval batch 0 metrics update 0.653 ms, 1.19 s total -Saving checkpoint at epoch 169 val batch 0 -[ 2023-09-21 11:41:24 ] Completed saving temp checkpoint 420.058 ms, 1.61 s total -[ 2023-09-21 11:41:24 ] Completed replacing temp checkpoint with checkpoint 30.611 ms, 1.64 s total -[ 2023-09-21 11:41:24 ] Completed eval batch 1 to device 414.237 ms, 2.05 s total -[ 2023-09-21 11:41:24 ] Completed eval batch 1 forward 22.021 ms, 2.07 s total -[ 2023-09-21 11:41:24 ] Completed eval batch 1 recons_loss 0.360 ms, 2.07 s total -[ 2023-09-21 11:41:25 ] Completed eval batch 1 metrics update 253.209 ms, 2.33 s total -Saving checkpoint at epoch 169 val batch 1 -Epoch 169 val loss: 0.0562 -[ 2023-09-21 11:41:25 ] Completed saving temp checkpoint 465.446 ms, 2.79 s total -[ 2023-09-21 11:41:25 ] Completed replacing temp checkpoint with checkpoint 31.004 ms, 2.82 s total -[ 2023-09-21 11:41:25 ] Completed evaluating generator for epoch 169 50.821 ms, 2.87 s total - - -EPOCH :: 170 - - -[ 2023-09-21 11:41:25 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 170 from training step 0 - -[ 2023-09-21 11:41:28 ] Completed train batch 0 to device 2,872.545 ms, 2.87 s total -[ 2023-09-21 11:41:28 ] Completed train batch 0 generator forward 41.665 ms, 2.91 s total -[ 2023-09-21 11:41:28 ] Completed train batch 0 generator loss: 1.181 65.086 ms, 2.98 s total -[ 2023-09-21 11:41:29 ] Completed train batch 0 generator backward 924.455 ms, 3.90 s total -[ 2023-09-21 11:41:29 ] Completed train batch 0 discriminator loss 0.048 78.923 ms, 3.98 s total -[ 2023-09-21 11:41:29 ] Completed train batch 0 discriminator backward 68.686 ms, 4.05 s total -Epoch [170] Step [0/2], gen_loss: 0.397, disc_loss: 0.016 -[ 2023-09-21 11:41:29 ] Completed train batch 0 metrics update 9.333 ms, 4.06 s total -Saving checkpoint at epoch 170 train batch 0 -[ 2023-09-21 11:41:30 ] Completed saving temp checkpoint 433.949 ms, 4.49 s total -[ 2023-09-21 11:41:30 ] Completed replacing temp checkpoint with checkpoint 32.119 ms, 4.53 s total -[ 2023-09-21 11:41:31 ] Completed train batch 1 to device 1,121.554 ms, 5.65 s total -[ 2023-09-21 11:41:31 ] Completed train batch 1 generator forward 41.710 ms, 5.69 s total -[ 2023-09-21 11:41:31 ] Completed train batch 1 generator loss: 1.243 62.334 ms, 5.75 s total -[ 2023-09-21 11:41:33 ] Completed train batch 1 generator backward 1,849.906 ms, 7.60 s total -[ 2023-09-21 11:41:33 ] Completed train batch 1 discriminator loss 0.049 59.353 ms, 7.66 s total -[ 2023-09-21 11:41:33 ] Completed train batch 1 discriminator backward 59.574 ms, 7.72 s total -Epoch [170] Step [1/2], gen_loss: 0.401, disc_loss: 0.016 -[ 2023-09-21 11:41:33 ] Completed train batch 1 metrics update 5.229 ms, 7.73 s total -Saving checkpoint at epoch 170 train batch 1 -[ 2023-09-21 11:41:33 ] Completed saving temp checkpoint 430.513 ms, 8.16 s total -[ 2023-09-21 11:41:33 ] Completed replacing temp checkpoint with checkpoint 27.721 ms, 8.18 s total -Epoch [170] :: gen_loss: 0.399, disc_loss: 0.016 -[ 2023-09-21 11:41:33 ] Completed training generator for epoch 170 60.305 ms, 8.24 s total -[ 2023-09-21 11:41:33 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 170 from eval step 0 - -[ 2023-09-21 11:41:35 ] Completed eval batch 0 to device 1,168.538 ms, 1.17 s total -[ 2023-09-21 11:41:35 ] Completed eval batch 0 forward 43.054 ms, 1.21 s total -[ 2023-09-21 11:41:35 ] Completed eval batch 0 recons_loss 0.326 ms, 1.21 s total -[ 2023-09-21 11:41:35 ] Completed eval batch 0 metrics update 0.626 ms, 1.21 s total -Saving checkpoint at epoch 170 val batch 0 -[ 2023-09-21 11:41:35 ] Completed saving temp checkpoint 406.886 ms, 1.62 s total -[ 2023-09-21 11:41:35 ] Completed replacing temp checkpoint with checkpoint 23.977 ms, 1.64 s total -[ 2023-09-21 11:41:35 ] Completed eval batch 1 to device 417.337 ms, 2.06 s total -[ 2023-09-21 11:41:36 ] Completed eval batch 1 forward 39.745 ms, 2.10 s total -[ 2023-09-21 11:41:36 ] Completed eval batch 1 recons_loss 0.328 ms, 2.10 s total -[ 2023-09-21 11:41:36 ] Completed eval batch 1 metrics update 202.466 ms, 2.30 s total -Saving checkpoint at epoch 170 val batch 1 -Epoch 170 val loss: 0.0601 -[ 2023-09-21 11:41:36 ] Completed saving temp checkpoint 449.920 ms, 2.75 s total -[ 2023-09-21 11:41:36 ] Completed replacing temp checkpoint with checkpoint 17.849 ms, 2.77 s total -[ 2023-09-21 11:41:36 ] Completed evaluating generator for epoch 170 47.031 ms, 2.82 s total - - -EPOCH :: 171 - - -[ 2023-09-21 11:41:36 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 171 from training step 0 - -[ 2023-09-21 11:41:39 ] Completed train batch 0 to device 3,135.607 ms, 3.14 s total -[ 2023-09-21 11:41:39 ] Completed train batch 0 generator forward 34.738 ms, 3.17 s total -[ 2023-09-21 11:41:39 ] Completed train batch 0 generator loss: 1.227 70.455 ms, 3.24 s total -[ 2023-09-21 11:41:40 ] Completed train batch 0 generator backward 750.686 ms, 3.99 s total -[ 2023-09-21 11:41:40 ] Completed train batch 0 discriminator loss 0.050 57.345 ms, 4.05 s total -[ 2023-09-21 11:41:40 ] Completed train batch 0 discriminator backward 83.589 ms, 4.13 s total -Epoch [171] Step [0/2], gen_loss: 0.404, disc_loss: 0.017 -[ 2023-09-21 11:41:40 ] Completed train batch 0 metrics update 12.744 ms, 4.15 s total -Saving checkpoint at epoch 171 train batch 0 -[ 2023-09-21 11:41:41 ] Completed saving temp checkpoint 426.930 ms, 4.57 s total -[ 2023-09-21 11:41:41 ] Completed replacing temp checkpoint with checkpoint 22.216 ms, 4.59 s total -[ 2023-09-21 11:41:42 ] Completed train batch 1 to device 1,440.698 ms, 6.04 s total -[ 2023-09-21 11:41:42 ] Completed train batch 1 generator forward 25.730 ms, 6.06 s total -[ 2023-09-21 11:41:42 ] Completed train batch 1 generator loss: 1.190 65.041 ms, 6.13 s total -[ 2023-09-21 11:41:44 ] Completed train batch 1 generator backward 1,509.776 ms, 7.64 s total -[ 2023-09-21 11:41:44 ] Completed train batch 1 discriminator loss 0.050 57.125 ms, 7.69 s total -[ 2023-09-21 11:41:44 ] Completed train batch 1 discriminator backward 74.227 ms, 7.77 s total -Epoch [171] Step [1/2], gen_loss: 0.400, disc_loss: 0.017 -[ 2023-09-21 11:41:44 ] Completed train batch 1 metrics update 5.456 ms, 7.77 s total -Saving checkpoint at epoch 171 train batch 1 -[ 2023-09-21 11:41:44 ] Completed saving temp checkpoint 447.719 ms, 8.22 s total -[ 2023-09-21 11:41:44 ] Completed replacing temp checkpoint with checkpoint 20.580 ms, 8.24 s total -Epoch [171] :: gen_loss: 0.402, disc_loss: 0.017 -[ 2023-09-21 11:41:45 ] Completed training generator for epoch 171 52.815 ms, 8.29 s total -[ 2023-09-21 11:41:45 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 171 from eval step 0 - -[ 2023-09-21 11:41:46 ] Completed eval batch 0 to device 1,118.334 ms, 1.12 s total -[ 2023-09-21 11:41:46 ] Completed eval batch 0 forward 30.652 ms, 1.15 s total -[ 2023-09-21 11:41:46 ] Completed eval batch 0 recons_loss 0.232 ms, 1.15 s total -[ 2023-09-21 11:41:46 ] Completed eval batch 0 metrics update 0.502 ms, 1.15 s total -Saving checkpoint at epoch 171 val batch 0 -[ 2023-09-21 11:41:46 ] Completed saving temp checkpoint 430.765 ms, 1.58 s total -[ 2023-09-21 11:41:46 ] Completed replacing temp checkpoint with checkpoint 21.486 ms, 1.60 s total -[ 2023-09-21 11:41:47 ] Completed eval batch 1 to device 427.644 ms, 2.03 s total -[ 2023-09-21 11:41:47 ] Completed eval batch 1 forward 40.235 ms, 2.07 s total -[ 2023-09-21 11:41:47 ] Completed eval batch 1 recons_loss 0.379 ms, 2.07 s total -[ 2023-09-21 11:41:47 ] Completed eval batch 1 metrics update 183.428 ms, 2.25 s total -Saving checkpoint at epoch 171 val batch 1 -Epoch 171 val loss: 0.0552 -[ 2023-09-21 11:41:47 ] Completed saving temp checkpoint 458.372 ms, 2.71 s total -[ 2023-09-21 11:41:47 ] Completed replacing temp checkpoint with checkpoint 19.880 ms, 2.73 s total -[ 2023-09-21 11:41:47 ] Completed evaluating generator for epoch 171 47.259 ms, 2.78 s total - - -EPOCH :: 172 - - -[ 2023-09-21 11:41:47 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 172 from training step 0 - -[ 2023-09-21 11:41:51 ] Completed train batch 0 to device 3,208.948 ms, 3.21 s total -[ 2023-09-21 11:41:51 ] Completed train batch 0 generator forward 27.690 ms, 3.24 s total -[ 2023-09-21 11:41:51 ] Completed train batch 0 generator loss: 1.238 66.448 ms, 3.30 s total -[ 2023-09-21 11:41:51 ] Completed train batch 0 generator backward 569.341 ms, 3.87 s total -[ 2023-09-21 11:41:51 ] Completed train batch 0 discriminator loss 0.051 58.379 ms, 3.93 s total -[ 2023-09-21 11:41:51 ] Completed train batch 0 discriminator backward 96.484 ms, 4.03 s total -Epoch [172] Step [0/2], gen_loss: 0.403, disc_loss: 0.017 -[ 2023-09-21 11:41:51 ] Completed train batch 0 metrics update 1.866 ms, 4.03 s total -Saving checkpoint at epoch 172 train batch 0 -[ 2023-09-21 11:41:52 ] Completed saving temp checkpoint 686.612 ms, 4.72 s total -[ 2023-09-21 11:41:52 ] Completed replacing temp checkpoint with checkpoint 37.654 ms, 4.75 s total -[ 2023-09-21 11:41:54 ] Completed train batch 1 to device 1,433.906 ms, 6.19 s total -[ 2023-09-21 11:41:54 ] Completed train batch 1 generator forward 22.728 ms, 6.21 s total -[ 2023-09-21 11:41:54 ] Completed train batch 1 generator loss: 1.236 63.619 ms, 6.27 s total -[ 2023-09-21 11:41:55 ] Completed train batch 1 generator backward 1,048.845 ms, 7.32 s total -[ 2023-09-21 11:41:55 ] Completed train batch 1 discriminator loss 0.051 57.711 ms, 7.38 s total -[ 2023-09-21 11:41:55 ] Completed train batch 1 discriminator backward 59.919 ms, 7.44 s total -Epoch [172] Step [1/2], gen_loss: 0.411, disc_loss: 0.017 -[ 2023-09-21 11:41:55 ] Completed train batch 1 metrics update 2.161 ms, 7.44 s total -Saving checkpoint at epoch 172 train batch 1 -[ 2023-09-21 11:41:55 ] Completed saving temp checkpoint 526.052 ms, 7.97 s total -[ 2023-09-21 11:41:55 ] Completed replacing temp checkpoint with checkpoint 26.661 ms, 8.00 s total -Epoch [172] :: gen_loss: 0.407, disc_loss: 0.017 -[ 2023-09-21 11:41:55 ] Completed training generator for epoch 172 59.329 ms, 8.05 s total -[ 2023-09-21 11:41:55 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 172 from eval step 0 - -[ 2023-09-21 11:41:57 ] Completed eval batch 0 to device 1,175.525 ms, 1.18 s total -[ 2023-09-21 11:41:57 ] Completed eval batch 0 forward 32.246 ms, 1.21 s total -[ 2023-09-21 11:41:57 ] Completed eval batch 0 recons_loss 0.340 ms, 1.21 s total -[ 2023-09-21 11:41:57 ] Completed eval batch 0 metrics update 0.660 ms, 1.21 s total -Saving checkpoint at epoch 172 val batch 0 -[ 2023-09-21 11:41:57 ] Completed saving temp checkpoint 489.607 ms, 1.70 s total -[ 2023-09-21 11:41:57 ] Completed replacing temp checkpoint with checkpoint 29.349 ms, 1.73 s total -[ 2023-09-21 11:41:57 ] Completed eval batch 1 to device 386.135 ms, 2.11 s total -[ 2023-09-21 11:41:58 ] Completed eval batch 1 forward 21.802 ms, 2.14 s total -[ 2023-09-21 11:41:58 ] Completed eval batch 1 recons_loss 0.368 ms, 2.14 s total -[ 2023-09-21 11:41:58 ] Completed eval batch 1 metrics update 55.901 ms, 2.19 s total -Saving checkpoint at epoch 172 val batch 1 -Epoch 172 val loss: 0.0533 -[ 2023-09-21 11:41:58 ] Completed saving temp checkpoint 730.972 ms, 2.92 s total -[ 2023-09-21 11:41:58 ] Completed replacing temp checkpoint with checkpoint 33.438 ms, 2.96 s total -[ 2023-09-21 11:41:58 ] Completed evaluating generator for epoch 172 59.247 ms, 3.02 s total - - -EPOCH :: 173 - - -[ 2023-09-21 11:41:58 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 173 from training step 0 - -[ 2023-09-21 11:42:01 ] Completed train batch 0 to device 3,016.777 ms, 3.02 s total -[ 2023-09-21 11:42:01 ] Completed train batch 0 generator forward 26.796 ms, 3.04 s total -[ 2023-09-21 11:42:02 ] Completed train batch 0 generator loss: 1.180 66.310 ms, 3.11 s total -[ 2023-09-21 11:42:02 ] Completed train batch 0 generator backward 342.952 ms, 3.45 s total -[ 2023-09-21 11:42:02 ] Completed train batch 0 discriminator loss 0.048 58.064 ms, 3.51 s total -[ 2023-09-21 11:42:02 ] Completed train batch 0 discriminator backward 91.419 ms, 3.60 s total -Epoch [173] Step [0/2], gen_loss: 0.407, disc_loss: 0.016 -[ 2023-09-21 11:42:02 ] Completed train batch 0 metrics update 20.550 ms, 3.62 s total -Saving checkpoint at epoch 173 train batch 0 -[ 2023-09-21 11:42:02 ] Completed saving temp checkpoint 470.360 ms, 4.09 s total -[ 2023-09-21 11:42:03 ] Completed replacing temp checkpoint with checkpoint 33.998 ms, 4.13 s total -[ 2023-09-21 11:42:05 ] Completed train batch 1 to device 2,049.233 ms, 6.18 s total -[ 2023-09-21 11:42:05 ] Completed train batch 1 generator forward 24.536 ms, 6.20 s total -[ 2023-09-21 11:42:05 ] Completed train batch 1 generator loss: 1.249 63.467 ms, 6.26 s total -[ 2023-09-21 11:42:05 ] Completed train batch 1 generator backward 409.985 ms, 6.67 s total -[ 2023-09-21 11:42:05 ] Completed train batch 1 discriminator loss 0.045 56.335 ms, 6.73 s total -[ 2023-09-21 11:42:05 ] Completed train batch 1 discriminator backward 70.685 ms, 6.80 s total -Epoch [173] Step [1/2], gen_loss: 0.406, disc_loss: 0.015 -[ 2023-09-21 11:42:05 ] Completed train batch 1 metrics update 2.723 ms, 6.80 s total -Saving checkpoint at epoch 173 train batch 1 -[ 2023-09-21 11:42:06 ] Completed saving temp checkpoint 489.845 ms, 7.29 s total -[ 2023-09-21 11:42:06 ] Completed replacing temp checkpoint with checkpoint 30.426 ms, 7.32 s total -Epoch [173] :: gen_loss: 0.407, disc_loss: 0.016 -[ 2023-09-21 11:42:06 ] Completed training generator for epoch 173 54.302 ms, 7.38 s total -[ 2023-09-21 11:42:06 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 173 from eval step 0 - -[ 2023-09-21 11:42:07 ] Completed eval batch 0 to device 1,183.647 ms, 1.18 s total -[ 2023-09-21 11:42:07 ] Completed eval batch 0 forward 41.398 ms, 1.23 s total -[ 2023-09-21 11:42:07 ] Completed eval batch 0 recons_loss 0.324 ms, 1.23 s total -[ 2023-09-21 11:42:07 ] Completed eval batch 0 metrics update 0.676 ms, 1.23 s total -Saving checkpoint at epoch 173 val batch 0 -[ 2023-09-21 11:42:07 ] Completed saving temp checkpoint 469.214 ms, 1.70 s total -[ 2023-09-21 11:42:08 ] Completed replacing temp checkpoint with checkpoint 33.954 ms, 1.73 s total -[ 2023-09-21 11:42:08 ] Completed eval batch 1 to device 371.205 ms, 2.10 s total -[ 2023-09-21 11:42:08 ] Completed eval batch 1 forward 40.917 ms, 2.14 s total -[ 2023-09-21 11:42:08 ] Completed eval batch 1 recons_loss 0.531 ms, 2.14 s total -[ 2023-09-21 11:42:08 ] Completed eval batch 1 metrics update 176.558 ms, 2.32 s total -Saving checkpoint at epoch 173 val batch 1 -Epoch 173 val loss: 0.0566 -[ 2023-09-21 11:42:09 ] Completed saving temp checkpoint 511.542 ms, 2.83 s total -[ 2023-09-21 11:42:09 ] Completed replacing temp checkpoint with checkpoint 24.737 ms, 2.85 s total -[ 2023-09-21 11:42:09 ] Completed evaluating generator for epoch 173 63.037 ms, 2.92 s total - - -EPOCH :: 174 - - -[ 2023-09-21 11:42:09 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 174 from training step 0 - -[ 2023-09-21 11:42:12 ] Completed train batch 0 to device 3,146.555 ms, 3.15 s total -[ 2023-09-21 11:42:12 ] Completed train batch 0 generator forward 33.893 ms, 3.18 s total -[ 2023-09-21 11:42:12 ] Completed train batch 0 generator loss: 1.211 68.530 ms, 3.25 s total -[ 2023-09-21 11:42:13 ] Completed train batch 0 generator backward 589.703 ms, 3.84 s total -[ 2023-09-21 11:42:13 ] Completed train batch 0 discriminator loss 0.044 57.433 ms, 3.90 s total -[ 2023-09-21 11:42:13 ] Completed train batch 0 discriminator backward 87.933 ms, 3.98 s total -Epoch [174] Step [0/2], gen_loss: 0.408, disc_loss: 0.015 -[ 2023-09-21 11:42:13 ] Completed train batch 0 metrics update 4.095 ms, 3.99 s total -Saving checkpoint at epoch 174 train batch 0 -[ 2023-09-21 11:42:13 ] Completed saving temp checkpoint 513.667 ms, 4.50 s total -[ 2023-09-21 11:42:13 ] Completed replacing temp checkpoint with checkpoint 25.424 ms, 4.53 s total -[ 2023-09-21 11:42:15 ] Completed train batch 1 to device 1,604.250 ms, 6.13 s total -[ 2023-09-21 11:42:15 ] Completed train batch 1 generator forward 22.454 ms, 6.15 s total -[ 2023-09-21 11:42:15 ] Completed train batch 1 generator loss: 1.172 64.437 ms, 6.22 s total -[ 2023-09-21 11:42:16 ] Completed train batch 1 generator backward 1,237.122 ms, 7.46 s total -[ 2023-09-21 11:42:16 ] Completed train batch 1 discriminator loss 0.042 56.834 ms, 7.51 s total -[ 2023-09-21 11:42:16 ] Completed train batch 1 discriminator backward 72.926 ms, 7.59 s total -Epoch [174] Step [1/2], gen_loss: 0.411, disc_loss: 0.014 -[ 2023-09-21 11:42:16 ] Completed train batch 1 metrics update 1.797 ms, 7.59 s total -Saving checkpoint at epoch 174 train batch 1 -[ 2023-09-21 11:42:17 ] Completed saving temp checkpoint 504.615 ms, 8.09 s total -[ 2023-09-21 11:42:17 ] Completed replacing temp checkpoint with checkpoint 32.890 ms, 8.12 s total -Epoch [174] :: gen_loss: 0.410, disc_loss: 0.014 -[ 2023-09-21 11:42:17 ] Completed training generator for epoch 174 72.298 ms, 8.20 s total -[ 2023-09-21 11:42:17 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 174 from eval step 0 - -[ 2023-09-21 11:42:18 ] Completed eval batch 0 to device 1,114.382 ms, 1.11 s total -[ 2023-09-21 11:42:18 ] Completed eval batch 0 forward 26.863 ms, 1.14 s total -[ 2023-09-21 11:42:18 ] Completed eval batch 0 recons_loss 0.299 ms, 1.14 s total -[ 2023-09-21 11:42:18 ] Completed eval batch 0 metrics update 0.604 ms, 1.14 s total -Saving checkpoint at epoch 174 val batch 0 -[ 2023-09-21 11:42:19 ] Completed saving temp checkpoint 475.944 ms, 1.62 s total -[ 2023-09-21 11:42:19 ] Completed replacing temp checkpoint with checkpoint 29.186 ms, 1.65 s total -[ 2023-09-21 11:42:19 ] Completed eval batch 1 to device 481.662 ms, 2.13 s total -[ 2023-09-21 11:42:19 ] Completed eval batch 1 forward 21.773 ms, 2.15 s total -[ 2023-09-21 11:42:19 ] Completed eval batch 1 recons_loss 0.345 ms, 2.15 s total -[ 2023-09-21 11:42:19 ] Completed eval batch 1 metrics update 10.519 ms, 2.16 s total -Saving checkpoint at epoch 174 val batch 1 -Epoch 174 val loss: 0.0612 -[ 2023-09-21 11:42:20 ] Completed saving temp checkpoint 505.052 ms, 2.67 s total -[ 2023-09-21 11:42:20 ] Completed replacing temp checkpoint with checkpoint 26.120 ms, 2.69 s total -[ 2023-09-21 11:42:20 ] Completed evaluating generator for epoch 174 57.481 ms, 2.75 s total - - -EPOCH :: 175 - - -[ 2023-09-21 11:42:20 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 175 from training step 0 - -[ 2023-09-21 11:42:23 ] Completed train batch 0 to device 3,042.224 ms, 3.04 s total -[ 2023-09-21 11:42:23 ] Completed train batch 0 generator forward 43.902 ms, 3.09 s total -[ 2023-09-21 11:42:23 ] Completed train batch 0 generator loss: 1.180 65.134 ms, 3.15 s total -[ 2023-09-21 11:42:23 ] Completed train batch 0 generator backward 593.289 ms, 3.74 s total -[ 2023-09-21 11:42:23 ] Completed train batch 0 discriminator loss 0.042 58.392 ms, 3.80 s total -[ 2023-09-21 11:42:24 ] Completed train batch 0 discriminator backward 106.200 ms, 3.91 s total -Epoch [175] Step [0/2], gen_loss: 0.410, disc_loss: 0.014 -[ 2023-09-21 11:42:24 ] Completed train batch 0 metrics update 5.814 ms, 3.91 s total -Saving checkpoint at epoch 175 train batch 0 -[ 2023-09-21 11:42:24 ] Completed saving temp checkpoint 466.276 ms, 4.38 s total -[ 2023-09-21 11:42:24 ] Completed replacing temp checkpoint with checkpoint 31.236 ms, 4.41 s total -[ 2023-09-21 11:42:25 ] Completed train batch 1 to device 1,338.562 ms, 5.75 s total -[ 2023-09-21 11:42:25 ] Completed train batch 1 generator forward 40.453 ms, 5.79 s total -[ 2023-09-21 11:42:25 ] Completed train batch 1 generator loss: 1.206 64.340 ms, 5.86 s total -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 11:54:56 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 11:54:56 ] Completed importing Timer 0.022 ms, 0.00 s total -[ 2023-09-21 11:55:00 ] Completed importing everything else 4,608.471 ms, 4.61 s total -| distributed init (rank 2): env:// -| distributed init (rank 4): env:// -| distributed init (rank 3): env:// -| distributed init (rank 0): env:// -| distributed init (rank 5): env:// -| distributed init (rank 1): env:// -[ 2023-09-21 11:55:08 ] Completed preliminaries 7,586.636 ms, 12.20 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 11:55:08 ] Completed build datasets 10.651 ms, 12.21 s total -[ 2023-09-21 11:55:08 ] Completed build samplers 0.042 ms, 12.21 s total -[ 2023-09-21 11:55:08 ] Completed build dataloaders 0.130 ms, 12.21 s total -[ 2023-09-21 11:55:08 ] Completed generator to device 77.599 ms, 12.28 s total -[ 2023-09-21 11:55:08 ] Completed discriminator to device 7.065 ms, 12.29 s total -[ 2023-09-21 11:55:08 ] Completed loss functions 388.715 ms, 12.68 s total -[ 2023-09-21 11:55:08 ] Completed models prepped for distribution 84.495 ms, 12.76 s total -[ 2023-09-21 11:55:08 ] Completed optimizers 0.692 ms, 12.76 s total -[ 2023-09-21 11:55:08 ] Completed grad scalers 0.029 ms, 12.76 s total -[ 2023-09-21 11:55:09 ] Completed checkpoint retrieval 460.794 ms, 13.23 s total - - -EPOCH :: 175 - - -[ 2023-09-21 11:55:09 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 175 from training step 1 - -[ 2023-09-21 11:55:12 ] Completed train batch 1 to device 3,578.585 ms, 3.58 s total -[ 2023-09-21 11:55:13 ] Completed train batch 1 generator forward 1,133.782 ms, 4.71 s total -[ 2023-09-21 11:55:14 ] Completed train batch 1 generator loss: 1.264 78.271 ms, 4.79 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 11:55:15 ] Completed train batch 1 generator backward 1,466.445 ms, 6.26 s total -[ 2023-09-21 11:55:15 ] Completed train batch 1 discriminator loss 0.041 65.422 ms, 6.32 s total -[ 2023-09-21 11:55:15 ] Completed train batch 1 discriminator backward 63.294 ms, 6.39 s total -Epoch [175] Step [1/2], gen_loss: 0.414, disc_loss: 0.014 -[ 2023-09-21 11:55:15 ] Completed train batch 1 metrics update 5.272 ms, 6.39 s total -Saving checkpoint at epoch 175 train batch 1 -[ 2023-09-21 11:55:16 ] Completed saving temp checkpoint 483.116 ms, 6.87 s total -[ 2023-09-21 11:55:16 ] Completed replacing temp checkpoint with checkpoint 134.847 ms, 7.01 s total -Epoch [175] :: gen_loss: 0.412, disc_loss: 0.014 -[ 2023-09-21 11:55:16 ] Completed training generator for epoch 175 55.545 ms, 7.06 s total -[ 2023-09-21 11:55:16 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 175 from eval step 0 - -[ 2023-09-21 11:55:17 ] Completed eval batch 0 to device 1,080.645 ms, 1.08 s total -[ 2023-09-21 11:55:17 ] Completed eval batch 0 forward 25.617 ms, 1.11 s total -[ 2023-09-21 11:55:17 ] Completed eval batch 0 recons_loss 0.396 ms, 1.11 s total -[ 2023-09-21 11:55:17 ] Completed eval batch 0 metrics update 0.757 ms, 1.11 s total -Saving checkpoint at epoch 175 val batch 0 -[ 2023-09-21 11:55:17 ] Completed saving temp checkpoint 396.540 ms, 1.50 s total -[ 2023-09-21 11:55:17 ] Completed replacing temp checkpoint with checkpoint 22.988 ms, 1.53 s total -[ 2023-09-21 11:55:18 ] Completed eval batch 1 to device 454.453 ms, 1.98 s total -[ 2023-09-21 11:55:18 ] Completed eval batch 1 forward 40.149 ms, 2.02 s total -[ 2023-09-21 11:55:18 ] Completed eval batch 1 recons_loss 0.619 ms, 2.02 s total -[ 2023-09-21 11:55:18 ] Completed eval batch 1 metrics update 182.432 ms, 2.20 s total -Saving checkpoint at epoch 175 val batch 1 -Epoch 175 val loss: 0.0616 -[ 2023-09-21 11:55:18 ] Completed saving temp checkpoint 453.196 ms, 2.66 s total -[ 2023-09-21 11:55:18 ] Completed replacing temp checkpoint with checkpoint 18.401 ms, 2.68 s total -[ 2023-09-21 11:55:19 ] Completed evaluating generator for epoch 175 73.499 ms, 2.75 s total - - -EPOCH :: 176 - - -[ 2023-09-21 11:55:19 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 176 from training step 0 - -[ 2023-09-21 11:55:22 ] Completed train batch 0 to device 3,184.023 ms, 3.18 s total -[ 2023-09-21 11:55:22 ] Completed train batch 0 generator forward 29.091 ms, 3.21 s total -[ 2023-09-21 11:55:22 ] Completed train batch 0 generator loss: 1.190 66.950 ms, 3.28 s total -[ 2023-09-21 11:55:22 ] Completed train batch 0 generator backward 551.713 ms, 3.83 s total -[ 2023-09-21 11:55:22 ] Completed train batch 0 discriminator loss 0.041 57.718 ms, 3.89 s total -[ 2023-09-21 11:55:23 ] Completed train batch 0 discriminator backward 104.326 ms, 3.99 s total -Epoch [176] Step [0/2], gen_loss: 0.414, disc_loss: 0.014 -[ 2023-09-21 11:55:23 ] Completed train batch 0 metrics update 0.768 ms, 3.99 s total -Saving checkpoint at epoch 176 train batch 0 -[ 2023-09-21 11:55:23 ] Completed saving temp checkpoint 530.215 ms, 4.52 s total -[ 2023-09-21 11:55:23 ] Completed replacing temp checkpoint with checkpoint 51.631 ms, 4.58 s total -[ 2023-09-21 11:55:25 ] Completed train batch 1 to device 1,584.070 ms, 6.16 s total -[ 2023-09-21 11:55:25 ] Completed train batch 1 generator forward 22.577 ms, 6.18 s total -[ 2023-09-21 11:55:25 ] Completed train batch 1 generator loss: 1.222 63.747 ms, 6.25 s total -[ 2023-09-21 11:55:26 ] Completed train batch 1 generator backward 1,224.750 ms, 7.47 s total -[ 2023-09-21 11:55:26 ] Completed train batch 1 discriminator loss 0.044 56.647 ms, 7.53 s total -[ 2023-09-21 11:55:26 ] Completed train batch 1 discriminator backward 81.077 ms, 7.61 s total -Epoch [176] Step [1/2], gen_loss: 0.411, disc_loss: 0.015 -[ 2023-09-21 11:55:26 ] Completed train batch 1 metrics update 3.098 ms, 7.61 s total -Saving checkpoint at epoch 176 train batch 1 -[ 2023-09-21 11:55:27 ] Completed saving temp checkpoint 434.500 ms, 8.05 s total -[ 2023-09-21 11:55:27 ] Completed replacing temp checkpoint with checkpoint 21.815 ms, 8.07 s total -Epoch [176] :: gen_loss: 0.412, disc_loss: 0.014 -[ 2023-09-21 11:55:27 ] Completed training generator for epoch 176 51.041 ms, 8.12 s total -[ 2023-09-21 11:55:27 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 176 from eval step 0 - -[ 2023-09-21 11:55:28 ] Completed eval batch 0 to device 1,155.664 ms, 1.16 s total -[ 2023-09-21 11:55:28 ] Completed eval batch 0 forward 32.331 ms, 1.19 s total -[ 2023-09-21 11:55:28 ] Completed eval batch 0 recons_loss 0.342 ms, 1.19 s total -[ 2023-09-21 11:55:28 ] Completed eval batch 0 metrics update 0.585 ms, 1.19 s total -Saving checkpoint at epoch 176 val batch 0 -[ 2023-09-21 11:55:28 ] Completed saving temp checkpoint 450.520 ms, 1.64 s total -[ 2023-09-21 11:55:28 ] Completed replacing temp checkpoint with checkpoint 32.414 ms, 1.67 s total -[ 2023-09-21 11:55:29 ] Completed eval batch 1 to device 514.295 ms, 2.19 s total -[ 2023-09-21 11:55:29 ] Completed eval batch 1 forward 40.463 ms, 2.23 s total -[ 2023-09-21 11:55:29 ] Completed eval batch 1 recons_loss 0.629 ms, 2.23 s total -[ 2023-09-21 11:55:29 ] Completed eval batch 1 metrics update 67.697 ms, 2.29 s total -Saving checkpoint at epoch 176 val batch 1 -Epoch 176 val loss: 0.0547 -[ 2023-09-21 11:55:29 ] Completed saving temp checkpoint 434.090 ms, 2.73 s total -[ 2023-09-21 11:55:29 ] Completed replacing temp checkpoint with checkpoint 23.422 ms, 2.75 s total -[ 2023-09-21 11:55:29 ] Completed evaluating generator for epoch 176 46.398 ms, 2.80 s total - - -EPOCH :: 177 - - -[ 2023-09-21 11:55:29 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 177 from training step 0 - -[ 2023-09-21 11:55:33 ] Completed train batch 0 to device 3,153.221 ms, 3.15 s total -[ 2023-09-21 11:55:33 ] Completed train batch 0 generator forward 28.456 ms, 3.18 s total -[ 2023-09-21 11:55:33 ] Completed train batch 0 generator loss: 1.243 70.794 ms, 3.25 s total -[ 2023-09-21 11:55:33 ] Completed train batch 0 generator backward 653.857 ms, 3.91 s total -[ 2023-09-21 11:55:33 ] Completed train batch 0 discriminator loss 0.047 57.252 ms, 3.96 s total -[ 2023-09-21 11:55:34 ] Completed train batch 0 discriminator backward 102.380 ms, 4.07 s total -Epoch [177] Step [0/2], gen_loss: 0.412, disc_loss: 0.015 -[ 2023-09-21 11:55:34 ] Completed train batch 0 metrics update 7.791 ms, 4.07 s total -Saving checkpoint at epoch 177 train batch 0 -[ 2023-09-21 11:55:34 ] Completed saving temp checkpoint 445.893 ms, 4.52 s total -[ 2023-09-21 11:55:34 ] Completed replacing temp checkpoint with checkpoint 21.854 ms, 4.54 s total -[ 2023-09-21 11:55:36 ] Completed train batch 1 to device 1,543.488 ms, 6.08 s total -[ 2023-09-21 11:55:36 ] Completed train batch 1 generator forward 41.575 ms, 6.13 s total -[ 2023-09-21 11:55:36 ] Completed train batch 1 generator loss: 1.186 65.806 ms, 6.19 s total -[ 2023-09-21 11:55:37 ] Completed train batch 1 generator backward 1,481.731 ms, 7.67 s total -[ 2023-09-21 11:55:37 ] Completed train batch 1 discriminator loss 0.046 56.613 ms, 7.73 s total -[ 2023-09-21 11:55:37 ] Completed train batch 1 discriminator backward 74.444 ms, 7.81 s total -Epoch [177] Step [1/2], gen_loss: 0.410, disc_loss: 0.015 -[ 2023-09-21 11:55:37 ] Completed train batch 1 metrics update 0.645 ms, 7.81 s total -Saving checkpoint at epoch 177 train batch 1 -[ 2023-09-21 11:55:38 ] Completed saving temp checkpoint 500.971 ms, 8.31 s total -[ 2023-09-21 11:55:38 ] Completed replacing temp checkpoint with checkpoint 22.579 ms, 8.33 s total -Epoch [177] :: gen_loss: 0.411, disc_loss: 0.015 -[ 2023-09-21 11:55:38 ] Completed training generator for epoch 177 50.330 ms, 8.38 s total -[ 2023-09-21 11:55:38 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 177 from eval step 0 - -[ 2023-09-21 11:55:39 ] Completed eval batch 0 to device 1,141.644 ms, 1.14 s total -[ 2023-09-21 11:55:39 ] Completed eval batch 0 forward 26.630 ms, 1.17 s total -[ 2023-09-21 11:55:39 ] Completed eval batch 0 recons_loss 0.285 ms, 1.17 s total -[ 2023-09-21 11:55:39 ] Completed eval batch 0 metrics update 0.537 ms, 1.17 s total -Saving checkpoint at epoch 177 val batch 0 -[ 2023-09-21 11:55:39 ] Completed saving temp checkpoint 427.430 ms, 1.60 s total -[ 2023-09-21 11:55:39 ] Completed replacing temp checkpoint with checkpoint 16.816 ms, 1.61 s total -[ 2023-09-21 11:55:40 ] Completed eval batch 1 to device 493.746 ms, 2.11 s total -[ 2023-09-21 11:55:40 ] Completed eval batch 1 forward 22.222 ms, 2.13 s total -[ 2023-09-21 11:55:40 ] Completed eval batch 1 recons_loss 0.388 ms, 2.13 s total -[ 2023-09-21 11:55:40 ] Completed eval batch 1 metrics update 98.831 ms, 2.23 s total -Saving checkpoint at epoch 177 val batch 1 -Epoch 177 val loss: 0.0603 -[ 2023-09-21 11:55:40 ] Completed saving temp checkpoint 405.121 ms, 2.63 s total -[ 2023-09-21 11:55:41 ] Completed replacing temp checkpoint with checkpoint 23.565 ms, 2.66 s total -[ 2023-09-21 11:55:41 ] Completed evaluating generator for epoch 177 45.899 ms, 2.70 s total - - -EPOCH :: 178 - - -[ 2023-09-21 11:55:41 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 178 from training step 0 - -[ 2023-09-21 11:55:43 ] Completed train batch 0 to device 2,822.681 ms, 2.82 s total -[ 2023-09-21 11:55:43 ] Completed train batch 0 generator forward 25.290 ms, 2.85 s total -[ 2023-09-21 11:55:43 ] Completed train batch 0 generator loss: 1.255 64.436 ms, 2.91 s total -[ 2023-09-21 11:55:44 ] Completed train batch 0 generator backward 951.519 ms, 3.86 s total -[ 2023-09-21 11:55:44 ] Completed train batch 0 discriminator loss 0.043 56.639 ms, 3.92 s total -[ 2023-09-21 11:55:45 ] Completed train batch 0 discriminator backward 100.051 ms, 4.02 s total -Epoch [178] Step [0/2], gen_loss: 0.414, disc_loss: 0.015 -[ 2023-09-21 11:55:45 ] Completed train batch 0 metrics update 5.685 ms, 4.03 s total -Saving checkpoint at epoch 178 train batch 0 -[ 2023-09-21 11:55:45 ] Completed saving temp checkpoint 422.428 ms, 4.45 s total -[ 2023-09-21 11:55:45 ] Completed replacing temp checkpoint with checkpoint 22.470 ms, 4.47 s total -[ 2023-09-21 11:55:46 ] Completed train batch 1 to device 1,124.516 ms, 5.60 s total -[ 2023-09-21 11:55:46 ] Completed train batch 1 generator forward 40.621 ms, 5.64 s total -[ 2023-09-21 11:55:46 ] Completed train batch 1 generator loss: 1.147 62.268 ms, 5.70 s total -[ 2023-09-21 11:55:48 ] Completed train batch 1 generator backward 1,337.882 ms, 7.04 s total -[ 2023-09-21 11:55:48 ] Completed train batch 1 discriminator loss 0.040 56.477 ms, 7.09 s total -[ 2023-09-21 11:55:48 ] Completed train batch 1 discriminator backward 72.523 ms, 7.17 s total -Epoch [178] Step [1/2], gen_loss: 0.406, disc_loss: 0.014 -[ 2023-09-21 11:55:48 ] Completed train batch 1 metrics update 1.318 ms, 7.17 s total -Saving checkpoint at epoch 178 train batch 1 -[ 2023-09-21 11:55:48 ] Completed saving temp checkpoint 451.931 ms, 7.62 s total -[ 2023-09-21 11:55:48 ] Completed replacing temp checkpoint with checkpoint 23.515 ms, 7.64 s total -Epoch [178] :: gen_loss: 0.410, disc_loss: 0.014 -[ 2023-09-21 11:55:48 ] Completed training generator for epoch 178 49.204 ms, 7.69 s total -[ 2023-09-21 11:55:48 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 178 from eval step 0 - -[ 2023-09-21 11:55:49 ] Completed eval batch 0 to device 1,048.537 ms, 1.05 s total -[ 2023-09-21 11:55:49 ] Completed eval batch 0 forward 29.968 ms, 1.08 s total -[ 2023-09-21 11:55:49 ] Completed eval batch 0 recons_loss 0.231 ms, 1.08 s total -[ 2023-09-21 11:55:49 ] Completed eval batch 0 metrics update 0.508 ms, 1.08 s total -Saving checkpoint at epoch 178 val batch 0 -[ 2023-09-21 11:55:50 ] Completed saving temp checkpoint 400.022 ms, 1.48 s total -[ 2023-09-21 11:55:50 ] Completed replacing temp checkpoint with checkpoint 30.433 ms, 1.51 s total -[ 2023-09-21 11:55:50 ] Completed eval batch 1 to device 545.215 ms, 2.05 s total -[ 2023-09-21 11:55:50 ] Completed eval batch 1 forward 40.308 ms, 2.10 s total -[ 2023-09-21 11:55:50 ] Completed eval batch 1 recons_loss 0.579 ms, 2.10 s total -[ 2023-09-21 11:55:50 ] Completed eval batch 1 metrics update 130.806 ms, 2.23 s total -Saving checkpoint at epoch 178 val batch 1 -Epoch 178 val loss: 0.0627 -[ 2023-09-21 11:55:51 ] Completed saving temp checkpoint 441.129 ms, 2.67 s total -[ 2023-09-21 11:55:51 ] Completed replacing temp checkpoint with checkpoint 22.438 ms, 2.69 s total -[ 2023-09-21 11:55:51 ] Completed evaluating generator for epoch 178 44.152 ms, 2.73 s total - - -EPOCH :: 179 - - -[ 2023-09-21 11:55:51 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 179 from training step 0 - -[ 2023-09-21 11:55:54 ] Completed train batch 0 to device 3,120.576 ms, 3.12 s total -[ 2023-09-21 11:55:54 ] Completed train batch 0 generator forward 27.411 ms, 3.15 s total -[ 2023-09-21 11:55:54 ] Completed train batch 0 generator loss: 1.361 67.896 ms, 3.22 s total -[ 2023-09-21 11:55:55 ] Completed train batch 0 generator backward 602.768 ms, 3.82 s total -[ 2023-09-21 11:55:55 ] Completed train batch 0 discriminator loss 0.044 76.709 ms, 3.90 s total -[ 2023-09-21 11:55:55 ] Completed train batch 0 discriminator backward 68.586 ms, 3.96 s total -Epoch [179] Step [0/2], gen_loss: 0.420, disc_loss: 0.016 -[ 2023-09-21 11:55:55 ] Completed train batch 0 metrics update 3.663 ms, 3.97 s total -Saving checkpoint at epoch 179 train batch 0 -[ 2023-09-21 11:55:55 ] Completed saving temp checkpoint 468.887 ms, 4.44 s total -[ 2023-09-21 11:55:55 ] Completed replacing temp checkpoint with checkpoint 23.875 ms, 4.46 s total -[ 2023-09-21 11:55:57 ] Completed train batch 1 to device 1,534.235 ms, 5.99 s total -[ 2023-09-21 11:55:57 ] Completed train batch 1 generator forward 24.919 ms, 6.02 s total -[ 2023-09-21 11:55:57 ] Completed train batch 1 generator loss: 1.214 64.782 ms, 6.08 s total -[ 2023-09-21 11:55:58 ] Completed train batch 1 generator backward 1,100.762 ms, 7.19 s total -[ 2023-09-21 11:55:58 ] Completed train batch 1 discriminator loss 0.047 57.867 ms, 7.24 s total -[ 2023-09-21 11:55:58 ] Completed train batch 1 discriminator backward 61.491 ms, 7.30 s total -Epoch [179] Step [1/2], gen_loss: 0.409, disc_loss: 0.016 -[ 2023-09-21 11:55:58 ] Completed train batch 1 metrics update 1.742 ms, 7.31 s total -Saving checkpoint at epoch 179 train batch 1 -[ 2023-09-21 11:55:59 ] Completed saving temp checkpoint 438.934 ms, 7.75 s total -[ 2023-09-21 11:55:59 ] Completed replacing temp checkpoint with checkpoint 20.262 ms, 7.77 s total -Epoch [179] :: gen_loss: 0.415, disc_loss: 0.016 -[ 2023-09-21 11:55:59 ] Completed training generator for epoch 179 46.613 ms, 7.81 s total -[ 2023-09-21 11:55:59 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 179 from eval step 0 - -[ 2023-09-21 11:56:00 ] Completed eval batch 0 to device 1,105.376 ms, 1.11 s total -[ 2023-09-21 11:56:00 ] Completed eval batch 0 forward 40.177 ms, 1.15 s total -[ 2023-09-21 11:56:00 ] Completed eval batch 0 recons_loss 0.290 ms, 1.15 s total -[ 2023-09-21 11:56:00 ] Completed eval batch 0 metrics update 0.574 ms, 1.15 s total -Saving checkpoint at epoch 179 val batch 0 -[ 2023-09-21 11:56:00 ] Completed saving temp checkpoint 413.616 ms, 1.56 s total -[ 2023-09-21 11:56:00 ] Completed replacing temp checkpoint with checkpoint 28.615 ms, 1.59 s total -[ 2023-09-21 11:56:01 ] Completed eval batch 1 to device 410.275 ms, 2.00 s total -[ 2023-09-21 11:56:01 ] Completed eval batch 1 forward 38.299 ms, 2.04 s total -[ 2023-09-21 11:56:01 ] Completed eval batch 1 recons_loss 0.336 ms, 2.04 s total -[ 2023-09-21 11:56:01 ] Completed eval batch 1 metrics update 214.229 ms, 2.25 s total -Saving checkpoint at epoch 179 val batch 1 -Epoch 179 val loss: 0.0540 -[ 2023-09-21 11:56:02 ] Completed saving temp checkpoint 491.611 ms, 2.74 s total -[ 2023-09-21 11:56:02 ] Completed replacing temp checkpoint with checkpoint 28.067 ms, 2.77 s total -[ 2023-09-21 11:56:02 ] Completed evaluating generator for epoch 179 46.420 ms, 2.82 s total - - -EPOCH :: 180 - - -[ 2023-09-21 11:56:02 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 180 from training step 0 - -[ 2023-09-21 11:56:05 ] Completed train batch 0 to device 3,051.947 ms, 3.05 s total -[ 2023-09-21 11:56:05 ] Completed train batch 0 generator forward 42.083 ms, 3.09 s total -[ 2023-09-21 11:56:05 ] Completed train batch 0 generator loss: 1.235 64.929 ms, 3.16 s total -[ 2023-09-21 11:56:05 ] Completed train batch 0 generator backward 602.480 ms, 3.76 s total -[ 2023-09-21 11:56:05 ] Completed train batch 0 discriminator loss 0.052 56.642 ms, 3.82 s total -[ 2023-09-21 11:56:06 ] Completed train batch 0 discriminator backward 96.670 ms, 3.91 s total -Epoch [180] Step [0/2], gen_loss: 0.416, disc_loss: 0.016 -[ 2023-09-21 11:56:06 ] Completed train batch 0 metrics update 5.926 ms, 3.92 s total -Saving checkpoint at epoch 180 train batch 0 -[ 2023-09-21 11:56:06 ] Completed saving temp checkpoint 385.767 ms, 4.31 s total -[ 2023-09-21 11:56:06 ] Completed replacing temp checkpoint with checkpoint 27.090 ms, 4.33 s total -[ 2023-09-21 11:56:07 ] Completed train batch 1 to device 1,502.097 ms, 5.84 s total -[ 2023-09-21 11:56:07 ] Completed train batch 1 generator forward 41.273 ms, 5.88 s total -[ 2023-09-21 11:56:08 ] Completed train batch 1 generator loss: 1.205 63.900 ms, 5.94 s total -[ 2023-09-21 11:56:09 ] Completed train batch 1 generator backward 1,008.468 ms, 6.95 s total -[ 2023-09-21 11:56:09 ] Completed train batch 1 discriminator loss 0.043 56.410 ms, 7.01 s total -[ 2023-09-21 11:56:09 ] Completed train batch 1 discriminator backward 75.282 ms, 7.08 s total -Epoch [180] Step [1/2], gen_loss: 0.408, disc_loss: 0.015 -[ 2023-09-21 11:56:09 ] Completed train batch 1 metrics update 1.516 ms, 7.08 s total -Saving checkpoint at epoch 180 train batch 1 -[ 2023-09-21 11:56:09 ] Completed saving temp checkpoint 384.637 ms, 7.47 s total -[ 2023-09-21 11:56:09 ] Completed replacing temp checkpoint with checkpoint 24.087 ms, 7.49 s total -Epoch [180] :: gen_loss: 0.412, disc_loss: 0.016 -[ 2023-09-21 11:56:09 ] Completed training generator for epoch 180 51.234 ms, 7.54 s total -[ 2023-09-21 11:56:09 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 180 from eval step 0 - -[ 2023-09-21 11:56:10 ] Completed eval batch 0 to device 1,031.088 ms, 1.03 s total -[ 2023-09-21 11:56:10 ] Completed eval batch 0 forward 40.342 ms, 1.07 s total -[ 2023-09-21 11:56:10 ] Completed eval batch 0 recons_loss 0.261 ms, 1.07 s total -[ 2023-09-21 11:56:10 ] Completed eval batch 0 metrics update 100.407 ms, 1.17 s total -Saving checkpoint at epoch 180 val batch 0 -[ 2023-09-21 11:56:11 ] Completed saving temp checkpoint 409.334 ms, 1.58 s total -[ 2023-09-21 11:56:11 ] Completed replacing temp checkpoint with checkpoint 25.429 ms, 1.61 s total -[ 2023-09-21 11:56:11 ] Completed eval batch 1 to device 346.283 ms, 1.95 s total -[ 2023-09-21 11:56:11 ] Completed eval batch 1 forward 41.030 ms, 1.99 s total -[ 2023-09-21 11:56:11 ] Completed eval batch 1 recons_loss 0.527 ms, 1.99 s total -[ 2023-09-21 11:56:12 ] Completed eval batch 1 metrics update 386.130 ms, 2.38 s total -Saving checkpoint at epoch 180 val batch 1 -Epoch 180 val loss: 0.0587 -[ 2023-09-21 11:56:12 ] Completed saving temp checkpoint 426.155 ms, 2.81 s total -[ 2023-09-21 11:56:12 ] Completed replacing temp checkpoint with checkpoint 32.344 ms, 2.84 s total -[ 2023-09-21 11:56:12 ] Completed evaluating generator for epoch 180 48.435 ms, 2.89 s total - - -EPOCH :: 181 - - -[ 2023-09-21 11:56:12 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 181 from training step 0 - -[ 2023-09-21 11:56:15 ] Completed train batch 0 to device 3,264.514 ms, 3.26 s total -[ 2023-09-21 11:56:15 ] Completed train batch 0 generator forward 43.745 ms, 3.31 s total -[ 2023-09-21 11:56:15 ] Completed train batch 0 generator loss: 1.195 66.612 ms, 3.37 s total -[ 2023-09-21 11:56:16 ] Completed train batch 0 generator backward 523.194 ms, 3.90 s total -[ 2023-09-21 11:56:16 ] Completed train batch 0 discriminator loss 0.044 59.108 ms, 3.96 s total -[ 2023-09-21 11:56:16 ] Completed train batch 0 discriminator backward 90.093 ms, 4.05 s total -Epoch [181] Step [0/2], gen_loss: 0.407, disc_loss: 0.015 -[ 2023-09-21 11:56:16 ] Completed train batch 0 metrics update 17.860 ms, 4.07 s total -Saving checkpoint at epoch 181 train batch 0 -[ 2023-09-21 11:56:17 ] Completed saving temp checkpoint 460.309 ms, 4.53 s total -[ 2023-09-21 11:56:17 ] Completed replacing temp checkpoint with checkpoint 28.818 ms, 4.55 s total -[ 2023-09-21 11:56:18 ] Completed train batch 1 to device 1,752.306 ms, 6.31 s total -[ 2023-09-21 11:56:18 ] Completed train batch 1 generator forward 22.534 ms, 6.33 s total -[ 2023-09-21 11:56:18 ] Completed train batch 1 generator loss: 1.209 63.508 ms, 6.39 s total -[ 2023-09-21 11:56:20 ] Completed train batch 1 generator backward 1,226.040 ms, 7.62 s total -[ 2023-09-21 11:56:20 ] Completed train batch 1 discriminator loss 0.044 56.978 ms, 7.68 s total -[ 2023-09-21 11:56:20 ] Completed train batch 1 discriminator backward 58.997 ms, 7.73 s total -Epoch [181] Step [1/2], gen_loss: 0.412, disc_loss: 0.015 -[ 2023-09-21 11:56:20 ] Completed train batch 1 metrics update 5.495 ms, 7.74 s total -Saving checkpoint at epoch 181 train batch 1 -[ 2023-09-21 11:56:20 ] Completed saving temp checkpoint 399.986 ms, 8.14 s total -[ 2023-09-21 11:56:20 ] Completed replacing temp checkpoint with checkpoint 15.908 ms, 8.16 s total -Epoch [181] :: gen_loss: 0.410, disc_loss: 0.015 -[ 2023-09-21 11:56:20 ] Completed training generator for epoch 181 47.869 ms, 8.20 s total -[ 2023-09-21 11:56:20 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 181 from eval step 0 - -[ 2023-09-21 11:56:21 ] Completed eval batch 0 to device 1,191.484 ms, 1.19 s total -[ 2023-09-21 11:56:21 ] Completed eval batch 0 forward 23.989 ms, 1.22 s total -[ 2023-09-21 11:56:21 ] Completed eval batch 0 recons_loss 0.233 ms, 1.22 s total -[ 2023-09-21 11:56:21 ] Completed eval batch 0 metrics update 0.535 ms, 1.22 s total -Saving checkpoint at epoch 181 val batch 0 -[ 2023-09-21 11:56:22 ] Completed saving temp checkpoint 414.483 ms, 1.63 s total -[ 2023-09-21 11:56:22 ] Completed replacing temp checkpoint with checkpoint 35.669 ms, 1.67 s total -[ 2023-09-21 11:56:22 ] Completed eval batch 1 to device 467.453 ms, 2.13 s total -[ 2023-09-21 11:56:22 ] Completed eval batch 1 forward 40.327 ms, 2.17 s total -[ 2023-09-21 11:56:22 ] Completed eval batch 1 recons_loss 0.565 ms, 2.17 s total -[ 2023-09-21 11:56:23 ] Completed eval batch 1 metrics update 219.018 ms, 2.39 s total -Saving checkpoint at epoch 181 val batch 1 -Epoch 181 val loss: 0.0640 -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 12:08:56 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 12:08:56 ] Completed importing Timer 0.026 ms, 0.00 s total -| distributed init (rank 1): env:// -| distributed init (rank 2): env:// -| distributed init (rank 5): env:// -| distributed init (rank 4): env:// -| distributed init (rank 3): env:// -[ 2023-09-21 12:09:05 ] Completed importing everything else 9,238.949 ms, 9.24 s total -| distributed init (rank 0): env:// -[ 2023-09-21 12:09:08 ] Completed preliminaries 3,502.036 ms, 12.74 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 12:09:08 ] Completed build datasets 10.847 ms, 12.75 s total -[ 2023-09-21 12:09:08 ] Completed build samplers 0.047 ms, 12.75 s total -[ 2023-09-21 12:09:08 ] Completed build dataloaders 0.133 ms, 12.75 s total -[ 2023-09-21 12:09:09 ] Completed generator to device 77.193 ms, 12.83 s total -[ 2023-09-21 12:09:09 ] Completed discriminator to device 7.026 ms, 12.84 s total -[ 2023-09-21 12:09:09 ] Completed loss functions 408.844 ms, 13.25 s total -[ 2023-09-21 12:09:09 ] Completed models prepped for distribution 85.618 ms, 13.33 s total -[ 2023-09-21 12:09:09 ] Completed optimizers 0.674 ms, 13.33 s total -[ 2023-09-21 12:09:09 ] Completed grad scalers 0.029 ms, 13.33 s total -[ 2023-09-21 12:09:10 ] Completed checkpoint retrieval 484.520 ms, 13.82 s total - - -EPOCH :: 181 - - -[ 2023-09-21 12:09:10 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 181 from training step 2 - -Epoch [181] :: gen_loss: 0.410, disc_loss: 0.015 -[ 2023-09-21 12:09:10 ] Completed training generator for epoch 181 134.924 ms, 0.13 s total -[ 2023-09-21 12:09:10 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 181 from eval step 1 - -[ 2023-09-21 12:09:11 ] Completed eval batch 1 to device 1,253.566 ms, 1.25 s total -[ 2023-09-21 12:09:12 ] Completed eval batch 1 forward 1,040.842 ms, 2.29 s total -[ 2023-09-21 12:09:12 ] Completed eval batch 1 recons_loss 0.272 ms, 2.29 s total -[ 2023-09-21 12:09:13 ] Completed eval batch 1 metrics update 785.372 ms, 3.08 s total -Saving checkpoint at epoch 181 val batch 1 -Epoch 181 val loss: 0.0630 -[ 2023-09-21 12:09:13 ] Completed saving temp checkpoint 416.044 ms, 3.50 s total -[ 2023-09-21 12:09:13 ] Completed replacing temp checkpoint with checkpoint 143.875 ms, 3.64 s total -[ 2023-09-21 12:09:13 ] Completed evaluating generator for epoch 181 38.465 ms, 3.68 s total - - -EPOCH :: 182 - - -[ 2023-09-21 12:09:13 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 182 from training step 0 - -[ 2023-09-21 12:09:16 ] Completed train batch 0 to device 2,893.146 ms, 2.89 s total -[ 2023-09-21 12:09:16 ] Completed train batch 0 generator forward 46.166 ms, 2.94 s total -[ 2023-09-21 12:09:16 ] Completed train batch 0 generator loss: 1.259 80.743 ms, 3.02 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 12:09:17 ] Completed train batch 0 generator backward 932.663 ms, 3.95 s total -[ 2023-09-21 12:09:17 ] Completed train batch 0 discriminator loss 0.042 85.500 ms, 4.04 s total -[ 2023-09-21 12:09:17 ] Completed train batch 0 discriminator backward 89.178 ms, 4.13 s total -Epoch [182] Step [0/2], gen_loss: 0.420, disc_loss: 0.015 -[ 2023-09-21 12:09:17 ] Completed train batch 0 metrics update 8.357 ms, 4.14 s total -Saving checkpoint at epoch 182 train batch 0 -[ 2023-09-21 12:09:18 ] Completed saving temp checkpoint 366.287 ms, 4.50 s total -[ 2023-09-21 12:09:18 ] Completed replacing temp checkpoint with checkpoint 19.085 ms, 4.52 s total -[ 2023-09-21 12:09:19 ] Completed train batch 1 to device 1,059.093 ms, 5.58 s total -[ 2023-09-21 12:09:19 ] Completed train batch 1 generator forward 26.755 ms, 5.61 s total -[ 2023-09-21 12:09:19 ] Completed train batch 1 generator loss: 1.223 63.740 ms, 5.67 s total -[ 2023-09-21 12:09:21 ] Completed train batch 1 generator backward 1,560.909 ms, 7.23 s total -[ 2023-09-21 12:09:21 ] Completed train batch 1 discriminator loss 0.045 56.066 ms, 7.29 s total -[ 2023-09-21 12:09:21 ] Completed train batch 1 discriminator backward 74.776 ms, 7.36 s total -Epoch [182] Step [1/2], gen_loss: 0.415, disc_loss: 0.014 -[ 2023-09-21 12:09:21 ] Completed train batch 1 metrics update 5.025 ms, 7.37 s total -Saving checkpoint at epoch 182 train batch 1 -[ 2023-09-21 12:09:21 ] Completed saving temp checkpoint 454.935 ms, 7.82 s total -[ 2023-09-21 12:09:21 ] Completed replacing temp checkpoint with checkpoint 22.081 ms, 7.84 s total -Epoch [182] :: gen_loss: 0.417, disc_loss: 0.015 -[ 2023-09-21 12:09:21 ] Completed training generator for epoch 182 55.383 ms, 7.90 s total -[ 2023-09-21 12:09:21 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 182 from eval step 0 - -[ 2023-09-21 12:09:22 ] Completed eval batch 0 to device 1,103.286 ms, 1.10 s total -[ 2023-09-21 12:09:22 ] Completed eval batch 0 forward 41.576 ms, 1.14 s total -[ 2023-09-21 12:09:22 ] Completed eval batch 0 recons_loss 0.269 ms, 1.15 s total -[ 2023-09-21 12:09:22 ] Completed eval batch 0 metrics update 0.541 ms, 1.15 s total -Saving checkpoint at epoch 182 val batch 0 -[ 2023-09-21 12:09:23 ] Completed saving temp checkpoint 377.670 ms, 1.52 s total -[ 2023-09-21 12:09:23 ] Completed replacing temp checkpoint with checkpoint 21.492 ms, 1.54 s total -[ 2023-09-21 12:09:23 ] Completed eval batch 1 to device 427.531 ms, 1.97 s total -[ 2023-09-21 12:09:23 ] Completed eval batch 1 forward 21.251 ms, 1.99 s total -[ 2023-09-21 12:09:23 ] Completed eval batch 1 recons_loss 0.332 ms, 1.99 s total -[ 2023-09-21 12:09:24 ] Completed eval batch 1 metrics update 250.768 ms, 2.24 s total -Saving checkpoint at epoch 182 val batch 1 -Epoch 182 val loss: 0.0538 -[ 2023-09-21 12:09:24 ] Completed saving temp checkpoint 449.917 ms, 2.69 s total -[ 2023-09-21 12:09:24 ] Completed replacing temp checkpoint with checkpoint 29.528 ms, 2.72 s total -[ 2023-09-21 12:09:24 ] Completed evaluating generator for epoch 182 50.438 ms, 2.77 s total - - -EPOCH :: 183 - - -[ 2023-09-21 12:09:24 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 183 from training step 0 - -[ 2023-09-21 12:09:27 ] Completed train batch 0 to device 3,153.371 ms, 3.15 s total -[ 2023-09-21 12:09:27 ] Completed train batch 0 generator forward 25.154 ms, 3.18 s total -[ 2023-09-21 12:09:27 ] Completed train batch 0 generator loss: 1.243 64.513 ms, 3.24 s total -[ 2023-09-21 12:09:28 ] Completed train batch 0 generator backward 628.141 ms, 3.87 s total -[ 2023-09-21 12:09:28 ] Completed train batch 0 discriminator loss 0.046 59.483 ms, 3.93 s total -[ 2023-09-21 12:09:28 ] Completed train batch 0 discriminator backward 101.156 ms, 4.03 s total -Epoch [183] Step [0/2], gen_loss: 0.406, disc_loss: 0.014 -[ 2023-09-21 12:09:28 ] Completed train batch 0 metrics update 4.840 ms, 4.04 s total -Saving checkpoint at epoch 183 train batch 0 -[ 2023-09-21 12:09:29 ] Completed saving temp checkpoint 452.494 ms, 4.49 s total -[ 2023-09-21 12:09:29 ] Completed replacing temp checkpoint with checkpoint 24.637 ms, 4.51 s total -[ 2023-09-21 12:09:30 ] Completed train batch 1 to device 1,665.051 ms, 6.18 s total -[ 2023-09-21 12:09:30 ] Completed train batch 1 generator forward 22.292 ms, 6.20 s total -[ 2023-09-21 12:09:30 ] Completed train batch 1 generator loss: 1.231 62.524 ms, 6.26 s total -[ 2023-09-21 12:09:31 ] Completed train batch 1 generator backward 1,162.524 ms, 7.43 s total -[ 2023-09-21 12:09:32 ] Completed train batch 1 discriminator loss 0.042 55.731 ms, 7.48 s total -[ 2023-09-21 12:09:32 ] Completed train batch 1 discriminator backward 81.581 ms, 7.56 s total -Epoch [183] Step [1/2], gen_loss: 0.410, disc_loss: 0.014 -[ 2023-09-21 12:09:32 ] Completed train batch 1 metrics update 5.386 ms, 7.57 s total -Saving checkpoint at epoch 183 train batch 1 -[ 2023-09-21 12:09:32 ] Completed saving temp checkpoint 605.962 ms, 8.17 s total -[ 2023-09-21 12:09:32 ] Completed replacing temp checkpoint with checkpoint 36.372 ms, 8.21 s total -Epoch [183] :: gen_loss: 0.408, disc_loss: 0.014 -[ 2023-09-21 12:09:32 ] Completed training generator for epoch 183 69.601 ms, 8.28 s total -[ 2023-09-21 12:09:32 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 183 from eval step 0 - -[ 2023-09-21 12:09:33 ] Completed eval batch 0 to device 1,015.590 ms, 1.02 s total -[ 2023-09-21 12:09:33 ] Completed eval batch 0 forward 32.649 ms, 1.05 s total -[ 2023-09-21 12:09:33 ] Completed eval batch 0 recons_loss 0.515 ms, 1.05 s total -[ 2023-09-21 12:09:33 ] Completed eval batch 0 metrics update 0.899 ms, 1.05 s total -Saving checkpoint at epoch 183 val batch 0 -[ 2023-09-21 12:09:34 ] Completed saving temp checkpoint 705.208 ms, 1.75 s total -[ 2023-09-21 12:09:34 ] Completed replacing temp checkpoint with checkpoint 39.912 ms, 1.79 s total -[ 2023-09-21 12:09:34 ] Completed eval batch 1 to device 62.496 ms, 1.86 s total -[ 2023-09-21 12:09:34 ] Completed eval batch 1 forward 21.572 ms, 1.88 s total -[ 2023-09-21 12:09:34 ] Completed eval batch 1 recons_loss 0.328 ms, 1.88 s total -[ 2023-09-21 12:09:34 ] Completed eval batch 1 metrics update 222.138 ms, 2.10 s total -Saving checkpoint at epoch 183 val batch 1 -Epoch 183 val loss: 0.0536 -[ 2023-09-21 12:09:35 ] Completed saving temp checkpoint 718.259 ms, 2.82 s total -[ 2023-09-21 12:09:35 ] Completed replacing temp checkpoint with checkpoint 32.616 ms, 2.85 s total -[ 2023-09-21 12:09:35 ] Completed evaluating generator for epoch 183 53.109 ms, 2.91 s total - - -EPOCH :: 184 - - -[ 2023-09-21 12:09:35 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 184 from training step 0 - -[ 2023-09-21 12:09:38 ] Completed train batch 0 to device 3,056.016 ms, 3.06 s total -[ 2023-09-21 12:09:38 ] Completed train batch 0 generator forward 27.357 ms, 3.08 s total -[ 2023-09-21 12:09:38 ] Completed train batch 0 generator loss: 1.255 65.893 ms, 3.15 s total -[ 2023-09-21 12:09:39 ] Completed train batch 0 generator backward 434.521 ms, 3.58 s total -[ 2023-09-21 12:09:39 ] Completed train batch 0 discriminator loss 0.042 56.820 ms, 3.64 s total -[ 2023-09-21 12:09:39 ] Completed train batch 0 discriminator backward 109.093 ms, 3.75 s total -Epoch [184] Step [0/2], gen_loss: 0.419, disc_loss: 0.014 -[ 2023-09-21 12:09:39 ] Completed train batch 0 metrics update 26.213 ms, 3.78 s total -Saving checkpoint at epoch 184 train batch 0 -[ 2023-09-21 12:09:40 ] Completed saving temp checkpoint 660.412 ms, 4.44 s total -[ 2023-09-21 12:09:40 ] Completed replacing temp checkpoint with checkpoint 35.361 ms, 4.47 s total -[ 2023-09-21 12:09:41 ] Completed train batch 1 to device 1,447.497 ms, 5.92 s total -[ 2023-09-21 12:09:41 ] Completed train batch 1 generator forward 40.921 ms, 5.96 s total -[ 2023-09-21 12:09:41 ] Completed train batch 1 generator loss: 1.223 62.770 ms, 6.02 s total -[ 2023-09-21 12:09:42 ] Completed train batch 1 generator backward 1,141.387 ms, 7.16 s total -[ 2023-09-21 12:09:42 ] Completed train batch 1 discriminator loss 0.040 56.528 ms, 7.22 s total -[ 2023-09-21 12:09:43 ] Completed train batch 1 discriminator backward 78.499 ms, 7.30 s total -Epoch [184] Step [1/2], gen_loss: 0.413, disc_loss: 0.014 -[ 2023-09-21 12:09:43 ] Completed train batch 1 metrics update 5.011 ms, 7.30 s total -Saving checkpoint at epoch 184 train batch 1 -[ 2023-09-21 12:09:43 ] Completed saving temp checkpoint 642.069 ms, 7.95 s total -[ 2023-09-21 12:09:43 ] Completed replacing temp checkpoint with checkpoint 23.633 ms, 7.97 s total -Epoch [184] :: gen_loss: 0.416, disc_loss: 0.014 -[ 2023-09-21 12:09:43 ] Completed training generator for epoch 184 67.647 ms, 8.04 s total -[ 2023-09-21 12:09:43 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 184 from eval step 0 - -[ 2023-09-21 12:09:44 ] Completed eval batch 0 to device 1,047.085 ms, 1.05 s total -[ 2023-09-21 12:09:44 ] Completed eval batch 0 forward 28.804 ms, 1.08 s total -[ 2023-09-21 12:09:44 ] Completed eval batch 0 recons_loss 0.304 ms, 1.08 s total -[ 2023-09-21 12:09:44 ] Completed eval batch 0 metrics update 0.618 ms, 1.08 s total -Saving checkpoint at epoch 184 val batch 0 -[ 2023-09-21 12:09:45 ] Completed saving temp checkpoint 604.690 ms, 1.68 s total -[ 2023-09-21 12:09:45 ] Completed replacing temp checkpoint with checkpoint 40.234 ms, 1.72 s total -[ 2023-09-21 12:09:45 ] Completed eval batch 1 to device 194.263 ms, 1.92 s total -[ 2023-09-21 12:09:45 ] Completed eval batch 1 forward 39.394 ms, 1.96 s total -[ 2023-09-21 12:09:45 ] Completed eval batch 1 recons_loss 0.561 ms, 1.96 s total -[ 2023-09-21 12:09:45 ] Completed eval batch 1 metrics update 104.185 ms, 2.06 s total -Saving checkpoint at epoch 184 val batch 1 -Epoch 184 val loss: 0.0520 -[ 2023-09-21 12:09:46 ] Completed saving temp checkpoint 640.058 ms, 2.70 s total -[ 2023-09-21 12:09:46 ] Completed replacing temp checkpoint with checkpoint 42.253 ms, 2.74 s total -[ 2023-09-21 12:09:46 ] Completed evaluating generator for epoch 184 63.184 ms, 2.81 s total - - -EPOCH :: 185 - - -[ 2023-09-21 12:09:46 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 185 from training step 0 - -[ 2023-09-21 12:09:49 ] Completed train batch 0 to device 2,967.089 ms, 2.97 s total -[ 2023-09-21 12:09:49 ] Completed train batch 0 generator forward 30.898 ms, 3.00 s total -[ 2023-09-21 12:09:49 ] Completed train batch 0 generator loss: 1.270 65.516 ms, 3.06 s total -[ 2023-09-21 12:09:50 ] Completed train batch 0 generator backward 635.398 ms, 3.70 s total -[ 2023-09-21 12:09:50 ] Completed train batch 0 discriminator loss 0.042 61.089 ms, 3.76 s total -[ 2023-09-21 12:09:50 ] Completed train batch 0 discriminator backward 99.166 ms, 3.86 s total -Epoch [185] Step [0/2], gen_loss: 0.408, disc_loss: 0.014 -[ 2023-09-21 12:09:50 ] Completed train batch 0 metrics update 24.431 ms, 3.88 s total -Saving checkpoint at epoch 185 train batch 0 -[ 2023-09-21 12:09:51 ] Completed saving temp checkpoint 652.367 ms, 4.54 s total -[ 2023-09-21 12:09:51 ] Completed replacing temp checkpoint with checkpoint 33.555 ms, 4.57 s total -[ 2023-09-21 12:09:52 ] Completed train batch 1 to device 1,129.459 ms, 5.70 s total -[ 2023-09-21 12:09:52 ] Completed train batch 1 generator forward 40.725 ms, 5.74 s total -[ 2023-09-21 12:09:52 ] Completed train batch 1 generator loss: 1.278 62.868 ms, 5.80 s total -[ 2023-09-21 12:09:53 ] Completed train batch 1 generator backward 1,344.056 ms, 7.15 s total -[ 2023-09-21 12:09:53 ] Completed train batch 1 discriminator loss 0.045 55.664 ms, 7.20 s total -[ 2023-09-21 12:09:53 ] Completed train batch 1 discriminator backward 95.327 ms, 7.30 s total -Epoch [185] Step [1/2], gen_loss: 0.414, disc_loss: 0.015 -[ 2023-09-21 12:09:53 ] Completed train batch 1 metrics update 4.228 ms, 7.30 s total -Saving checkpoint at epoch 185 train batch 1 -[ 2023-09-21 12:09:54 ] Completed saving temp checkpoint 646.293 ms, 7.95 s total -[ 2023-09-21 12:09:54 ] Completed replacing temp checkpoint with checkpoint 38.583 ms, 7.99 s total -Epoch [185] :: gen_loss: 0.411, disc_loss: 0.015 -[ 2023-09-21 12:09:54 ] Completed training generator for epoch 185 67.415 ms, 8.05 s total -[ 2023-09-21 12:09:54 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 185 from eval step 0 - -[ 2023-09-21 12:09:55 ] Completed eval batch 0 to device 1,064.657 ms, 1.06 s total -[ 2023-09-21 12:09:55 ] Completed eval batch 0 forward 24.706 ms, 1.09 s total -[ 2023-09-21 12:09:55 ] Completed eval batch 0 recons_loss 0.372 ms, 1.09 s total -[ 2023-09-21 12:09:55 ] Completed eval batch 0 metrics update 0.653 ms, 1.09 s total -Saving checkpoint at epoch 185 val batch 0 -[ 2023-09-21 12:09:56 ] Completed saving temp checkpoint 574.752 ms, 1.67 s total -[ 2023-09-21 12:09:56 ] Completed replacing temp checkpoint with checkpoint 35.158 ms, 1.70 s total -[ 2023-09-21 12:09:56 ] Completed eval batch 1 to device 141.773 ms, 1.84 s total -[ 2023-09-21 12:09:56 ] Completed eval batch 1 forward 21.060 ms, 1.86 s total -[ 2023-09-21 12:09:56 ] Completed eval batch 1 recons_loss 0.340 ms, 1.86 s total -[ 2023-09-21 12:09:56 ] Completed eval batch 1 metrics update 182.295 ms, 2.05 s total -Saving checkpoint at epoch 185 val batch 1 -Epoch 185 val loss: 0.0576 -[ 2023-09-21 12:09:57 ] Completed saving temp checkpoint 614.337 ms, 2.66 s total -[ 2023-09-21 12:09:57 ] Completed replacing temp checkpoint with checkpoint 34.842 ms, 2.69 s total -[ 2023-09-21 12:09:57 ] Completed evaluating generator for epoch 185 53.874 ms, 2.75 s total - - -EPOCH :: 186 - - -[ 2023-09-21 12:09:57 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 186 from training step 0 - -[ 2023-09-21 12:10:00 ] Completed train batch 0 to device 3,007.182 ms, 3.01 s total -[ 2023-09-21 12:10:00 ] Completed train batch 0 generator forward 26.387 ms, 3.03 s total -[ 2023-09-21 12:10:00 ] Completed train batch 0 generator loss: 1.191 67.014 ms, 3.10 s total -[ 2023-09-21 12:10:01 ] Completed train batch 0 generator backward 733.632 ms, 3.83 s total -[ 2023-09-21 12:10:01 ] Completed train batch 0 discriminator loss 0.051 56.597 ms, 3.89 s total -[ 2023-09-21 12:10:01 ] Completed train batch 0 discriminator backward 106.574 ms, 4.00 s total -Epoch [186] Step [0/2], gen_loss: 0.406, disc_loss: 0.017 -[ 2023-09-21 12:10:01 ] Completed train batch 0 metrics update 9.932 ms, 4.01 s total -Saving checkpoint at epoch 186 train batch 0 -[ 2023-09-21 12:10:01 ] Completed saving temp checkpoint 523.603 ms, 4.53 s total -[ 2023-09-21 12:10:01 ] Completed replacing temp checkpoint with checkpoint 34.547 ms, 4.57 s total -[ 2023-09-21 12:10:03 ] Completed train batch 1 to device 1,429.899 ms, 6.00 s total -[ 2023-09-21 12:10:03 ] Completed train batch 1 generator forward 40.693 ms, 6.04 s total -[ 2023-09-21 12:10:03 ] Completed train batch 1 generator loss: 1.168 63.650 ms, 6.10 s total -[ 2023-09-21 12:10:04 ] Completed train batch 1 generator backward 1,459.625 ms, 7.56 s total -[ 2023-09-21 12:10:04 ] Completed train batch 1 discriminator loss 0.049 55.599 ms, 7.61 s total -[ 2023-09-21 12:10:05 ] Completed train batch 1 discriminator backward 74.180 ms, 7.69 s total -Epoch [186] Step [1/2], gen_loss: 0.411, disc_loss: 0.016 -[ 2023-09-21 12:10:05 ] Completed train batch 1 metrics update 1.378 ms, 7.69 s total -Saving checkpoint at epoch 186 train batch 1 -[ 2023-09-21 12:10:05 ] Completed saving temp checkpoint 415.808 ms, 8.11 s total -[ 2023-09-21 12:10:05 ] Completed replacing temp checkpoint with checkpoint 25.966 ms, 8.13 s total -Epoch [186] :: gen_loss: 0.408, disc_loss: 0.017 -[ 2023-09-21 12:10:05 ] Completed training generator for epoch 186 56.432 ms, 8.19 s total -[ 2023-09-21 12:10:05 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 186 from eval step 0 - -[ 2023-09-21 12:10:06 ] Completed eval batch 0 to device 1,056.172 ms, 1.06 s total -[ 2023-09-21 12:10:06 ] Completed eval batch 0 forward 28.235 ms, 1.08 s total -[ 2023-09-21 12:10:06 ] Completed eval batch 0 recons_loss 0.318 ms, 1.08 s total -[ 2023-09-21 12:10:06 ] Completed eval batch 0 metrics update 64.212 ms, 1.15 s total -Saving checkpoint at epoch 186 val batch 0 -[ 2023-09-21 12:10:07 ] Completed saving temp checkpoint 320.335 ms, 1.47 s total -[ 2023-09-21 12:10:07 ] Completed replacing temp checkpoint with checkpoint 27.649 ms, 1.50 s total -[ 2023-09-21 12:10:07 ] Completed eval batch 1 to device 388.145 ms, 1.89 s total -[ 2023-09-21 12:10:07 ] Completed eval batch 1 forward 40.636 ms, 1.93 s total -[ 2023-09-21 12:10:07 ] Completed eval batch 1 recons_loss 0.553 ms, 1.93 s total -[ 2023-09-21 12:10:07 ] Completed eval batch 1 metrics update 470.458 ms, 2.40 s total -Saving checkpoint at epoch 186 val batch 1 -Epoch 186 val loss: 0.0653 -[ 2023-09-21 12:10:08 ] Completed saving temp checkpoint 368.315 ms, 2.77 s total -[ 2023-09-21 12:10:08 ] Completed replacing temp checkpoint with checkpoint 26.730 ms, 2.79 s total -[ 2023-09-21 12:10:08 ] Completed evaluating generator for epoch 186 52.461 ms, 2.84 s total - - -EPOCH :: 187 - - -[ 2023-09-21 12:10:08 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 187 from training step 0 - -[ 2023-09-21 12:10:11 ] Completed train batch 0 to device 2,925.142 ms, 2.93 s total -[ 2023-09-21 12:10:11 ] Completed train batch 0 generator forward 33.871 ms, 2.96 s total -[ 2023-09-21 12:10:11 ] Completed train batch 0 generator loss: 1.301 65.356 ms, 3.02 s total -[ 2023-09-21 12:10:12 ] Completed train batch 0 generator backward 861.857 ms, 3.89 s total -[ 2023-09-21 12:10:12 ] Completed train batch 0 discriminator loss 0.045 56.487 ms, 3.94 s total -[ 2023-09-21 12:10:12 ] Completed train batch 0 discriminator backward 107.322 ms, 4.05 s total -Epoch [187] Step [0/2], gen_loss: 0.420, disc_loss: 0.015 -[ 2023-09-21 12:10:12 ] Completed train batch 0 metrics update 4.270 ms, 4.05 s total -Saving checkpoint at epoch 187 train batch 0 -[ 2023-09-21 12:10:12 ] Completed saving temp checkpoint 368.999 ms, 4.42 s total -[ 2023-09-21 12:10:12 ] Completed replacing temp checkpoint with checkpoint 18.588 ms, 4.44 s total -[ 2023-09-21 12:10:14 ] Completed train batch 1 to device 1,160.039 ms, 5.60 s total -[ 2023-09-21 12:10:14 ] Completed train batch 1 generator forward 23.815 ms, 5.63 s total -[ 2023-09-21 12:10:14 ] Completed train batch 1 generator loss: 1.154 62.893 ms, 5.69 s total -[ 2023-09-21 12:10:15 ] Completed train batch 1 generator backward 1,803.792 ms, 7.49 s total -[ 2023-09-21 12:10:15 ] Completed train batch 1 discriminator loss 0.043 55.849 ms, 7.55 s total -[ 2023-09-21 12:10:16 ] Completed train batch 1 discriminator backward 72.336 ms, 7.62 s total -Epoch [187] Step [1/2], gen_loss: 0.407, disc_loss: 0.015 -[ 2023-09-21 12:10:16 ] Completed train batch 1 metrics update 1.431 ms, 7.62 s total -Saving checkpoint at epoch 187 train batch 1 -[ 2023-09-21 12:10:16 ] Completed saving temp checkpoint 532.572 ms, 8.15 s total -[ 2023-09-21 12:10:16 ] Completed replacing temp checkpoint with checkpoint 17.955 ms, 8.17 s total -Epoch [187] :: gen_loss: 0.414, disc_loss: 0.015 -[ 2023-09-21 12:10:16 ] Completed training generator for epoch 187 70.142 ms, 8.24 s total -[ 2023-09-21 12:10:16 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 187 from eval step 0 - -[ 2023-09-21 12:10:17 ] Completed eval batch 0 to device 1,050.685 ms, 1.05 s total -[ 2023-09-21 12:10:17 ] Completed eval batch 0 forward 29.533 ms, 1.08 s total -[ 2023-09-21 12:10:17 ] Completed eval batch 0 recons_loss 0.360 ms, 1.08 s total -[ 2023-09-21 12:10:17 ] Completed eval batch 0 metrics update 0.580 ms, 1.08 s total -Saving checkpoint at epoch 187 val batch 0 -[ 2023-09-21 12:10:18 ] Completed saving temp checkpoint 383.832 ms, 1.46 s total -[ 2023-09-21 12:10:18 ] Completed replacing temp checkpoint with checkpoint 21.252 ms, 1.49 s total -[ 2023-09-21 12:10:18 ] Completed eval batch 1 to device 457.564 ms, 1.94 s total -[ 2023-09-21 12:10:18 ] Completed eval batch 1 forward 40.154 ms, 1.98 s total -[ 2023-09-21 12:10:18 ] Completed eval batch 1 recons_loss 0.540 ms, 1.98 s total -[ 2023-09-21 12:10:18 ] Completed eval batch 1 metrics update 236.482 ms, 2.22 s total -Saving checkpoint at epoch 187 val batch 1 -Epoch 187 val loss: 0.0614 -[ 2023-09-21 12:10:19 ] Completed saving temp checkpoint 447.948 ms, 2.67 s total -[ 2023-09-21 12:10:19 ] Completed replacing temp checkpoint with checkpoint 23.904 ms, 2.69 s total -[ 2023-09-21 12:10:19 ] Completed evaluating generator for epoch 187 52.807 ms, 2.75 s total - - -EPOCH :: 188 - - -[ 2023-09-21 12:10:19 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 188 from training step 0 - -[ 2023-09-21 12:10:22 ] Completed train batch 0 to device 3,012.585 ms, 3.01 s total -[ 2023-09-21 12:10:22 ] Completed train batch 0 generator forward 42.225 ms, 3.05 s total -[ 2023-09-21 12:10:22 ] Completed train batch 0 generator loss: 1.187 65.096 ms, 3.12 s total -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 12:22:54 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 12:22:54 ] Completed importing Timer 0.024 ms, 0.00 s total -[ 2023-09-21 12:22:58 ] Completed importing everything else 4,479.507 ms, 4.48 s total -| distributed init (rank 0): env:// -| distributed init (rank 3): env:// -| distributed init (rank 2): env:// -| distributed init (rank 5): env:// -| distributed init (rank 4): env:// -| distributed init (rank 1): env:// -[ 2023-09-21 12:23:06 ] Completed preliminaries 7,543.978 ms, 12.02 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 12:23:06 ] Completed build datasets 10.945 ms, 12.03 s total -[ 2023-09-21 12:23:06 ] Completed build samplers 0.045 ms, 12.03 s total -[ 2023-09-21 12:23:06 ] Completed build dataloaders 0.132 ms, 12.03 s total -[ 2023-09-21 12:23:06 ] Completed generator to device 95.311 ms, 12.13 s total -[ 2023-09-21 12:23:06 ] Completed discriminator to device 7.136 ms, 12.14 s total -[ 2023-09-21 12:23:06 ] Completed loss functions 440.268 ms, 12.58 s total -[ 2023-09-21 12:23:06 ] Completed models prepped for distribution 33.044 ms, 12.61 s total -[ 2023-09-21 12:23:06 ] Completed optimizers 0.683 ms, 12.61 s total -[ 2023-09-21 12:23:06 ] Completed grad scalers 0.028 ms, 12.61 s total -[ 2023-09-21 12:23:07 ] Completed checkpoint retrieval 465.850 ms, 13.08 s total - - -EPOCH :: 187 - - -[ 2023-09-21 12:23:07 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 187 from training step 2 - -Epoch [187] :: gen_loss: 0.414, disc_loss: 0.015 -[ 2023-09-21 12:23:07 ] Completed training generator for epoch 187 128.920 ms, 0.13 s total -[ 2023-09-21 12:23:07 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 187 from eval step 2 - -[ 2023-09-21 12:23:07 ] Completed evaluating generator for epoch 187 132.591 ms, 0.13 s total - - -EPOCH :: 188 - - -[ 2023-09-21 12:23:07 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 188 from training step 0 - -[ 2023-09-21 12:23:11 ] Completed train batch 0 to device 3,763.023 ms, 3.76 s total -[ 2023-09-21 12:23:12 ] Completed train batch 0 generator forward 1,493.576 ms, 5.26 s total -[ 2023-09-21 12:23:12 ] Completed train batch 0 generator loss: 1.240 76.171 ms, 5.33 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 12:23:14 ] Completed train batch 0 generator backward 1,430.317 ms, 6.76 s total -[ 2023-09-21 12:23:14 ] Completed train batch 0 discriminator loss 0.044 61.575 ms, 6.82 s total -[ 2023-09-21 12:23:14 ] Completed train batch 0 discriminator backward 97.384 ms, 6.92 s total -Epoch [188] Step [0/2], gen_loss: 0.413, disc_loss: 0.014 -[ 2023-09-21 12:23:14 ] Completed train batch 0 metrics update 6.181 ms, 6.93 s total -Saving checkpoint at epoch 188 train batch 0 -[ 2023-09-21 12:23:14 ] Completed saving temp checkpoint 463.299 ms, 7.39 s total -[ 2023-09-21 12:23:14 ] Completed replacing temp checkpoint with checkpoint 125.565 ms, 7.52 s total -[ 2023-09-21 12:23:15 ] Completed train batch 1 to device 21.164 ms, 7.54 s total -[ 2023-09-21 12:23:15 ] Completed train batch 1 generator forward 36.600 ms, 7.57 s total -[ 2023-09-21 12:23:15 ] Completed train batch 1 generator loss: 1.227 61.871 ms, 7.64 s total -[ 2023-09-21 12:23:16 ] Completed train batch 1 generator backward 1,283.026 ms, 8.92 s total -[ 2023-09-21 12:23:16 ] Completed train batch 1 discriminator loss 0.045 56.626 ms, 8.98 s total -[ 2023-09-21 12:23:16 ] Completed train batch 1 discriminator backward 78.775 ms, 9.06 s total -Epoch [188] Step [1/2], gen_loss: 0.408, disc_loss: 0.014 -[ 2023-09-21 12:23:16 ] Completed train batch 1 metrics update 2.633 ms, 9.06 s total -Saving checkpoint at epoch 188 train batch 1 -[ 2023-09-21 12:23:17 ] Completed saving temp checkpoint 492.256 ms, 9.55 s total -[ 2023-09-21 12:23:17 ] Completed replacing temp checkpoint with checkpoint 30.528 ms, 9.58 s total -Epoch [188] :: gen_loss: 0.411, disc_loss: 0.014 -[ 2023-09-21 12:23:17 ] Completed training generator for epoch 188 44.266 ms, 9.62 s total -[ 2023-09-21 12:23:17 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 188 from eval step 0 - -[ 2023-09-21 12:23:18 ] Completed eval batch 0 to device 1,137.692 ms, 1.14 s total -[ 2023-09-21 12:23:18 ] Completed eval batch 0 forward 27.644 ms, 1.17 s total -[ 2023-09-21 12:23:18 ] Completed eval batch 0 recons_loss 0.375 ms, 1.17 s total -[ 2023-09-21 12:23:18 ] Completed eval batch 0 metrics update 0.741 ms, 1.17 s total -Saving checkpoint at epoch 188 val batch 0 -[ 2023-09-21 12:23:18 ] Completed saving temp checkpoint 423.826 ms, 1.59 s total -[ 2023-09-21 12:23:18 ] Completed replacing temp checkpoint with checkpoint 20.268 ms, 1.61 s total -[ 2023-09-21 12:23:19 ] Completed eval batch 1 to device 420.724 ms, 2.03 s total -[ 2023-09-21 12:23:19 ] Completed eval batch 1 forward 39.800 ms, 2.07 s total -[ 2023-09-21 12:23:19 ] Completed eval batch 1 recons_loss 0.624 ms, 2.07 s total -[ 2023-09-21 12:23:19 ] Completed eval batch 1 metrics update 165.474 ms, 2.24 s total -Saving checkpoint at epoch 188 val batch 1 -Epoch 188 val loss: 0.0583 -[ 2023-09-21 12:23:19 ] Completed saving temp checkpoint 449.881 ms, 2.69 s total -[ 2023-09-21 12:23:19 ] Completed replacing temp checkpoint with checkpoint 30.888 ms, 2.72 s total -[ 2023-09-21 12:23:19 ] Completed evaluating generator for epoch 188 52.674 ms, 2.77 s total - - -EPOCH :: 189 - - -[ 2023-09-21 12:23:19 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 189 from training step 0 - -[ 2023-09-21 12:23:22 ] Completed train batch 0 to device 3,083.714 ms, 3.08 s total -[ 2023-09-21 12:23:22 ] Completed train batch 0 generator forward 33.438 ms, 3.12 s total -[ 2023-09-21 12:23:23 ] Completed train batch 0 generator loss: 1.199 67.598 ms, 3.18 s total -[ 2023-09-21 12:23:23 ] Completed train batch 0 generator backward 713.460 ms, 3.90 s total -[ 2023-09-21 12:23:23 ] Completed train batch 0 discriminator loss 0.043 58.003 ms, 3.96 s total -[ 2023-09-21 12:23:23 ] Completed train batch 0 discriminator backward 117.793 ms, 4.07 s total -Epoch [189] Step [0/2], gen_loss: 0.410, disc_loss: 0.014 -[ 2023-09-21 12:23:23 ] Completed train batch 0 metrics update 15.029 ms, 4.09 s total -Saving checkpoint at epoch 189 train batch 0 -[ 2023-09-21 12:23:24 ] Completed saving temp checkpoint 459.088 ms, 4.55 s total -[ 2023-09-21 12:23:24 ] Completed replacing temp checkpoint with checkpoint 33.252 ms, 4.58 s total -[ 2023-09-21 12:23:25 ] Completed train batch 1 to device 1,278.973 ms, 5.86 s total -[ 2023-09-21 12:23:25 ] Completed train batch 1 generator forward 22.148 ms, 5.88 s total -[ 2023-09-21 12:23:25 ] Completed train batch 1 generator loss: 1.149 63.396 ms, 5.95 s total -[ 2023-09-21 12:23:27 ] Completed train batch 1 generator backward 1,362.920 ms, 7.31 s total -[ 2023-09-21 12:23:27 ] Completed train batch 1 discriminator loss 0.041 56.368 ms, 7.37 s total -[ 2023-09-21 12:23:27 ] Completed train batch 1 discriminator backward 90.904 ms, 7.46 s total -Epoch [189] Step [1/2], gen_loss: 0.413, disc_loss: 0.014 -[ 2023-09-21 12:23:27 ] Completed train batch 1 metrics update 5.716 ms, 7.46 s total -Saving checkpoint at epoch 189 train batch 1 -[ 2023-09-21 12:23:27 ] Completed saving temp checkpoint 489.600 ms, 7.95 s total -[ 2023-09-21 12:23:27 ] Completed replacing temp checkpoint with checkpoint 23.439 ms, 7.97 s total -Epoch [189] :: gen_loss: 0.412, disc_loss: 0.014 -[ 2023-09-21 12:23:27 ] Completed training generator for epoch 189 69.021 ms, 8.04 s total -[ 2023-09-21 12:23:27 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 189 from eval step 0 - -[ 2023-09-21 12:23:29 ] Completed eval batch 0 to device 1,097.442 ms, 1.10 s total -[ 2023-09-21 12:23:29 ] Completed eval batch 0 forward 24.745 ms, 1.12 s total -[ 2023-09-21 12:23:29 ] Completed eval batch 0 recons_loss 0.304 ms, 1.12 s total -[ 2023-09-21 12:23:29 ] Completed eval batch 0 metrics update 0.587 ms, 1.12 s total -Saving checkpoint at epoch 189 val batch 0 -[ 2023-09-21 12:23:29 ] Completed saving temp checkpoint 415.682 ms, 1.54 s total -[ 2023-09-21 12:23:29 ] Completed replacing temp checkpoint with checkpoint 30.897 ms, 1.57 s total -[ 2023-09-21 12:23:29 ] Completed eval batch 1 to device 405.033 ms, 1.97 s total -[ 2023-09-21 12:23:29 ] Completed eval batch 1 forward 39.887 ms, 2.01 s total -[ 2023-09-21 12:23:29 ] Completed eval batch 1 recons_loss 0.332 ms, 2.01 s total -[ 2023-09-21 12:23:30 ] Completed eval batch 1 metrics update 261.545 ms, 2.28 s total -Saving checkpoint at epoch 189 val batch 1 -Epoch 189 val loss: 0.0676 -[ 2023-09-21 12:23:30 ] Completed saving temp checkpoint 456.603 ms, 2.73 s total -[ 2023-09-21 12:23:30 ] Completed replacing temp checkpoint with checkpoint 29.096 ms, 2.76 s total -[ 2023-09-21 12:23:30 ] Completed evaluating generator for epoch 189 48.669 ms, 2.81 s total - - -EPOCH :: 190 - - -[ 2023-09-21 12:23:30 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 190 from training step 0 - -[ 2023-09-21 12:23:33 ] Completed train batch 0 to device 3,187.687 ms, 3.19 s total -[ 2023-09-21 12:23:33 ] Completed train batch 0 generator forward 33.726 ms, 3.22 s total -[ 2023-09-21 12:23:34 ] Completed train batch 0 generator loss: 1.236 73.204 ms, 3.29 s total -[ 2023-09-21 12:23:34 ] Completed train batch 0 generator backward 545.387 ms, 3.84 s total -[ 2023-09-21 12:23:34 ] Completed train batch 0 discriminator loss 0.041 58.661 ms, 3.90 s total -[ 2023-09-21 12:23:34 ] Completed train batch 0 discriminator backward 100.072 ms, 4.00 s total -Epoch [190] Step [0/2], gen_loss: 0.417, disc_loss: 0.014 -[ 2023-09-21 12:23:34 ] Completed train batch 0 metrics update 5.955 ms, 4.00 s total -Saving checkpoint at epoch 190 train batch 0 -[ 2023-09-21 12:23:35 ] Completed saving temp checkpoint 432.377 ms, 4.44 s total -[ 2023-09-21 12:23:35 ] Completed replacing temp checkpoint with checkpoint 21.446 ms, 4.46 s total -[ 2023-09-21 12:23:36 ] Completed train batch 1 to device 1,600.673 ms, 6.06 s total -[ 2023-09-21 12:23:36 ] Completed train batch 1 generator forward 22.378 ms, 6.08 s total -[ 2023-09-21 12:23:36 ] Completed train batch 1 generator loss: 1.306 66.371 ms, 6.15 s total -[ 2023-09-21 12:23:37 ] Completed train batch 1 generator backward 1,100.626 ms, 7.25 s total -[ 2023-09-21 12:23:38 ] Completed train batch 1 discriminator loss 0.039 56.464 ms, 7.31 s total -[ 2023-09-21 12:23:38 ] Completed train batch 1 discriminator backward 93.762 ms, 7.40 s total -Epoch [190] Step [1/2], gen_loss: 0.413, disc_loss: 0.013 -[ 2023-09-21 12:23:38 ] Completed train batch 1 metrics update 6.119 ms, 7.40 s total -Saving checkpoint at epoch 190 train batch 1 -[ 2023-09-21 12:23:38 ] Completed saving temp checkpoint 447.979 ms, 7.85 s total -[ 2023-09-21 12:23:38 ] Completed replacing temp checkpoint with checkpoint 16.709 ms, 7.87 s total -Epoch [190] :: gen_loss: 0.415, disc_loss: 0.013 -[ 2023-09-21 12:23:38 ] Completed training generator for epoch 190 54.859 ms, 7.92 s total -[ 2023-09-21 12:23:38 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 190 from eval step 0 - -[ 2023-09-21 12:23:39 ] Completed eval batch 0 to device 1,214.441 ms, 1.21 s total -[ 2023-09-21 12:23:39 ] Completed eval batch 0 forward 28.336 ms, 1.24 s total -[ 2023-09-21 12:23:39 ] Completed eval batch 0 recons_loss 0.385 ms, 1.24 s total -[ 2023-09-21 12:23:39 ] Completed eval batch 0 metrics update 0.796 ms, 1.24 s total -Saving checkpoint at epoch 190 val batch 0 -[ 2023-09-21 12:23:40 ] Completed saving temp checkpoint 401.555 ms, 1.65 s total -[ 2023-09-21 12:23:40 ] Completed replacing temp checkpoint with checkpoint 26.941 ms, 1.67 s total -[ 2023-09-21 12:23:40 ] Completed eval batch 1 to device 572.058 ms, 2.24 s total -[ 2023-09-21 12:23:40 ] Completed eval batch 1 forward 36.860 ms, 2.28 s total -[ 2023-09-21 12:23:40 ] Completed eval batch 1 recons_loss 0.357 ms, 2.28 s total -[ 2023-09-21 12:23:40 ] Completed eval batch 1 metrics update 31.269 ms, 2.31 s total -Saving checkpoint at epoch 190 val batch 1 -Epoch 190 val loss: 0.0593 -[ 2023-09-21 12:23:41 ] Completed saving temp checkpoint 427.613 ms, 2.74 s total -[ 2023-09-21 12:23:41 ] Completed replacing temp checkpoint with checkpoint 22.912 ms, 2.76 s total -[ 2023-09-21 12:23:41 ] Completed evaluating generator for epoch 190 52.895 ms, 2.82 s total - - -EPOCH :: 191 - - -[ 2023-09-21 12:23:41 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 191 from training step 0 - -[ 2023-09-21 12:23:44 ] Completed train batch 0 to device 3,117.582 ms, 3.12 s total -[ 2023-09-21 12:23:44 ] Completed train batch 0 generator forward 25.376 ms, 3.14 s total -[ 2023-09-21 12:23:44 ] Completed train batch 0 generator loss: 1.159 65.185 ms, 3.21 s total -[ 2023-09-21 12:23:45 ] Completed train batch 0 generator backward 633.837 ms, 3.84 s total -[ 2023-09-21 12:23:45 ] Completed train batch 0 discriminator loss 0.039 57.189 ms, 3.90 s total -[ 2023-09-21 12:23:45 ] Completed train batch 0 discriminator backward 112.665 ms, 4.01 s total -Epoch [191] Step [0/2], gen_loss: 0.411, disc_loss: 0.013 -[ 2023-09-21 12:23:45 ] Completed train batch 0 metrics update 5.266 ms, 4.02 s total -Saving checkpoint at epoch 191 train batch 0 -[ 2023-09-21 12:23:45 ] Completed saving temp checkpoint 473.872 ms, 4.49 s total -[ 2023-09-21 12:23:45 ] Completed replacing temp checkpoint with checkpoint 23.995 ms, 4.51 s total -[ 2023-09-21 12:23:47 ] Completed train batch 1 to device 1,489.385 ms, 6.00 s total -[ 2023-09-21 12:23:47 ] Completed train batch 1 generator forward 39.320 ms, 6.04 s total -[ 2023-09-21 12:23:47 ] Completed train batch 1 generator loss: 1.262 66.269 ms, 6.11 s total -[ 2023-09-21 12:23:48 ] Completed train batch 1 generator backward 1,413.970 ms, 7.52 s total -[ 2023-09-21 12:23:49 ] Completed train batch 1 discriminator loss 0.038 57.051 ms, 7.58 s total -[ 2023-09-21 12:23:49 ] Completed train batch 1 discriminator backward 85.943 ms, 7.67 s total -Epoch [191] Step [1/2], gen_loss: 0.412, disc_loss: 0.013 -[ 2023-09-21 12:23:49 ] Completed train batch 1 metrics update 5.018 ms, 7.67 s total -Saving checkpoint at epoch 191 train batch 1 -[ 2023-09-21 12:23:49 ] Completed saving temp checkpoint 446.736 ms, 8.12 s total -[ 2023-09-21 12:23:49 ] Completed replacing temp checkpoint with checkpoint 15.192 ms, 8.13 s total -Epoch [191] :: gen_loss: 0.411, disc_loss: 0.013 -[ 2023-09-21 12:23:49 ] Completed training generator for epoch 191 57.947 ms, 8.19 s total -[ 2023-09-21 12:23:49 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 191 from eval step 0 - -[ 2023-09-21 12:23:50 ] Completed eval batch 0 to device 1,157.330 ms, 1.16 s total -[ 2023-09-21 12:23:50 ] Completed eval batch 0 forward 31.169 ms, 1.19 s total -[ 2023-09-21 12:23:50 ] Completed eval batch 0 recons_loss 0.293 ms, 1.19 s total -[ 2023-09-21 12:23:50 ] Completed eval batch 0 metrics update 0.539 ms, 1.19 s total -Saving checkpoint at epoch 191 val batch 0 -[ 2023-09-21 12:23:51 ] Completed saving temp checkpoint 412.478 ms, 1.60 s total -[ 2023-09-21 12:23:51 ] Completed replacing temp checkpoint with checkpoint 24.637 ms, 1.63 s total -[ 2023-09-21 12:23:51 ] Completed eval batch 1 to device 563.473 ms, 2.19 s total -[ 2023-09-21 12:23:51 ] Completed eval batch 1 forward 40.032 ms, 2.23 s total -[ 2023-09-21 12:23:51 ] Completed eval batch 1 recons_loss 0.423 ms, 2.23 s total -[ 2023-09-21 12:23:52 ] Completed eval batch 1 metrics update 200.085 ms, 2.43 s total -Saving checkpoint at epoch 191 val batch 1 -Epoch 191 val loss: 0.0552 -[ 2023-09-21 12:23:52 ] Completed saving temp checkpoint 452.547 ms, 2.88 s total -[ 2023-09-21 12:23:52 ] Completed replacing temp checkpoint with checkpoint 25.730 ms, 2.91 s total -[ 2023-09-21 12:23:52 ] Completed evaluating generator for epoch 191 48.637 ms, 2.96 s total - - -EPOCH :: 192 - - -[ 2023-09-21 12:23:52 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 192 from training step 0 - -[ 2023-09-21 12:23:55 ] Completed train batch 0 to device 3,241.744 ms, 3.24 s total -[ 2023-09-21 12:23:55 ] Completed train batch 0 generator forward 25.753 ms, 3.27 s total -[ 2023-09-21 12:23:55 ] Completed train batch 0 generator loss: 1.247 65.247 ms, 3.33 s total -[ 2023-09-21 12:23:56 ] Completed train batch 0 generator backward 547.491 ms, 3.88 s total -[ 2023-09-21 12:23:56 ] Completed train batch 0 discriminator loss 0.038 57.796 ms, 3.94 s total -[ 2023-09-21 12:23:56 ] Completed train batch 0 discriminator backward 99.364 ms, 4.04 s total -Epoch [192] Step [0/2], gen_loss: 0.411, disc_loss: 0.013 -[ 2023-09-21 12:23:56 ] Completed train batch 0 metrics update 7.102 ms, 4.04 s total -Saving checkpoint at epoch 192 train batch 0 -[ 2023-09-21 12:23:57 ] Completed saving temp checkpoint 483.848 ms, 4.53 s total -[ 2023-09-21 12:23:57 ] Completed replacing temp checkpoint with checkpoint 26.575 ms, 4.55 s total -[ 2023-09-21 12:23:58 ] Completed train batch 1 to device 1,691.256 ms, 6.25 s total -[ 2023-09-21 12:23:58 ] Completed train batch 1 generator forward 43.603 ms, 6.29 s total -[ 2023-09-21 12:23:58 ] Completed train batch 1 generator loss: 1.251 67.249 ms, 6.36 s total -[ 2023-09-21 12:23:59 ] Completed train batch 1 generator backward 1,027.922 ms, 7.38 s total -[ 2023-09-21 12:24:00 ] Completed train batch 1 discriminator loss 0.042 58.045 ms, 7.44 s total -[ 2023-09-21 12:24:00 ] Completed train batch 1 discriminator backward 58.829 ms, 7.50 s total -Epoch [192] Step [1/2], gen_loss: 0.415, disc_loss: 0.014 -[ 2023-09-21 12:24:00 ] Completed train batch 1 metrics update 2.288 ms, 7.50 s total -Saving checkpoint at epoch 192 train batch 1 -[ 2023-09-21 12:24:00 ] Completed saving temp checkpoint 665.600 ms, 8.17 s total -[ 2023-09-21 12:24:00 ] Completed replacing temp checkpoint with checkpoint 31.574 ms, 8.20 s total -Epoch [192] :: gen_loss: 0.413, disc_loss: 0.013 -[ 2023-09-21 12:24:00 ] Completed training generator for epoch 192 55.774 ms, 8.26 s total -[ 2023-09-21 12:24:00 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 192 from eval step 0 - -[ 2023-09-21 12:24:01 ] Completed eval batch 0 to device 1,080.124 ms, 1.08 s total -[ 2023-09-21 12:24:01 ] Completed eval batch 0 forward 25.499 ms, 1.11 s total -[ 2023-09-21 12:24:01 ] Completed eval batch 0 recons_loss 0.321 ms, 1.11 s total -[ 2023-09-21 12:24:01 ] Completed eval batch 0 metrics update 0.626 ms, 1.11 s total -Saving checkpoint at epoch 192 val batch 0 -[ 2023-09-21 12:24:02 ] Completed saving temp checkpoint 632.848 ms, 1.74 s total -[ 2023-09-21 12:24:02 ] Completed replacing temp checkpoint with checkpoint 19.759 ms, 1.76 s total -[ 2023-09-21 12:24:02 ] Completed eval batch 1 to device 166.814 ms, 1.93 s total -[ 2023-09-21 12:24:02 ] Completed eval batch 1 forward 21.700 ms, 1.95 s total -[ 2023-09-21 12:24:02 ] Completed eval batch 1 recons_loss 0.379 ms, 1.95 s total -[ 2023-09-21 12:24:02 ] Completed eval batch 1 metrics update 164.777 ms, 2.11 s total -Saving checkpoint at epoch 192 val batch 1 -Epoch 192 val loss: 0.0572 -[ 2023-09-21 12:24:03 ] Completed saving temp checkpoint 665.400 ms, 2.78 s total -[ 2023-09-21 12:24:03 ] Completed replacing temp checkpoint with checkpoint 19.320 ms, 2.80 s total -[ 2023-09-21 12:24:03 ] Completed evaluating generator for epoch 192 48.380 ms, 2.85 s total - - -EPOCH :: 193 - - -[ 2023-09-21 12:24:03 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 193 from training step 0 - -[ 2023-09-21 12:24:06 ] Completed train batch 0 to device 3,119.316 ms, 3.12 s total -[ 2023-09-21 12:24:06 ] Completed train batch 0 generator forward 29.544 ms, 3.15 s total -[ 2023-09-21 12:24:06 ] Completed train batch 0 generator loss: 1.198 67.361 ms, 3.22 s total -[ 2023-09-21 12:24:07 ] Completed train batch 0 generator backward 406.095 ms, 3.62 s total -[ 2023-09-21 12:24:07 ] Completed train batch 0 discriminator loss 0.042 57.192 ms, 3.68 s total -[ 2023-09-21 12:24:07 ] Completed train batch 0 discriminator backward 106.393 ms, 3.79 s total -Epoch [193] Step [0/2], gen_loss: 0.412, disc_loss: 0.014 -[ 2023-09-21 12:24:07 ] Completed train batch 0 metrics update 7.818 ms, 3.79 s total -Saving checkpoint at epoch 193 train batch 0 -[ 2023-09-21 12:24:08 ] Completed saving temp checkpoint 687.953 ms, 4.48 s total -[ 2023-09-21 12:24:08 ] Completed replacing temp checkpoint with checkpoint 34.313 ms, 4.52 s total -[ 2023-09-21 12:24:09 ] Completed train batch 1 to device 1,349.279 ms, 5.87 s total -[ 2023-09-21 12:24:09 ] Completed train batch 1 generator forward 41.304 ms, 5.91 s total -[ 2023-09-21 12:24:09 ] Completed train batch 1 generator loss: 1.263 64.711 ms, 5.97 s total -[ 2023-09-21 12:24:10 ] Completed train batch 1 generator backward 1,148.921 ms, 7.12 s total -[ 2023-09-21 12:24:10 ] Completed train batch 1 discriminator loss 0.042 56.679 ms, 7.18 s total -[ 2023-09-21 12:24:10 ] Completed train batch 1 discriminator backward 76.811 ms, 7.25 s total -Epoch [193] Step [1/2], gen_loss: 0.414, disc_loss: 0.014 -[ 2023-09-21 12:24:10 ] Completed train batch 1 metrics update 3.243 ms, 7.26 s total -Saving checkpoint at epoch 193 train batch 1 -[ 2023-09-21 12:24:11 ] Completed saving temp checkpoint 708.120 ms, 7.97 s total -[ 2023-09-21 12:24:11 ] Completed replacing temp checkpoint with checkpoint 25.528 ms, 7.99 s total -Epoch [193] :: gen_loss: 0.413, disc_loss: 0.014 -[ 2023-09-21 12:24:11 ] Completed training generator for epoch 193 58.442 ms, 8.05 s total -[ 2023-09-21 12:24:11 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 193 from eval step 0 - -[ 2023-09-21 12:24:12 ] Completed eval batch 0 to device 1,056.693 ms, 1.06 s total -[ 2023-09-21 12:24:12 ] Completed eval batch 0 forward 43.483 ms, 1.10 s total -[ 2023-09-21 12:24:12 ] Completed eval batch 0 recons_loss 0.371 ms, 1.10 s total -[ 2023-09-21 12:24:12 ] Completed eval batch 0 metrics update 0.662 ms, 1.10 s total -Saving checkpoint at epoch 193 val batch 0 -[ 2023-09-21 12:24:13 ] Completed saving temp checkpoint 667.856 ms, 1.77 s total -[ 2023-09-21 12:24:13 ] Completed replacing temp checkpoint with checkpoint 40.373 ms, 1.81 s total -[ 2023-09-21 12:24:13 ] Completed eval batch 1 to device 187.863 ms, 2.00 s total -[ 2023-09-21 12:24:13 ] Completed eval batch 1 forward 40.514 ms, 2.04 s total -[ 2023-09-21 12:24:13 ] Completed eval batch 1 recons_loss 0.571 ms, 2.04 s total -[ 2023-09-21 12:24:13 ] Completed eval batch 1 metrics update 0.507 ms, 2.04 s total -Saving checkpoint at epoch 193 val batch 1 -Epoch 193 val loss: 0.0598 -[ 2023-09-21 12:24:14 ] Completed saving temp checkpoint 665.127 ms, 2.70 s total -[ 2023-09-21 12:24:14 ] Completed replacing temp checkpoint with checkpoint 40.411 ms, 2.74 s total -[ 2023-09-21 12:24:14 ] Completed evaluating generator for epoch 193 57.136 ms, 2.80 s total - - -EPOCH :: 194 - - -[ 2023-09-21 12:24:14 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 194 from training step 0 - -[ 2023-09-21 12:24:17 ] Completed train batch 0 to device 2,898.174 ms, 2.90 s total -[ 2023-09-21 12:24:17 ] Completed train batch 0 generator forward 42.895 ms, 2.94 s total -[ 2023-09-21 12:24:17 ] Completed train batch 0 generator loss: 1.141 65.994 ms, 3.01 s total -[ 2023-09-21 12:24:18 ] Completed train batch 0 generator backward 613.431 ms, 3.62 s total -[ 2023-09-21 12:24:18 ] Completed train batch 0 discriminator loss 0.043 57.006 ms, 3.68 s total -[ 2023-09-21 12:24:18 ] Completed train batch 0 discriminator backward 97.453 ms, 3.77 s total -Epoch [194] Step [0/2], gen_loss: 0.412, disc_loss: 0.014 -[ 2023-09-21 12:24:18 ] Completed train batch 0 metrics update 5.236 ms, 3.78 s total -Saving checkpoint at epoch 194 train batch 0 -[ 2023-09-21 12:24:18 ] Completed saving temp checkpoint 639.113 ms, 4.42 s total -[ 2023-09-21 12:24:19 ] Completed replacing temp checkpoint with checkpoint 36.680 ms, 4.46 s total -[ 2023-09-21 12:24:20 ] Completed train batch 1 to device 1,197.758 ms, 5.65 s total -[ 2023-09-21 12:24:20 ] Completed train batch 1 generator forward 26.874 ms, 5.68 s total -[ 2023-09-21 12:24:20 ] Completed train batch 1 generator loss: 1.193 210.179 ms, 5.89 s total -[ 2023-09-21 12:24:21 ] Completed train batch 1 generator backward 1,169.166 ms, 7.06 s total -[ 2023-09-21 12:24:21 ] Completed train batch 1 discriminator loss 0.040 56.419 ms, 7.12 s total -[ 2023-09-21 12:24:21 ] Completed train batch 1 discriminator backward 74.362 ms, 7.19 s total -Epoch [194] Step [1/2], gen_loss: 0.414, disc_loss: 0.014 -[ 2023-09-21 12:24:21 ] Completed train batch 1 metrics update 5.440 ms, 7.20 s total -Saving checkpoint at epoch 194 train batch 1 -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -[ 2023-09-21 12:36:53 ] Completed Start 0.000 ms, 0.00 s total -[ 2023-09-21 12:36:53 ] Completed importing Timer 0.023 ms, 0.00 s total -[ 2023-09-21 12:36:57 ] Completed importing everything else 4,671.341 ms, 4.67 s total -| distributed init (rank 0): env:// -| distributed init (rank 5): env:// -| distributed init (rank 1): env:// -| distributed init (rank 3): env:// -| distributed init (rank 2): env:// -| distributed init (rank 4): env:// -[ 2023-09-21 12:37:05 ] Completed preliminaries 7,707.168 ms, 12.38 s total -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -[ 2023-09-21 12:37:05 ] Completed build datasets 11.271 ms, 12.39 s total -[ 2023-09-21 12:37:05 ] Completed build samplers 0.044 ms, 12.39 s total -[ 2023-09-21 12:37:05 ] Completed build dataloaders 0.130 ms, 12.39 s total -[ 2023-09-21 12:37:05 ] Completed generator to device 91.123 ms, 12.48 s total -[ 2023-09-21 12:37:05 ] Completed discriminator to device 6.993 ms, 12.49 s total -[ 2023-09-21 12:37:06 ] Completed loss functions 394.935 ms, 12.88 s total -[ 2023-09-21 12:37:06 ] Completed models prepped for distribution 71.491 ms, 12.95 s total -[ 2023-09-21 12:37:06 ] Completed optimizers 0.679 ms, 12.96 s total -[ 2023-09-21 12:37:06 ] Completed grad scalers 0.035 ms, 12.96 s total -[ 2023-09-21 12:37:06 ] Completed checkpoint retrieval 502.733 ms, 13.46 s total - - -EPOCH :: 194 - - -[ 2023-09-21 12:37:06 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 194 from training step 1 - -[ 2023-09-21 12:37:10 ] Completed train batch 1 to device 3,623.676 ms, 3.62 s total -[ 2023-09-21 12:37:11 ] Completed train batch 1 generator forward 1,125.412 ms, 4.75 s total -[ 2023-09-21 12:37:11 ] Completed train batch 1 generator loss: 1.270 76.517 ms, 4.83 s total -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -[ 2023-09-21 12:37:12 ] Completed train batch 1 generator backward 1,308.666 ms, 6.13 s total -[ 2023-09-21 12:37:12 ] Completed train batch 1 discriminator loss 0.045 65.729 ms, 6.20 s total -[ 2023-09-21 12:37:12 ] Completed train batch 1 discriminator backward 62.404 ms, 6.26 s total -Epoch [194] Step [1/2], gen_loss: 0.414, disc_loss: 0.014 -[ 2023-09-21 12:37:12 ] Completed train batch 1 metrics update 1.571 ms, 6.26 s total -Saving checkpoint at epoch 194 train batch 1 -[ 2023-09-21 12:37:13 ] Completed saving temp checkpoint 451.428 ms, 6.72 s total -[ 2023-09-21 12:37:13 ] Completed replacing temp checkpoint with checkpoint 141.186 ms, 6.86 s total -Epoch [194] :: gen_loss: 0.413, disc_loss: 0.014 -[ 2023-09-21 12:37:13 ] Completed training generator for epoch 194 56.728 ms, 6.91 s total -[ 2023-09-21 12:37:13 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 194 from eval step 0 - -[ 2023-09-21 12:37:14 ] Completed eval batch 0 to device 1,055.535 ms, 1.06 s total -[ 2023-09-21 12:37:14 ] Completed eval batch 0 forward 26.940 ms, 1.08 s total -[ 2023-09-21 12:37:14 ] Completed eval batch 0 recons_loss 0.374 ms, 1.08 s total -[ 2023-09-21 12:37:14 ] Completed eval batch 0 metrics update 0.803 ms, 1.08 s total -Saving checkpoint at epoch 194 val batch 0 -[ 2023-09-21 12:37:15 ] Completed saving temp checkpoint 424.888 ms, 1.51 s total -[ 2023-09-21 12:37:15 ] Completed replacing temp checkpoint with checkpoint 31.211 ms, 1.54 s total -[ 2023-09-21 12:37:15 ] Completed eval batch 1 to device 472.777 ms, 2.01 s total -[ 2023-09-21 12:37:15 ] Completed eval batch 1 forward 26.040 ms, 2.04 s total -[ 2023-09-21 12:37:15 ] Completed eval batch 1 recons_loss 0.359 ms, 2.04 s total -[ 2023-09-21 12:37:15 ] Completed eval batch 1 metrics update 43.657 ms, 2.08 s total -Saving checkpoint at epoch 194 val batch 1 -Epoch 194 val loss: 0.0665 -[ 2023-09-21 12:37:16 ] Completed saving temp checkpoint 404.610 ms, 2.49 s total -[ 2023-09-21 12:37:16 ] Completed replacing temp checkpoint with checkpoint 30.426 ms, 2.52 s total -[ 2023-09-21 12:37:16 ] Completed evaluating generator for epoch 194 43.911 ms, 2.56 s total - - -EPOCH :: 195 - - -[ 2023-09-21 12:37:16 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 195 from training step 0 - -[ 2023-09-21 12:37:18 ] Completed train batch 0 to device 2,836.128 ms, 2.84 s total -[ 2023-09-21 12:37:19 ] Completed train batch 0 generator forward 25.964 ms, 2.86 s total -[ 2023-09-21 12:37:19 ] Completed train batch 0 generator loss: 1.267 66.155 ms, 2.93 s total -[ 2023-09-21 12:37:20 ] Completed train batch 0 generator backward 1,055.717 ms, 3.98 s total -[ 2023-09-21 12:37:20 ] Completed train batch 0 discriminator loss 0.044 57.146 ms, 4.04 s total -[ 2023-09-21 12:37:20 ] Completed train batch 0 discriminator backward 107.724 ms, 4.15 s total -Epoch [195] Step [0/2], gen_loss: 0.421, disc_loss: 0.014 -[ 2023-09-21 12:37:20 ] Completed train batch 0 metrics update 8.808 ms, 4.16 s total -Saving checkpoint at epoch 195 train batch 0 -[ 2023-09-21 12:37:20 ] Completed saving temp checkpoint 431.353 ms, 4.59 s total -[ 2023-09-21 12:37:20 ] Completed replacing temp checkpoint with checkpoint 29.621 ms, 4.62 s total -[ 2023-09-21 12:37:21 ] Completed train batch 1 to device 1,183.230 ms, 5.80 s total -[ 2023-09-21 12:37:21 ] Completed train batch 1 generator forward 39.904 ms, 5.84 s total -[ 2023-09-21 12:37:22 ] Completed train batch 1 generator loss: 1.188 64.083 ms, 5.91 s total -[ 2023-09-21 12:37:23 ] Completed train batch 1 generator backward 1,649.275 ms, 7.56 s total -[ 2023-09-21 12:37:23 ] Completed train batch 1 discriminator loss 0.041 56.331 ms, 7.61 s total -[ 2023-09-21 12:37:23 ] Completed train batch 1 discriminator backward 72.054 ms, 7.68 s total -Epoch [195] Step [1/2], gen_loss: 0.406, disc_loss: 0.014 -[ 2023-09-21 12:37:23 ] Completed train batch 1 metrics update 2.764 ms, 7.69 s total -Saving checkpoint at epoch 195 train batch 1 -[ 2023-09-21 12:37:24 ] Completed saving temp checkpoint 465.934 ms, 8.15 s total -[ 2023-09-21 12:37:24 ] Completed replacing temp checkpoint with checkpoint 29.662 ms, 8.18 s total -Epoch [195] :: gen_loss: 0.414, disc_loss: 0.014 -[ 2023-09-21 12:37:24 ] Completed training generator for epoch 195 48.553 ms, 8.23 s total -[ 2023-09-21 12:37:24 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 195 from eval step 0 - -[ 2023-09-21 12:37:25 ] Completed eval batch 0 to device 1,129.987 ms, 1.13 s total -[ 2023-09-21 12:37:25 ] Completed eval batch 0 forward 24.808 ms, 1.15 s total -[ 2023-09-21 12:37:25 ] Completed eval batch 0 recons_loss 0.294 ms, 1.16 s total -[ 2023-09-21 12:37:25 ] Completed eval batch 0 metrics update 0.560 ms, 1.16 s total -Saving checkpoint at epoch 195 val batch 0 -[ 2023-09-21 12:37:25 ] Completed saving temp checkpoint 446.513 ms, 1.60 s total -[ 2023-09-21 12:37:26 ] Completed replacing temp checkpoint with checkpoint 28.658 ms, 1.63 s total -[ 2023-09-21 12:37:26 ] Completed eval batch 1 to device 387.610 ms, 2.02 s total -[ 2023-09-21 12:37:26 ] Completed eval batch 1 forward 39.514 ms, 2.06 s total -[ 2023-09-21 12:37:26 ] Completed eval batch 1 recons_loss 0.500 ms, 2.06 s total -[ 2023-09-21 12:37:26 ] Completed eval batch 1 metrics update 196.808 ms, 2.26 s total -Saving checkpoint at epoch 195 val batch 1 -Epoch 195 val loss: 0.0577 -[ 2023-09-21 12:37:27 ] Completed saving temp checkpoint 461.424 ms, 2.72 s total -[ 2023-09-21 12:37:27 ] Completed replacing temp checkpoint with checkpoint 26.860 ms, 2.74 s total -[ 2023-09-21 12:37:27 ] Completed evaluating generator for epoch 195 52.582 ms, 2.80 s total - - -EPOCH :: 196 - - -[ 2023-09-21 12:37:27 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 196 from training step 0 - -[ 2023-09-21 12:37:30 ] Completed train batch 0 to device 3,199.336 ms, 3.20 s total -[ 2023-09-21 12:37:30 ] Completed train batch 0 generator forward 36.014 ms, 3.24 s total -[ 2023-09-21 12:37:30 ] Completed train batch 0 generator loss: 1.224 67.519 ms, 3.30 s total -[ 2023-09-21 12:37:31 ] Completed train batch 0 generator backward 596.916 ms, 3.90 s total -[ 2023-09-21 12:37:31 ] Completed train batch 0 discriminator loss 0.045 57.664 ms, 3.96 s total -[ 2023-09-21 12:37:31 ] Completed train batch 0 discriminator backward 89.635 ms, 4.05 s total -Epoch [196] Step [0/2], gen_loss: 0.412, disc_loss: 0.014 -[ 2023-09-21 12:37:31 ] Completed train batch 0 metrics update 27.127 ms, 4.07 s total -Saving checkpoint at epoch 196 train batch 0 -[ 2023-09-21 12:37:31 ] Completed saving temp checkpoint 474.765 ms, 4.55 s total -[ 2023-09-21 12:37:31 ] Completed replacing temp checkpoint with checkpoint 24.831 ms, 4.57 s total -[ 2023-09-21 12:37:33 ] Completed train batch 1 to device 1,820.142 ms, 6.39 s total -[ 2023-09-21 12:37:33 ] Completed train batch 1 generator forward 41.231 ms, 6.44 s total -[ 2023-09-21 12:37:33 ] Completed train batch 1 generator loss: 1.299 64.701 ms, 6.50 s total -[ 2023-09-21 12:37:34 ] Completed train batch 1 generator backward 919.217 ms, 7.42 s total -[ 2023-09-21 12:37:34 ] Completed train batch 1 discriminator loss 0.044 56.335 ms, 7.48 s total -[ 2023-09-21 12:37:34 ] Completed train batch 1 discriminator backward 70.745 ms, 7.55 s total -Epoch [196] Step [1/2], gen_loss: 0.407, disc_loss: 0.014 -[ 2023-09-21 12:37:34 ] Completed train batch 1 metrics update 4.616 ms, 7.55 s total -Saving checkpoint at epoch 196 train batch 1 -[ 2023-09-21 12:37:35 ] Completed saving temp checkpoint 475.563 ms, 8.03 s total -[ 2023-09-21 12:37:35 ] Completed replacing temp checkpoint with checkpoint 24.597 ms, 8.05 s total -Epoch [196] :: gen_loss: 0.410, disc_loss: 0.014 -[ 2023-09-21 12:37:35 ] Completed training generator for epoch 196 55.430 ms, 8.11 s total -[ 2023-09-21 12:37:35 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 196 from eval step 0 - -[ 2023-09-21 12:37:36 ] Completed eval batch 0 to device 1,112.970 ms, 1.11 s total -[ 2023-09-21 12:37:36 ] Completed eval batch 0 forward 25.194 ms, 1.14 s total -[ 2023-09-21 12:37:36 ] Completed eval batch 0 recons_loss 0.294 ms, 1.14 s total -[ 2023-09-21 12:37:36 ] Completed eval batch 0 metrics update 0.626 ms, 1.14 s total -Saving checkpoint at epoch 196 val batch 0 -[ 2023-09-21 12:37:36 ] Completed saving temp checkpoint 460.657 ms, 1.60 s total -[ 2023-09-21 12:37:36 ] Completed replacing temp checkpoint with checkpoint 33.225 ms, 1.63 s total -[ 2023-09-21 12:37:37 ] Completed eval batch 1 to device 452.853 ms, 2.09 s total -[ 2023-09-21 12:37:37 ] Completed eval batch 1 forward 22.050 ms, 2.11 s total -[ 2023-09-21 12:37:37 ] Completed eval batch 1 recons_loss 0.354 ms, 2.11 s total -[ 2023-09-21 12:37:37 ] Completed eval batch 1 metrics update 247.851 ms, 2.36 s total -Saving checkpoint at epoch 196 val batch 1 -Epoch 196 val loss: 0.0568 -[ 2023-09-21 12:37:38 ] Completed saving temp checkpoint 508.984 ms, 2.87 s total -[ 2023-09-21 12:37:38 ] Completed replacing temp checkpoint with checkpoint 24.996 ms, 2.89 s total -[ 2023-09-21 12:37:38 ] Completed evaluating generator for epoch 196 52.235 ms, 2.94 s total - - -EPOCH :: 197 - - -[ 2023-09-21 12:37:38 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 197 from training step 0 - -[ 2023-09-21 12:37:41 ] Completed train batch 0 to device 3,177.233 ms, 3.18 s total -[ 2023-09-21 12:37:41 ] Completed train batch 0 generator forward 34.093 ms, 3.21 s total -[ 2023-09-21 12:37:41 ] Completed train batch 0 generator loss: 1.335 76.146 ms, 3.29 s total -[ 2023-09-21 12:37:41 ] Completed train batch 0 generator backward 452.451 ms, 3.74 s total -[ 2023-09-21 12:37:42 ] Completed train batch 0 discriminator loss 0.041 64.362 ms, 3.80 s total -[ 2023-09-21 12:37:42 ] Completed train batch 0 discriminator backward 92.707 ms, 3.90 s total -Epoch [197] Step [0/2], gen_loss: 0.418, disc_loss: 0.014 -[ 2023-09-21 12:37:42 ] Completed train batch 0 metrics update 3.511 ms, 3.90 s total -Saving checkpoint at epoch 197 train batch 0 -[ 2023-09-21 12:37:42 ] Completed saving temp checkpoint 488.428 ms, 4.39 s total -[ 2023-09-21 12:37:42 ] Completed replacing temp checkpoint with checkpoint 33.013 ms, 4.42 s total -[ 2023-09-21 12:37:44 ] Completed train batch 1 to device 1,877.048 ms, 6.30 s total -[ 2023-09-21 12:37:44 ] Completed train batch 1 generator forward 22.691 ms, 6.32 s total -[ 2023-09-21 12:37:44 ] Completed train batch 1 generator loss: 1.281 64.011 ms, 6.39 s total -[ 2023-09-21 12:37:45 ] Completed train batch 1 generator backward 966.175 ms, 7.35 s total -[ 2023-09-21 12:37:45 ] Completed train batch 1 discriminator loss 0.041 56.787 ms, 7.41 s total -[ 2023-09-21 12:37:45 ] Completed train batch 1 discriminator backward 75.567 ms, 7.48 s total -Epoch [197] Step [1/2], gen_loss: 0.411, disc_loss: 0.013 -[ 2023-09-21 12:37:45 ] Completed train batch 1 metrics update 3.615 ms, 7.49 s total -Saving checkpoint at epoch 197 train batch 1 -[ 2023-09-21 12:37:46 ] Completed saving temp checkpoint 476.774 ms, 7.96 s total -[ 2023-09-21 12:37:46 ] Completed replacing temp checkpoint with checkpoint 23.169 ms, 7.99 s total -Epoch [197] :: gen_loss: 0.415, disc_loss: 0.014 -[ 2023-09-21 12:37:46 ] Completed training generator for epoch 197 53.490 ms, 8.04 s total -[ 2023-09-21 12:37:46 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 197 from eval step 0 - -[ 2023-09-21 12:37:47 ] Completed eval batch 0 to device 1,179.252 ms, 1.18 s total -[ 2023-09-21 12:37:47 ] Completed eval batch 0 forward 24.216 ms, 1.20 s total -[ 2023-09-21 12:37:47 ] Completed eval batch 0 recons_loss 0.250 ms, 1.20 s total -[ 2023-09-21 12:37:47 ] Completed eval batch 0 metrics update 0.569 ms, 1.20 s total -Saving checkpoint at epoch 197 val batch 0 -[ 2023-09-21 12:37:47 ] Completed saving temp checkpoint 442.414 ms, 1.65 s total -[ 2023-09-21 12:37:47 ] Completed replacing temp checkpoint with checkpoint 29.570 ms, 1.68 s total -[ 2023-09-21 12:37:48 ] Completed eval batch 1 to device 542.401 ms, 2.22 s total -[ 2023-09-21 12:37:48 ] Completed eval batch 1 forward 22.293 ms, 2.24 s total -[ 2023-09-21 12:37:48 ] Completed eval batch 1 recons_loss 0.406 ms, 2.24 s total -[ 2023-09-21 12:37:48 ] Completed eval batch 1 metrics update 4.930 ms, 2.25 s total -Saving checkpoint at epoch 197 val batch 1 -Epoch 197 val loss: 0.0523 -[ 2023-09-21 12:37:49 ] Completed saving temp checkpoint 515.372 ms, 2.76 s total -[ 2023-09-21 12:37:49 ] Completed replacing temp checkpoint with checkpoint 25.815 ms, 2.79 s total -[ 2023-09-21 12:37:49 ] Completed evaluating generator for epoch 197 50.537 ms, 2.84 s total - - -EPOCH :: 198 - - -[ 2023-09-21 12:37:49 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 198 from training step 0 - -[ 2023-09-21 12:37:52 ] Completed train batch 0 to device 3,136.246 ms, 3.14 s total -[ 2023-09-21 12:37:52 ] Completed train batch 0 generator forward 27.287 ms, 3.16 s total -[ 2023-09-21 12:37:52 ] Completed train batch 0 generator loss: 1.288 66.925 ms, 3.23 s total -[ 2023-09-21 12:37:52 ] Completed train batch 0 generator backward 581.281 ms, 3.81 s total -[ 2023-09-21 12:37:52 ] Completed train batch 0 discriminator loss 0.042 57.231 ms, 3.87 s total -[ 2023-09-21 12:37:53 ] Completed train batch 0 discriminator backward 102.083 ms, 3.97 s total -Epoch [198] Step [0/2], gen_loss: 0.413, disc_loss: 0.013 -[ 2023-09-21 12:37:53 ] Completed train batch 0 metrics update 6.490 ms, 3.98 s total -Saving checkpoint at epoch 198 train batch 0 -[ 2023-09-21 12:37:53 ] Completed saving temp checkpoint 460.335 ms, 4.44 s total -[ 2023-09-21 12:37:53 ] Completed replacing temp checkpoint with checkpoint 27.580 ms, 4.47 s total -[ 2023-09-21 12:37:54 ] Completed train batch 1 to device 1,417.674 ms, 5.88 s total -[ 2023-09-21 12:37:55 ] Completed train batch 1 generator forward 33.211 ms, 5.92 s total -[ 2023-09-21 12:37:55 ] Completed train batch 1 generator loss: 1.257 65.584 ms, 5.98 s total -[ 2023-09-21 12:37:56 ] Completed train batch 1 generator backward 1,292.290 ms, 7.27 s total -[ 2023-09-21 12:37:56 ] Completed train batch 1 discriminator loss 0.038 56.155 ms, 7.33 s total -[ 2023-09-21 12:37:56 ] Completed train batch 1 discriminator backward 78.001 ms, 7.41 s total -Epoch [198] Step [1/2], gen_loss: 0.411, disc_loss: 0.013 -[ 2023-09-21 12:37:56 ] Completed train batch 1 metrics update 0.878 ms, 7.41 s total -Saving checkpoint at epoch 198 train batch 1 -[ 2023-09-21 12:37:57 ] Completed saving temp checkpoint 490.681 ms, 7.90 s total -[ 2023-09-21 12:37:57 ] Completed replacing temp checkpoint with checkpoint 27.750 ms, 7.93 s total -Epoch [198] :: gen_loss: 0.412, disc_loss: 0.013 -[ 2023-09-21 12:37:57 ] Completed training generator for epoch 198 49.332 ms, 7.98 s total -[ 2023-09-21 12:37:57 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 198 from eval step 0 - -[ 2023-09-21 12:37:58 ] Completed eval batch 0 to device 1,173.745 ms, 1.17 s total -[ 2023-09-21 12:37:58 ] Completed eval batch 0 forward 42.868 ms, 1.22 s total -[ 2023-09-21 12:37:58 ] Completed eval batch 0 recons_loss 0.382 ms, 1.22 s total -[ 2023-09-21 12:37:58 ] Completed eval batch 0 metrics update 0.835 ms, 1.22 s total -Saving checkpoint at epoch 198 val batch 0 -[ 2023-09-21 12:37:58 ] Completed saving temp checkpoint 418.189 ms, 1.64 s total -[ 2023-09-21 12:37:58 ] Completed replacing temp checkpoint with checkpoint 24.562 ms, 1.66 s total -[ 2023-09-21 12:37:59 ] Completed eval batch 1 to device 403.744 ms, 2.06 s total -[ 2023-09-21 12:37:59 ] Completed eval batch 1 forward 39.488 ms, 2.10 s total -[ 2023-09-21 12:37:59 ] Completed eval batch 1 recons_loss 0.564 ms, 2.10 s total -[ 2023-09-21 12:37:59 ] Completed eval batch 1 metrics update 117.865 ms, 2.22 s total -Saving checkpoint at epoch 198 val batch 1 -Epoch 198 val loss: 0.0553 -[ 2023-09-21 12:37:59 ] Completed saving temp checkpoint 415.808 ms, 2.64 s total -[ 2023-09-21 12:37:59 ] Completed replacing temp checkpoint with checkpoint 29.471 ms, 2.67 s total -[ 2023-09-21 12:37:59 ] Completed evaluating generator for epoch 198 48.223 ms, 2.72 s total - - -EPOCH :: 199 - - -[ 2023-09-21 12:37:59 ] Completed Start training 0.000 ms, 0.00 s total - -Training / resuming epoch 199 from training step 0 - -[ 2023-09-21 12:38:02 ] Completed train batch 0 to device 2,974.733 ms, 2.97 s total -[ 2023-09-21 12:38:02 ] Completed train batch 0 generator forward 43.949 ms, 3.02 s total -[ 2023-09-21 12:38:02 ] Completed train batch 0 generator loss: 1.228 66.572 ms, 3.09 s total -[ 2023-09-21 12:38:03 ] Completed train batch 0 generator backward 849.776 ms, 3.94 s total -[ 2023-09-21 12:38:03 ] Completed train batch 0 discriminator loss 0.041 58.974 ms, 3.99 s total -[ 2023-09-21 12:38:03 ] Completed train batch 0 discriminator backward 105.970 ms, 4.10 s total -Epoch [199] Step [0/2], gen_loss: 0.413, disc_loss: 0.013 -[ 2023-09-21 12:38:03 ] Completed train batch 0 metrics update 8.685 ms, 4.11 s total -Saving checkpoint at epoch 199 train batch 0 -[ 2023-09-21 12:38:04 ] Completed saving temp checkpoint 440.545 ms, 4.55 s total -[ 2023-09-21 12:38:04 ] Completed replacing temp checkpoint with checkpoint 28.927 ms, 4.58 s total -[ 2023-09-21 12:38:05 ] Completed train batch 1 to device 1,080.842 ms, 5.66 s total -[ 2023-09-21 12:38:05 ] Completed train batch 1 generator forward 41.359 ms, 5.70 s total -[ 2023-09-21 12:38:05 ] Completed train batch 1 generator loss: 1.261 62.661 ms, 5.76 s total -[ 2023-09-21 12:38:07 ] Completed train batch 1 generator backward 1,982.879 ms, 7.75 s total -[ 2023-09-21 12:38:07 ] Completed train batch 1 discriminator loss 0.040 56.241 ms, 7.80 s total -[ 2023-09-21 12:38:07 ] Completed train batch 1 discriminator backward 90.145 ms, 7.89 s total -Epoch [199] Step [1/2], gen_loss: 0.410, disc_loss: 0.013 -[ 2023-09-21 12:38:07 ] Completed train batch 1 metrics update 5.615 ms, 7.90 s total -Saving checkpoint at epoch 199 train batch 1 -[ 2023-09-21 12:38:08 ] Completed saving temp checkpoint 489.762 ms, 8.39 s total -[ 2023-09-21 12:38:08 ] Completed replacing temp checkpoint with checkpoint 22.297 ms, 8.41 s total -Epoch [199] :: gen_loss: 0.411, disc_loss: 0.013 -[ 2023-09-21 12:38:08 ] Completed training generator for epoch 199 57.412 ms, 8.47 s total -[ 2023-09-21 12:38:08 ] Completed Start evaluation 0.000 ms, 0.00 s total - -Evaluating / resuming epoch 199 from eval step 0 - -[ 2023-09-21 12:38:09 ] Completed eval batch 0 to device 1,121.779 ms, 1.12 s total -[ 2023-09-21 12:38:09 ] Completed eval batch 0 forward 25.215 ms, 1.15 s total -[ 2023-09-21 12:38:09 ] Completed eval batch 0 recons_loss 0.346 ms, 1.15 s total -[ 2023-09-21 12:38:09 ] Completed eval batch 0 metrics update 0.597 ms, 1.15 s total -Saving checkpoint at epoch 199 val batch 0 -[ 2023-09-21 12:38:09 ] Completed saving temp checkpoint 435.496 ms, 1.58 s total -[ 2023-09-21 12:38:09 ] Completed replacing temp checkpoint with checkpoint 32.362 ms, 1.62 s total -[ 2023-09-21 12:38:10 ] Completed eval batch 1 to device 398.376 ms, 2.01 s total -[ 2023-09-21 12:38:10 ] Completed eval batch 1 forward 40.298 ms, 2.05 s total -[ 2023-09-21 12:38:10 ] Completed eval batch 1 recons_loss 0.564 ms, 2.06 s total -[ 2023-09-21 12:38:10 ] Completed eval batch 1 metrics update 236.426 ms, 2.29 s total -Saving checkpoint at epoch 199 val batch 1 -Epoch 199 val loss: 0.0556 -[ 2023-09-21 12:38:10 ] Completed saving temp checkpoint 407.429 ms, 2.70 s total -[ 2023-09-21 12:38:10 ] Completed replacing temp checkpoint with checkpoint 25.343 ms, 2.72 s total -[ 2023-09-21 12:38:11 ] Completed evaluating generator for epoch 199 54.197 ms, 2.78 s total -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 2623573) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python -Traceback (most recent call last): - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in - sys.exit(main()) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper - return f(*args, **kwargs) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main - run(args) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run - elastic_launch( - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -======================================================== -train_cycling_gen.py FAILED --------------------------------------------------------- -Failures: -[1]: - time : 2023-09-21_12:38:12 - host : sc11 - rank : 1 (local_rank: 1) - exitcode : -6 (pid: 2623574) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2623574 -[2]: - time : 2023-09-21_12:38:12 - host : sc11 - rank : 2 (local_rank: 2) - exitcode : -6 (pid: 2623575) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2623575 -[3]: - time : 2023-09-21_12:38:12 - host : sc11 - rank : 3 (local_rank: 3) - exitcode : -6 (pid: 2623576) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2623576 -[4]: - time : 2023-09-21_12:38:12 - host : sc11 - rank : 4 (local_rank: 4) - exitcode : -6 (pid: 2623577) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2623577 -[5]: - time : 2023-09-21_12:38:12 - host : sc11 - rank : 5 (local_rank: 5) - exitcode : -6 (pid: 2623578) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2623578 --------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2023-09-21_12:38:12 - host : sc11 - rank : 0 (local_rank: 0) - exitcode : -6 (pid: 2623573) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2623573 -======================================================== diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_1.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_1.txt deleted file mode 100644 index b3cf250b..00000000 --- a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_1.txt +++ /dev/null @@ -1,1310 +0,0 @@ -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 11): env:// -| distributed init (rank 9): env:// -| distributed init (rank 10): env:// -| distributed init (rank 6): env:// -| distributed init (rank 7): env:// -| distributed init (rank 8): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 11): env:// -| distributed init (rank 6): env:// -| distributed init (rank 8): env:// -| distributed init (rank 9): env:// -| distributed init (rank 10): env:// -| distributed init (rank 7): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 9): env:// -| distributed init (rank 7): env:// -| distributed init (rank 11): env:// -| distributed init (rank 6): env:// -| distributed init (rank 10): env:// -| distributed init (rank 8): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 11): env:// -| distributed init (rank 7): env:// -| distributed init (rank 6): env:// -| distributed init (rank 10): env:// -| distributed init (rank 8): env:// -| distributed init (rank 9): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 11): env:// -| distributed init (rank 7): env:// -| distributed init (rank 9): env:// -| distributed init (rank 6): env:// -| distributed init (rank 10): env:// -| distributed init (rank 8): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 10): env:// -| distributed init (rank 9): env:// -| distributed init (rank 7): env:// -| distributed init (rank 6): env:// -| distributed init (rank 8): env:// -| distributed init (rank 11): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 7): env:// -| distributed init (rank 6): env:// -| distributed init (rank 8): env:// -| distributed init (rank 11): env:// -| distributed init (rank 9): env:// -| distributed init (rank 10): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 6): env:// -| distributed init (rank 7): env:// -| distributed init (rank 8): env:// -| distributed init (rank 11): env:// -| distributed init (rank 9): env:// -| distributed init (rank 10): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 11): env:// -| distributed init (rank 10): env:// -| distributed init (rank 9): env:// -| distributed init (rank 7): env:// -| distributed init (rank 6): env:// -| distributed init (rank 8): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 9): env:// -| distributed init (rank 8): env:// -| distributed init (rank 7): env:// -| distributed init (rank 10): env:// -| distributed init (rank 11): env:// -| distributed init (rank 6): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 11): env:// -| distributed init (rank 9): env:// -| distributed init (rank 7): env:// -| distributed init (rank 10): env:// -| distributed init (rank 6): env:// -| distributed init (rank 8): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 10): env:// -| distributed init (rank 9): env:// -| distributed init (rank 11): env:// -| distributed init (rank 8): env:// -| distributed init (rank 6): env:// -| distributed init (rank 7): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 7): env:// -| distributed init (rank 11): env:// -| distributed init (rank 9): env:// -| distributed init (rank 6): env:// -| distributed init (rank 8): env:// -| distributed init (rank 10): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 9): env:// -| distributed init (rank 6): env:// -| distributed init (rank 11): env:// -| distributed init (rank 7): env:// -| distributed init (rank 8): env:// -| distributed init (rank 10): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 7): env:// -| distributed init (rank 11): env:// -| distributed init (rank 9): env:// -| distributed init (rank 6): env:// -| distributed init (rank 8): env:// -| distributed init (rank 10): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 9): env:// -| distributed init (rank 8): env:// -| distributed init (rank 6): env:// -| distributed init (rank 7): env:// -| distributed init (rank 11): env:// -| distributed init (rank 10): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 7): env:// -| distributed init (rank 8): env:// -| distributed init (rank 9): env:// -| distributed init (rank 10): env:// -| distributed init (rank 6): env:// -| distributed init (rank 11): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 10): env:// -| distributed init (rank 11): env:// -| distributed init (rank 8): env:// -| distributed init (rank 7): env:// -| distributed init (rank 6): env:// -| distributed init (rank 9): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 11): env:// -| distributed init (rank 7): env:// -| distributed init (rank 6): env:// -| distributed init (rank 9): env:// -| distributed init (rank 10): env:// -| distributed init (rank 8): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 11): env:// -| distributed init (rank 6): env:// -| distributed init (rank 8): env:// -| distributed init (rank 7): env:// -| distributed init (rank 9): env:// -| distributed init (rank 10): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 9): env:// -| distributed init (rank 7): env:// -| distributed init (rank 6): env:// -| distributed init (rank 11): env:// -| distributed init (rank 10): env:// -| distributed init (rank 8): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 8): env:// -| distributed init (rank 7): env:// -| distributed init (rank 11): env:// -| distributed init (rank 10): env:// -| distributed init (rank 9): env:// -| distributed init (rank 6): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 6): env:// -| distributed init (rank 11): env:// -| distributed init (rank 7): env:// -| distributed init (rank 9): env:// -| distributed init (rank 8): env:// -| distributed init (rank 10): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 6): env:// -| distributed init (rank 7): env:// -| distributed init (rank 11): env:// -| distributed init (rank 8): env:// -| distributed init (rank 9): env:// -| distributed init (rank 10): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 9): env:// -| distributed init (rank 8): env:// -| distributed init (rank 7): env:// -| distributed init (rank 11): env:// -| distributed init (rank 10): env:// -| distributed init (rank 6): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 6): env:// -| distributed init (rank 7): env:// -| distributed init (rank 10): env:// -| distributed init (rank 9): env:// -| distributed init (rank 11): env:// -| distributed init (rank 8): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 6): env:// -| distributed init (rank 11): env:// -| distributed init (rank 7): env:// -| distributed init (rank 9): env:// -| distributed init (rank 8): env:// -| distributed init (rank 10): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 11): env:// -| distributed init (rank 9): env:// -| distributed init (rank 6): env:// -| distributed init (rank 7): env:// -| distributed init (rank 10): env:// -| distributed init (rank 8): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 7): env:// -| distributed init (rank 9): env:// -| distributed init (rank 6): env:// -| distributed init (rank 11): env:// -| distributed init (rank 8): env:// -| distributed init (rank 10): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 11): env:// -| distributed init (rank 6): env:// -| distributed init (rank 8): env:// -| distributed init (rank 7): env:// -| distributed init (rank 10): env:// -| distributed init (rank 9): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 7): env:// -| distributed init (rank 8): env:// -| distributed init (rank 6): env:// -| distributed init (rank 9): env:// -| distributed init (rank 11): env:// -| distributed init (rank 10): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 2789333) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python -Traceback (most recent call last): - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in - sys.exit(main()) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper - return f(*args, **kwargs) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main - run(args) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run - elastic_launch( - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -======================================================== -train_cycling_gen.py FAILED --------------------------------------------------------- -Failures: -[1]: - time : 2023-09-21_12:38:12 - host : sc14 - rank : 7 (local_rank: 1) - exitcode : -6 (pid: 2789334) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2789334 -[2]: - time : 2023-09-21_12:38:12 - host : sc14 - rank : 8 (local_rank: 2) - exitcode : -6 (pid: 2789335) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2789335 -[3]: - time : 2023-09-21_12:38:12 - host : sc14 - rank : 9 (local_rank: 3) - exitcode : -6 (pid: 2789336) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2789336 -[4]: - time : 2023-09-21_12:38:12 - host : sc14 - rank : 10 (local_rank: 4) - exitcode : -6 (pid: 2789337) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2789337 -[5]: - time : 2023-09-21_12:38:12 - host : sc14 - rank : 11 (local_rank: 5) - exitcode : -6 (pid: 2789338) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2789338 --------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2023-09-21_12:38:12 - host : sc14 - rank : 6 (local_rank: 0) - exitcode : -6 (pid: 2789333) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2789333 -======================================================== diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_10.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_10.txt deleted file mode 100644 index 23b06cbd..00000000 --- a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_10.txt +++ /dev/null @@ -1,1310 +0,0 @@ -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 63): env:// -| distributed init (rank 64): env:// -| distributed init (rank 62): env:// -| distributed init (rank 65): env:// -| distributed init (rank 60): env:// -| distributed init (rank 61): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 63): env:// -| distributed init (rank 65): env:// -| distributed init (rank 60): env:// -| distributed init (rank 62): env:// -| distributed init (rank 61): env:// -| distributed init (rank 64): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 62): env:// -| distributed init (rank 64): env:// -| distributed init (rank 63): env:// -| distributed init (rank 65): env:// -| distributed init (rank 60): env:// -| distributed init (rank 61): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 63): env:// -| distributed init (rank 64): env:// -| distributed init (rank 65): env:// -| distributed init (rank 60): env:// -| distributed init (rank 61): env:// -| distributed init (rank 62): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 63): env:// -| distributed init (rank 64): env:// -| distributed init (rank 65): env:// -| distributed init (rank 60): env:// -| distributed init (rank 62): env:// -| distributed init (rank 61): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 61): env:// -| distributed init (rank 60): env:// -| distributed init (rank 64): env:// -| distributed init (rank 63): env:// -| distributed init (rank 62): env:// -| distributed init (rank 65): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 63): env:// -| distributed init (rank 65): env:// -| distributed init (rank 62): env:// -| distributed init (rank 64): env:// -| distributed init (rank 60): env:// -| distributed init (rank 61): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 61): env:// -| distributed init (rank 63): env:// -| distributed init (rank 62): env:// -| distributed init (rank 64): env:// -| distributed init (rank 60): env:// -| distributed init (rank 65): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 61): env:// -| distributed init (rank 62): env:// -| distributed init (rank 65): env:// -| distributed init (rank 64): env:// -| distributed init (rank 63): env:// -| distributed init (rank 60): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 63): env:// -| distributed init (rank 64): env:// -| distributed init (rank 62): env:// -| distributed init (rank 65): env:// -| distributed init (rank 61): env:// -| distributed init (rank 60): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 62): env:// -| distributed init (rank 61): env:// -| distributed init (rank 63): env:// -| distributed init (rank 65): env:// -| distributed init (rank 60): env:// -| distributed init (rank 64): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 62): env:// -| distributed init (rank 64): env:// -| distributed init (rank 61): env:// -| distributed init (rank 63): env:// -| distributed init (rank 60): env:// -| distributed init (rank 65): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 60): env:// -| distributed init (rank 64): env:// -| distributed init (rank 65): env:// -| distributed init (rank 61): env:// -| distributed init (rank 62): env:// -| distributed init (rank 63): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 65): env:// -| distributed init (rank 60): env:// -| distributed init (rank 62): env:// -| distributed init (rank 64): env:// -| distributed init (rank 63): env:// -| distributed init (rank 61): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 60): env:// -| distributed init (rank 65): env:// -| distributed init (rank 64): env:// -| distributed init (rank 61): env:// -| distributed init (rank 62): env:// -| distributed init (rank 63): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 65): env:// -| distributed init (rank 61): env:// -| distributed init (rank 62): env:// -| distributed init (rank 63): env:// -| distributed init (rank 60): env:// -| distributed init (rank 64): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 63): env:// -| distributed init (rank 64): env:// -| distributed init (rank 61): env:// -| distributed init (rank 65): env:// -| distributed init (rank 60): env:// -| distributed init (rank 62): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 65): env:// -| distributed init (rank 64): env:// -| distributed init (rank 63): env:// -| distributed init (rank 60): env:// -| distributed init (rank 62): env:// -| distributed init (rank 61): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 61): env:// -| distributed init (rank 62): env:// -| distributed init (rank 63): env:// -| distributed init (rank 60): env:// -| distributed init (rank 64): env:// -| distributed init (rank 65): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 61): env:// -| distributed init (rank 64): env:// -| distributed init (rank 62): env:// -| distributed init (rank 63): env:// -| distributed init (rank 65): env:// -| distributed init (rank 60): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 63): env:// -| distributed init (rank 64): env:// -| distributed init (rank 62): env:// -| distributed init (rank 61): env:// -| distributed init (rank 65): env:// -| distributed init (rank 60): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 61): env:// -| distributed init (rank 65): env:// -| distributed init (rank 62): env:// -| distributed init (rank 60): env:// -| distributed init (rank 64): env:// -| distributed init (rank 63): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 62): env:// -| distributed init (rank 60): env:// -| distributed init (rank 64): env:// -| distributed init (rank 65): env:// -| distributed init (rank 63): env:// -| distributed init (rank 61): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 62): env:// -| distributed init (rank 60): env:// -| distributed init (rank 65): env:// -| distributed init (rank 61): env:// -| distributed init (rank 64): env:// -| distributed init (rank 63): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 63): env:// -| distributed init (rank 61): env:// -| distributed init (rank 60): env:// -| distributed init (rank 64): env:// -| distributed init (rank 65): env:// -| distributed init (rank 62): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 60): env:// -| distributed init (rank 64): env:// -| distributed init (rank 62): env:// -| distributed init (rank 63): env:// -| distributed init (rank 65): env:// -| distributed init (rank 61): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 64): env:// -| distributed init (rank 60): env:// -| distributed init (rank 65): env:// -| distributed init (rank 63): env:// -| distributed init (rank 61): env:// -| distributed init (rank 62): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 60): env:// -| distributed init (rank 62): env:// -| distributed init (rank 61): env:// -| distributed init (rank 64): env:// -| distributed init (rank 63): env:// -| distributed init (rank 65): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 63): env:// -| distributed init (rank 62): env:// -| distributed init (rank 64): env:// -| distributed init (rank 65): env:// -| distributed init (rank 60): env:// -| distributed init (rank 61): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 65): env:// -| distributed init (rank 62): env:// -| distributed init (rank 63): env:// -| distributed init (rank 64): env:// -| distributed init (rank 61): env:// -| distributed init (rank 60): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 64): env:// -| distributed init (rank 60): env:// -| distributed init (rank 65): env:// -| distributed init (rank 62): env:// -| distributed init (rank 61): env:// -| distributed init (rank 63): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 1663528) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python -Traceback (most recent call last): - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in - sys.exit(main()) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper - return f(*args, **kwargs) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main - run(args) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run - elastic_launch( - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -======================================================== -train_cycling_gen.py FAILED --------------------------------------------------------- -Failures: -[1]: - time : 2023-09-21_12:38:12 - host : sc20 - rank : 61 (local_rank: 1) - exitcode : -6 (pid: 1663529) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 1663529 -[2]: - time : 2023-09-21_12:38:12 - host : sc20 - rank : 62 (local_rank: 2) - exitcode : -6 (pid: 1663530) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 1663530 -[3]: - time : 2023-09-21_12:38:12 - host : sc20 - rank : 63 (local_rank: 3) - exitcode : -6 (pid: 1663531) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 1663531 -[4]: - time : 2023-09-21_12:38:12 - host : sc20 - rank : 64 (local_rank: 4) - exitcode : -6 (pid: 1663532) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 1663532 -[5]: - time : 2023-09-21_12:38:12 - host : sc20 - rank : 65 (local_rank: 5) - exitcode : -6 (pid: 1663533) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 1663533 --------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2023-09-21_12:38:12 - host : sc20 - rank : 60 (local_rank: 0) - exitcode : -6 (pid: 1663528) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 1663528 -======================================================== diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_2.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_2.txt deleted file mode 100644 index fa722cbb..00000000 --- a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_2.txt +++ /dev/null @@ -1,1310 +0,0 @@ -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 14): env:// -| distributed init (rank 12): env:// -| distributed init (rank 15): env:// -| distributed init (rank 17): env:// -| distributed init (rank 13): env:// -| distributed init (rank 16): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 17): env:// -| distributed init (rank 12): env:// -| distributed init (rank 13): env:// -| distributed init (rank 14): env:// -| distributed init (rank 15): env:// -| distributed init (rank 16): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 16): env:// -| distributed init (rank 15): env:// -| distributed init (rank 12): env:// -| distributed init (rank 14): env:// -| distributed init (rank 17): env:// -| distributed init (rank 13): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 16): env:// -| distributed init (rank 15): env:// -| distributed init (rank 17): env:// -| distributed init (rank 12): env:// -| distributed init (rank 14): env:// -| distributed init (rank 13): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 12): env:// -| distributed init (rank 16): env:// -| distributed init (rank 17): env:// -| distributed init (rank 13): env:// -| distributed init (rank 15): env:// -| distributed init (rank 14): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 12): env:// -| distributed init (rank 13): env:// -| distributed init (rank 16): env:// -| distributed init (rank 14): env:// -| distributed init (rank 17): env:// -| distributed init (rank 15): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 17): env:// -| distributed init (rank 15): env:// -| distributed init (rank 14): env:// -| distributed init (rank 16): env:// -| distributed init (rank 12): env:// -| distributed init (rank 13): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 13): env:// -| distributed init (rank 16): env:// -| distributed init (rank 12): env:// -| distributed init (rank 14): env:// -| distributed init (rank 15): env:// -| distributed init (rank 17): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 14): env:// -| distributed init (rank 13): env:// -| distributed init (rank 16): env:// -| distributed init (rank 15): env:// -| distributed init (rank 12): env:// -| distributed init (rank 17): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 17): env:// -| distributed init (rank 14): env:// -| distributed init (rank 15): env:// -| distributed init (rank 12): env:// -| distributed init (rank 16): env:// -| distributed init (rank 13): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 14): env:// -| distributed init (rank 15): env:// -| distributed init (rank 16): env:// -| distributed init (rank 17): env:// -| distributed init (rank 12): env:// -| distributed init (rank 13): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 15): env:// -| distributed init (rank 13): env:// -| distributed init (rank 16): env:// -| distributed init (rank 12): env:// -| distributed init (rank 14): env:// -| distributed init (rank 17): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 12): env:// -| distributed init (rank 15): env:// -| distributed init (rank 16): env:// -| distributed init (rank 17): env:// -| distributed init (rank 13): env:// -| distributed init (rank 14): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 17): env:// -| distributed init (rank 15): env:// -| distributed init (rank 13): env:// -| distributed init (rank 16): env:// -| distributed init (rank 12): env:// -| distributed init (rank 14): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 17): env:// -| distributed init (rank 13): env:// -| distributed init (rank 12): env:// -| distributed init (rank 14): env:// -| distributed init (rank 15): env:// -| distributed init (rank 16): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 14): env:// -| distributed init (rank 16): env:// -| distributed init (rank 12): env:// -| distributed init (rank 15): env:// -| distributed init (rank 13): env:// -| distributed init (rank 17): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 17): env:// -| distributed init (rank 16): env:// -| distributed init (rank 13): env:// -| distributed init (rank 12): env:// -| distributed init (rank 15): env:// -| distributed init (rank 14): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 17): env:// -| distributed init (rank 14): env:// -| distributed init (rank 15): env:// -| distributed init (rank 13): env:// -| distributed init (rank 16): env:// -| distributed init (rank 12): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 12): env:// -| distributed init (rank 16): env:// -| distributed init (rank 17): env:// -| distributed init (rank 15): env:// -| distributed init (rank 13): env:// -| distributed init (rank 14): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 16): env:// -| distributed init (rank 14): env:// -| distributed init (rank 15): env:// -| distributed init (rank 13): env:// -| distributed init (rank 12): env:// -| distributed init (rank 17): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 12): env:// -| distributed init (rank 15): env:// -| distributed init (rank 14): env:// -| distributed init (rank 16): env:// -| distributed init (rank 13): env:// -| distributed init (rank 17): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 12): env:// -| distributed init (rank 16): env:// -| distributed init (rank 17): env:// -| distributed init (rank 14): env:// -| distributed init (rank 13): env:// -| distributed init (rank 15): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 16): env:// -| distributed init (rank 17): env:// -| distributed init (rank 12): env:// -| distributed init (rank 14): env:// -| distributed init (rank 15): env:// -| distributed init (rank 13): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 13): env:// -| distributed init (rank 16): env:// -| distributed init (rank 15): env:// -| distributed init (rank 12): env:// -| distributed init (rank 14): env:// -| distributed init (rank 17): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 17): env:// -| distributed init (rank 16): env:// -| distributed init (rank 12): env:// -| distributed init (rank 15): env:// -| distributed init (rank 13): env:// -| distributed init (rank 14): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 15): env:// -| distributed init (rank 14): env:// -| distributed init (rank 17): env:// -| distributed init (rank 16): env:// -| distributed init (rank 12): env:// -| distributed init (rank 13): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 12): env:// -| distributed init (rank 16): env:// -| distributed init (rank 14): env:// -| distributed init (rank 13): env:// -| distributed init (rank 17): env:// -| distributed init (rank 15): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 15): env:// -| distributed init (rank 14): env:// -| distributed init (rank 12): env:// -| distributed init (rank 16): env:// -| distributed init (rank 13): env:// -| distributed init (rank 17): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 15): env:// -| distributed init (rank 12): env:// -| distributed init (rank 14): env:// -| distributed init (rank 13): env:// -| distributed init (rank 16): env:// -| distributed init (rank 17): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 16): env:// -| distributed init (rank 13): env:// -| distributed init (rank 14): env:// -| distributed init (rank 15): env:// -| distributed init (rank 12): env:// -| distributed init (rank 17): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 13): env:// -| distributed init (rank 15): env:// -| distributed init (rank 14): env:// -| distributed init (rank 16): env:// -| distributed init (rank 17): env:// -| distributed init (rank 12): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 598797) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python -Traceback (most recent call last): - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in - sys.exit(main()) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper - return f(*args, **kwargs) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main - run(args) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run - elastic_launch( - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -======================================================= -train_cycling_gen.py FAILED -------------------------------------------------------- -Failures: -[1]: - time : 2023-09-21_12:38:12 - host : sc12 - rank : 13 (local_rank: 1) - exitcode : -6 (pid: 598798) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 598798 -[2]: - time : 2023-09-21_12:38:12 - host : sc12 - rank : 14 (local_rank: 2) - exitcode : -6 (pid: 598799) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 598799 -[3]: - time : 2023-09-21_12:38:12 - host : sc12 - rank : 15 (local_rank: 3) - exitcode : -6 (pid: 598800) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 598800 -[4]: - time : 2023-09-21_12:38:12 - host : sc12 - rank : 16 (local_rank: 4) - exitcode : -6 (pid: 598801) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 598801 -[5]: - time : 2023-09-21_12:38:12 - host : sc12 - rank : 17 (local_rank: 5) - exitcode : -6 (pid: 598802) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 598802 -------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2023-09-21_12:38:12 - host : sc12 - rank : 12 (local_rank: 0) - exitcode : -6 (pid: 598797) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 598797 -======================================================= diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_3.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_3.txt deleted file mode 100644 index 90e93581..00000000 --- a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_3.txt +++ /dev/null @@ -1,1310 +0,0 @@ -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 20): env:// -| distributed init (rank 22): env:// -| distributed init (rank 18): env:// -| distributed init (rank 23): env:// -| distributed init (rank 21): env:// -| distributed init (rank 19): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 18): env:// -| distributed init (rank 19): env:// -| distributed init (rank 20): env:// -| distributed init (rank 21): env:// -| distributed init (rank 23): env:// -| distributed init (rank 22): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 18): env:// -| distributed init (rank 23): env:// -| distributed init (rank 20): env:// -| distributed init (rank 22): env:// -| distributed init (rank 19): env:// -| distributed init (rank 21): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 23): env:// -| distributed init (rank 20): env:// -| distributed init (rank 22): env:// -| distributed init (rank 21): env:// -| distributed init (rank 18): env:// -| distributed init (rank 19): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 20): env:// -| distributed init (rank 18): env:// -| distributed init (rank 21): env:// -| distributed init (rank 22): env:// -| distributed init (rank 19): env:// -| distributed init (rank 23): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 19): env:// -| distributed init (rank 21): env:// -| distributed init (rank 18): env:// -| distributed init (rank 22): env:// -| distributed init (rank 23): env:// -| distributed init (rank 20): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 20): env:// -| distributed init (rank 19): env:// -| distributed init (rank 21): env:// -| distributed init (rank 18): env:// -| distributed init (rank 23): env:// -| distributed init (rank 22): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 19): env:// -| distributed init (rank 21): env:// -| distributed init (rank 18): env:// -| distributed init (rank 22): env:// -| distributed init (rank 23): env:// -| distributed init (rank 20): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 21): env:// -| distributed init (rank 23): env:// -| distributed init (rank 20): env:// -| distributed init (rank 22): env:// -| distributed init (rank 19): env:// -| distributed init (rank 18): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 19): env:// -| distributed init (rank 22): env:// -| distributed init (rank 20): env:// -| distributed init (rank 18): env:// -| distributed init (rank 23): env:// -| distributed init (rank 21): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 23): env:// -| distributed init (rank 19): env:// -| distributed init (rank 21): env:// -| distributed init (rank 22): env:// -| distributed init (rank 20): env:// -| distributed init (rank 18): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 22): env:// -| distributed init (rank 18): env:// -| distributed init (rank 19): env:// -| distributed init (rank 20): env:// -| distributed init (rank 21): env:// -| distributed init (rank 23): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 20): env:// -| distributed init (rank 21): env:// -| distributed init (rank 19): env:// -| distributed init (rank 22): env:// -| distributed init (rank 18): env:// -| distributed init (rank 23): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 18): env:// -| distributed init (rank 23): env:// -| distributed init (rank 19): env:// -| distributed init (rank 20): env:// -| distributed init (rank 22): env:// -| distributed init (rank 21): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 21): env:// -| distributed init (rank 18): env:// -| distributed init (rank 19): env:// -| distributed init (rank 23): env:// -| distributed init (rank 22): env:// -| distributed init (rank 20): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 18): env:// -| distributed init (rank 19): env:// -| distributed init (rank 21): env:// -| distributed init (rank 23): env:// -| distributed init (rank 20): env:// -| distributed init (rank 22): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 19): env:// -| distributed init (rank 23): env:// -| distributed init (rank 22): env:// -| distributed init (rank 21): env:// -| distributed init (rank 18): env:// -| distributed init (rank 20): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 21): env:// -| distributed init (rank 18): env:// -| distributed init (rank 19): env:// -| distributed init (rank 23): env:// -| distributed init (rank 22): env:// -| distributed init (rank 20): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 20): env:// -| distributed init (rank 22): env:// -| distributed init (rank 18): env:// -| distributed init (rank 21): env:// -| distributed init (rank 23): env:// -| distributed init (rank 19): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 22): env:// -| distributed init (rank 21): env:// -| distributed init (rank 23): env:// -| distributed init (rank 18): env:// -| distributed init (rank 20): env:// -| distributed init (rank 19): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 19): env:// -| distributed init (rank 20): env:// -| distributed init (rank 18): env:// -| distributed init (rank 22): env:// -| distributed init (rank 23): env:// -| distributed init (rank 21): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 18): env:// -| distributed init (rank 23): env:// -| distributed init (rank 22): env:// -| distributed init (rank 19): env:// -| distributed init (rank 20): env:// -| distributed init (rank 21): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 18): env:// -| distributed init (rank 20): env:// -| distributed init (rank 22): env:// -| distributed init (rank 21): env:// -| distributed init (rank 23): env:// -| distributed init (rank 19): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 22): env:// -| distributed init (rank 21): env:// -| distributed init (rank 18): env:// -| distributed init (rank 23): env:// -| distributed init (rank 19): env:// -| distributed init (rank 20): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 18): env:// -| distributed init (rank 23): env:// -| distributed init (rank 22): env:// -| distributed init (rank 20): env:// -| distributed init (rank 21): env:// -| distributed init (rank 19): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 20): env:// -| distributed init (rank 23): env:// -| distributed init (rank 22): env:// -| distributed init (rank 19): env:// -| distributed init (rank 21): env:// -| distributed init (rank 18): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 21): env:// -| distributed init (rank 22): env:// -| distributed init (rank 23): env:// -| distributed init (rank 20): env:// -| distributed init (rank 18): env:// -| distributed init (rank 19): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 22): env:// -| distributed init (rank 19): env:// -| distributed init (rank 21): env:// -| distributed init (rank 18): env:// -| distributed init (rank 23): env:// -| distributed init (rank 20): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 19): env:// -| distributed init (rank 21): env:// -| distributed init (rank 23): env:// -| distributed init (rank 18): env:// -| distributed init (rank 22): env:// -| distributed init (rank 20): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 20): env:// -| distributed init (rank 18): env:// -| distributed init (rank 21): env:// -| distributed init (rank 19): env:// -| distributed init (rank 22): env:// -| distributed init (rank 23): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 21): env:// -| distributed init (rank 19): env:// -| distributed init (rank 22): env:// -| distributed init (rank 20): env:// -| distributed init (rank 18): env:// -| distributed init (rank 23): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 436434) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python -Traceback (most recent call last): - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in - sys.exit(main()) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper - return f(*args, **kwargs) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main - run(args) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run - elastic_launch( - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -======================================================= -train_cycling_gen.py FAILED -------------------------------------------------------- -Failures: -[1]: - time : 2023-09-21_12:38:12 - host : sc18.strongc.ai - rank : 19 (local_rank: 1) - exitcode : -6 (pid: 436435) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 436435 -[2]: - time : 2023-09-21_12:38:12 - host : sc18.strongc.ai - rank : 20 (local_rank: 2) - exitcode : -6 (pid: 436436) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 436436 -[3]: - time : 2023-09-21_12:38:12 - host : sc18.strongc.ai - rank : 21 (local_rank: 3) - exitcode : -6 (pid: 436437) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 436437 -[4]: - time : 2023-09-21_12:38:12 - host : sc18.strongc.ai - rank : 22 (local_rank: 4) - exitcode : -6 (pid: 436438) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 436438 -[5]: - time : 2023-09-21_12:38:12 - host : sc18.strongc.ai - rank : 23 (local_rank: 5) - exitcode : -6 (pid: 436439) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 436439 -------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2023-09-21_12:38:12 - host : sc18.strongc.ai - rank : 18 (local_rank: 0) - exitcode : -6 (pid: 436434) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 436434 -======================================================= diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_4.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_4.txt deleted file mode 100644 index a4626165..00000000 --- a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_4.txt +++ /dev/null @@ -1,1310 +0,0 @@ -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 29): env:// -| distributed init (rank 25): env:// -| distributed init (rank 26): env:// -| distributed init (rank 24): env:// -| distributed init (rank 28): env:// -| distributed init (rank 27): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 28): env:// -| distributed init (rank 26): env:// -| distributed init (rank 24): env:// -| distributed init (rank 29): env:// -| distributed init (rank 27): env:// -| distributed init (rank 25): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 28): env:// -| distributed init (rank 29): env:// -| distributed init (rank 26): env:// -| distributed init (rank 27): env:// -| distributed init (rank 24): env:// -| distributed init (rank 25): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 26): env:// -| distributed init (rank 27): env:// -| distributed init (rank 28): env:// -| distributed init (rank 29): env:// -| distributed init (rank 24): env:// -| distributed init (rank 25): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 29): env:// -| distributed init (rank 24): env:// -| distributed init (rank 28): env:// -| distributed init (rank 26): env:// -| distributed init (rank 25): env:// -| distributed init (rank 27): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 25): env:// -| distributed init (rank 27): env:// -| distributed init (rank 24): env:// -| distributed init (rank 29): env:// -| distributed init (rank 26): env:// -| distributed init (rank 28): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 26): env:// -| distributed init (rank 27): env:// -| distributed init (rank 24): env:// -| distributed init (rank 29): env:// -| distributed init (rank 28): env:// -| distributed init (rank 25): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 28): env:// -| distributed init (rank 26): env:// -| distributed init (rank 27): env:// -| distributed init (rank 25): env:// -| distributed init (rank 29): env:// -| distributed init (rank 24): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 28): env:// -| distributed init (rank 24): env:// -| distributed init (rank 26): env:// -| distributed init (rank 27): env:// -| distributed init (rank 29): env:// -| distributed init (rank 25): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 26): env:// -| distributed init (rank 29): env:// -| distributed init (rank 28): env:// -| distributed init (rank 25): env:// -| distributed init (rank 27): env:// -| distributed init (rank 24): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 27): env:// -| distributed init (rank 25): env:// -| distributed init (rank 26): env:// -| distributed init (rank 28): env:// -| distributed init (rank 24): env:// -| distributed init (rank 29): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 24): env:// -| distributed init (rank 25): env:// -| distributed init (rank 29): env:// -| distributed init (rank 27): env:// -| distributed init (rank 26): env:// -| distributed init (rank 28): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 24): env:// -| distributed init (rank 26): env:// -| distributed init (rank 28): env:// -| distributed init (rank 29): env:// -| distributed init (rank 27): env:// -| distributed init (rank 25): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 29): env:// -| distributed init (rank 28): env:// -| distributed init (rank 27): env:// -| distributed init (rank 25): env:// -| distributed init (rank 26): env:// -| distributed init (rank 24): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 26): env:// -| distributed init (rank 25): env:// -| distributed init (rank 28): env:// -| distributed init (rank 24): env:// -| distributed init (rank 27): env:// -| distributed init (rank 29): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 24): env:// -| distributed init (rank 26): env:// -| distributed init (rank 25): env:// -| distributed init (rank 28): env:// -| distributed init (rank 29): env:// -| distributed init (rank 27): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 28): env:// -| distributed init (rank 24): env:// -| distributed init (rank 25): env:// -| distributed init (rank 29): env:// -| distributed init (rank 26): env:// -| distributed init (rank 27): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 29): env:// -| distributed init (rank 24): env:// -| distributed init (rank 27): env:// -| distributed init (rank 26): env:// -| distributed init (rank 25): env:// -| distributed init (rank 28): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 25): env:// -| distributed init (rank 28): env:// -| distributed init (rank 27): env:// -| distributed init (rank 24): env:// -| distributed init (rank 29): env:// -| distributed init (rank 26): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 25): env:// -| distributed init (rank 27): env:// -| distributed init (rank 26): env:// -| distributed init (rank 24): env:// -| distributed init (rank 28): env:// -| distributed init (rank 29): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 24): env:// -| distributed init (rank 25): env:// -| distributed init (rank 29): env:// -| distributed init (rank 27): env:// -| distributed init (rank 26): env:// -| distributed init (rank 28): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 29): env:// -| distributed init (rank 24): env:// -| distributed init (rank 25): env:// -| distributed init (rank 26): env:// -| distributed init (rank 27): env:// -| distributed init (rank 28): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 28): env:// -| distributed init (rank 26): env:// -| distributed init (rank 29): env:// -| distributed init (rank 27): env:// -| distributed init (rank 24): env:// -| distributed init (rank 25): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 29): env:// -| distributed init (rank 24): env:// -| distributed init (rank 25): env:// -| distributed init (rank 26): env:// -| distributed init (rank 27): env:// -| distributed init (rank 28): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 24): env:// -| distributed init (rank 29): env:// -| distributed init (rank 25): env:// -| distributed init (rank 28): env:// -| distributed init (rank 27): env:// -| distributed init (rank 26): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 29): env:// -| distributed init (rank 28): env:// -| distributed init (rank 26): env:// -| distributed init (rank 25): env:// -| distributed init (rank 24): env:// -| distributed init (rank 27): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 25): env:// -| distributed init (rank 27): env:// -| distributed init (rank 24): env:// -| distributed init (rank 28): env:// -| distributed init (rank 29): env:// -| distributed init (rank 26): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 27): env:// -| distributed init (rank 29): env:// -| distributed init (rank 26): env:// -| distributed init (rank 24): env:// -| distributed init (rank 25): env:// -| distributed init (rank 28): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 24): env:// -| distributed init (rank 29): env:// -| distributed init (rank 26): env:// -| distributed init (rank 27): env:// -| distributed init (rank 25): env:// -| distributed init (rank 28): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 25): env:// -| distributed init (rank 26): env:// -| distributed init (rank 24): env:// -| distributed init (rank 27): env:// -| distributed init (rank 28): env:// -| distributed init (rank 29): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 26): env:// -| distributed init (rank 25): env:// -| distributed init (rank 27): env:// -| distributed init (rank 29): env:// -| distributed init (rank 28): env:// -| distributed init (rank 24): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 2749012) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python -Traceback (most recent call last): - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in - sys.exit(main()) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper - return f(*args, **kwargs) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main - run(args) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run - elastic_launch( - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -======================================================== -train_cycling_gen.py FAILED --------------------------------------------------------- -Failures: -[1]: - time : 2023-09-21_12:38:12 - host : sc13 - rank : 25 (local_rank: 1) - exitcode : -6 (pid: 2749013) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2749013 -[2]: - time : 2023-09-21_12:38:12 - host : sc13 - rank : 26 (local_rank: 2) - exitcode : -6 (pid: 2749014) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2749014 -[3]: - time : 2023-09-21_12:38:12 - host : sc13 - rank : 27 (local_rank: 3) - exitcode : -6 (pid: 2749015) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2749015 -[4]: - time : 2023-09-21_12:38:12 - host : sc13 - rank : 28 (local_rank: 4) - exitcode : -6 (pid: 2749016) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2749016 -[5]: - time : 2023-09-21_12:38:12 - host : sc13 - rank : 29 (local_rank: 5) - exitcode : -6 (pid: 2749017) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2749017 --------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2023-09-21_12:38:12 - host : sc13 - rank : 24 (local_rank: 0) - exitcode : -6 (pid: 2749012) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2749012 -======================================================== diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_5.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_5.txt deleted file mode 100644 index 68a974e8..00000000 --- a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_5.txt +++ /dev/null @@ -1,1310 +0,0 @@ -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 34): env:// -| distributed init (rank 35): env:// -| distributed init (rank 31): env:// -| distributed init (rank 32): env:// -| distributed init (rank 33): env:// -| distributed init (rank 30): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 34): env:// -| distributed init (rank 32): env:// -| distributed init (rank 31): env:// -| distributed init (rank 35): env:// -| distributed init (rank 30): env:// -| distributed init (rank 33): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 30): env:// -| distributed init (rank 33): env:// -| distributed init (rank 34): env:// -| distributed init (rank 32): env:// -| distributed init (rank 31): env:// -| distributed init (rank 35): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 35): env:// -| distributed init (rank 33): env:// -| distributed init (rank 30): env:// -| distributed init (rank 32): env:// -| distributed init (rank 31): env:// -| distributed init (rank 34): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 30): env:// -| distributed init (rank 35): env:// -| distributed init (rank 31): env:// -| distributed init (rank 34): env:// -| distributed init (rank 33): env:// -| distributed init (rank 32): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 32): env:// -| distributed init (rank 33): env:// -| distributed init (rank 31): env:// -| distributed init (rank 34): env:// -| distributed init (rank 35): env:// -| distributed init (rank 30): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 32): env:// -| distributed init (rank 33): env:// -| distributed init (rank 34): env:// -| distributed init (rank 35): env:// -| distributed init (rank 31): env:// -| distributed init (rank 30): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 35): env:// -| distributed init (rank 31): env:// -| distributed init (rank 34): env:// -| distributed init (rank 30): env:// -| distributed init (rank 33): env:// -| distributed init (rank 32): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 32): env:// -| distributed init (rank 31): env:// -| distributed init (rank 30): env:// -| distributed init (rank 34): env:// -| distributed init (rank 35): env:// -| distributed init (rank 33): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 31): env:// -| distributed init (rank 34): env:// -| distributed init (rank 35): env:// -| distributed init (rank 33): env:// -| distributed init (rank 32): env:// -| distributed init (rank 30): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 35): env:// -| distributed init (rank 31): env:// -| distributed init (rank 30): env:// -| distributed init (rank 34): env:// -| distributed init (rank 32): env:// -| distributed init (rank 33): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 32): env:// -| distributed init (rank 34): env:// -| distributed init (rank 35): env:// -| distributed init (rank 33): env:// -| distributed init (rank 31): env:// -| distributed init (rank 30): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 35): env:// -| distributed init (rank 34): env:// -| distributed init (rank 32): env:// -| distributed init (rank 31): env:// -| distributed init (rank 33): env:// -| distributed init (rank 30): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 34): env:// -| distributed init (rank 31): env:// -| distributed init (rank 30): env:// -| distributed init (rank 33): env:// -| distributed init (rank 32): env:// -| distributed init (rank 35): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 34): env:// -| distributed init (rank 31): env:// -| distributed init (rank 35): env:// -| distributed init (rank 33): env:// -| distributed init (rank 30): env:// -| distributed init (rank 32): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 31): env:// -| distributed init (rank 30): env:// -| distributed init (rank 34): env:// -| distributed init (rank 32): env:// -| distributed init (rank 35): env:// -| distributed init (rank 33): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 34): env:// -| distributed init (rank 30): env:// -| distributed init (rank 35): env:// -| distributed init (rank 32): env:// -| distributed init (rank 33): env:// -| distributed init (rank 31): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 31): env:// -| distributed init (rank 35): env:// -| distributed init (rank 34): env:// -| distributed init (rank 32): env:// -| distributed init (rank 30): env:// -| distributed init (rank 33): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 31): env:// -| distributed init (rank 30): env:// -| distributed init (rank 34): env:// -| distributed init (rank 35): env:// -| distributed init (rank 32): env:// -| distributed init (rank 33): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 34): env:// -| distributed init (rank 31): env:// -| distributed init (rank 32): env:// -| distributed init (rank 33): env:// -| distributed init (rank 30): env:// -| distributed init (rank 35): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 32): env:// -| distributed init (rank 35): env:// -| distributed init (rank 31): env:// -| distributed init (rank 33): env:// -| distributed init (rank 34): env:// -| distributed init (rank 30): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 30): env:// -| distributed init (rank 32): env:// -| distributed init (rank 35): env:// -| distributed init (rank 34): env:// -| distributed init (rank 31): env:// -| distributed init (rank 33): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 32): env:// -| distributed init (rank 35): env:// -| distributed init (rank 30): env:// -| distributed init (rank 34): env:// -| distributed init (rank 31): env:// -| distributed init (rank 33): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 35): env:// -| distributed init (rank 34): env:// -| distributed init (rank 32): env:// -| distributed init (rank 30): env:// -| distributed init (rank 33): env:// -| distributed init (rank 31): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 30): env:// -| distributed init (rank 34): env:// -| distributed init (rank 32): env:// -| distributed init (rank 33): env:// -| distributed init (rank 31): env:// -| distributed init (rank 35): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 32): env:// -| distributed init (rank 34): env:// -| distributed init (rank 35): env:// -| distributed init (rank 30): env:// -| distributed init (rank 33): env:// -| distributed init (rank 31): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 34): env:// -| distributed init (rank 31): env:// -| distributed init (rank 35): env:// -| distributed init (rank 30): env:// -| distributed init (rank 32): env:// -| distributed init (rank 33): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 30): env:// -| distributed init (rank 32): env:// -| distributed init (rank 33): env:// -| distributed init (rank 35): env:// -| distributed init (rank 31): env:// -| distributed init (rank 34): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 30): env:// -| distributed init (rank 35): env:// -| distributed init (rank 32): env:// -| distributed init (rank 33): env:// -| distributed init (rank 31): env:// -| distributed init (rank 34): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 34): env:// -| distributed init (rank 32): env:// -| distributed init (rank 31): env:// -| distributed init (rank 35): env:// -| distributed init (rank 30): env:// -| distributed init (rank 33): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 30): env:// -| distributed init (rank 34): env:// -| distributed init (rank 35): env:// -| distributed init (rank 31): env:// -| distributed init (rank 33): env:// -| distributed init (rank 32): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 3421449) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python -Traceback (most recent call last): - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in - sys.exit(main()) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper - return f(*args, **kwargs) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main - run(args) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run - elastic_launch( - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -======================================================== -train_cycling_gen.py FAILED --------------------------------------------------------- -Failures: -[1]: - time : 2023-09-21_12:38:12 - host : sc17 - rank : 31 (local_rank: 1) - exitcode : -6 (pid: 3421450) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 3421450 -[2]: - time : 2023-09-21_12:38:12 - host : sc17 - rank : 32 (local_rank: 2) - exitcode : -6 (pid: 3421451) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 3421451 -[3]: - time : 2023-09-21_12:38:12 - host : sc17 - rank : 33 (local_rank: 3) - exitcode : -6 (pid: 3421452) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 3421452 -[4]: - time : 2023-09-21_12:38:12 - host : sc17 - rank : 34 (local_rank: 4) - exitcode : -6 (pid: 3421453) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 3421453 -[5]: - time : 2023-09-21_12:38:12 - host : sc17 - rank : 35 (local_rank: 5) - exitcode : -6 (pid: 3421454) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 3421454 --------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2023-09-21_12:38:12 - host : sc17 - rank : 30 (local_rank: 0) - exitcode : -6 (pid: 3421449) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 3421449 -======================================================== diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_6.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_6.txt deleted file mode 100644 index 2882a604..00000000 --- a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_6.txt +++ /dev/null @@ -1,1310 +0,0 @@ -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 39): env:// -| distributed init (rank 40): env:// -| distributed init (rank 37): env:// -| distributed init (rank 38): env:// -| distributed init (rank 36): env:// -| distributed init (rank 41): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 38): env:// -| distributed init (rank 40): env:// -| distributed init (rank 37): env:// -| distributed init (rank 39): env:// -| distributed init (rank 36): env:// -| distributed init (rank 41): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 37): env:// -| distributed init (rank 41): env:// -| distributed init (rank 38): env:// -| distributed init (rank 36): env:// -| distributed init (rank 39): env:// -| distributed init (rank 40): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 37): env:// -| distributed init (rank 36): env:// -| distributed init (rank 40): env:// -| distributed init (rank 38): env:// -| distributed init (rank 39): env:// -| distributed init (rank 41): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 39): env:// -| distributed init (rank 37): env:// -| distributed init (rank 38): env:// -| distributed init (rank 36): env:// -| distributed init (rank 41): env:// -| distributed init (rank 40): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 36): env:// -| distributed init (rank 38): env:// -| distributed init (rank 39): env:// -| distributed init (rank 40): env:// -| distributed init (rank 41): env:// -| distributed init (rank 37): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 39): env:// -| distributed init (rank 38): env:// -| distributed init (rank 41): env:// -| distributed init (rank 37): env:// -| distributed init (rank 40): env:// -| distributed init (rank 36): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 37): env:// -| distributed init (rank 41): env:// -| distributed init (rank 39): env:// -| distributed init (rank 36): env:// -| distributed init (rank 40): env:// -| distributed init (rank 38): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 40): env:// -| distributed init (rank 39): env:// -| distributed init (rank 38): env:// -| distributed init (rank 41): env:// -| distributed init (rank 36): env:// -| distributed init (rank 37): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 38): env:// -| distributed init (rank 37): env:// -| distributed init (rank 39): env:// -| distributed init (rank 40): env:// -| distributed init (rank 41): env:// -| distributed init (rank 36): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 40): env:// -| distributed init (rank 37): env:// -| distributed init (rank 41): env:// -| distributed init (rank 38): env:// -| distributed init (rank 39): env:// -| distributed init (rank 36): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 39): env:// -| distributed init (rank 37): env:// -| distributed init (rank 38): env:// -| distributed init (rank 40): env:// -| distributed init (rank 41): env:// -| distributed init (rank 36): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 38): env:// -| distributed init (rank 36): env:// -| distributed init (rank 37): env:// -| distributed init (rank 41): env:// -| distributed init (rank 39): env:// -| distributed init (rank 40): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 37): env:// -| distributed init (rank 40): env:// -| distributed init (rank 38): env:// -| distributed init (rank 36): env:// -| distributed init (rank 39): env:// -| distributed init (rank 41): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 41): env:// -| distributed init (rank 38): env:// -| distributed init (rank 40): env:// -| distributed init (rank 39): env:// -| distributed init (rank 36): env:// -| distributed init (rank 37): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 40): env:// -| distributed init (rank 38): env:// -| distributed init (rank 36): env:// -| distributed init (rank 41): env:// -| distributed init (rank 39): env:// -| distributed init (rank 37): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 36): env:// -| distributed init (rank 41): env:// -| distributed init (rank 40): env:// -| distributed init (rank 38): env:// -| distributed init (rank 37): env:// -| distributed init (rank 39): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 40): env:// -| distributed init (rank 39): env:// -| distributed init (rank 38): env:// -| distributed init (rank 36): env:// -| distributed init (rank 37): env:// -| distributed init (rank 41): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 40): env:// -| distributed init (rank 39): env:// -| distributed init (rank 38): env:// -| distributed init (rank 37): env:// -| distributed init (rank 36): env:// -| distributed init (rank 41): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 36): env:// -| distributed init (rank 37): env:// -| distributed init (rank 40): env:// -| distributed init (rank 39): env:// -| distributed init (rank 41): env:// -| distributed init (rank 38): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 36): env:// -| distributed init (rank 38): env:// -| distributed init (rank 39): env:// -| distributed init (rank 40): env:// -| distributed init (rank 37): env:// -| distributed init (rank 41): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 41): env:// -| distributed init (rank 40): env:// -| distributed init (rank 38): env:// -| distributed init (rank 36): env:// -| distributed init (rank 37): env:// -| distributed init (rank 39): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 41): env:// -| distributed init (rank 39): env:// -| distributed init (rank 38): env:// -| distributed init (rank 40): env:// -| distributed init (rank 37): env:// -| distributed init (rank 36): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 37): env:// -| distributed init (rank 39): env:// -| distributed init (rank 40): env:// -| distributed init (rank 36): env:// -| distributed init (rank 38): env://| distributed init (rank 41): env:// - -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 38): env:// -| distributed init (rank 39): env:// -| distributed init (rank 41): env:// -| distributed init (rank 36): env:// -| distributed init (rank 37): env:// -| distributed init (rank 40): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 39): env:// -| distributed init (rank 38): env:// -| distributed init (rank 40): env:// -| distributed init (rank 41): env:// -| distributed init (rank 36): env:// -| distributed init (rank 37): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 38): env:// -| distributed init (rank 37): env:// -| distributed init (rank 36): env:// -| distributed init (rank 40): env:// -| distributed init (rank 41): env:// -| distributed init (rank 39): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 36): env:// -| distributed init (rank 37): env:// -| distributed init (rank 38): env:// -| distributed init (rank 41): env:// -| distributed init (rank 40): env:// -| distributed init (rank 39): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 37): env:// -| distributed init (rank 41): env:// -| distributed init (rank 39): env:// -| distributed init (rank 36): env:// -| distributed init (rank 38): env:// -| distributed init (rank 40): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 37): env:// -| distributed init (rank 38): env:// -| distributed init (rank 40): env:// -| distributed init (rank 41): env:// -| distributed init (rank 39): env:// -| distributed init (rank 36): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 39): env:// -| distributed init (rank 38): env:// -| distributed init (rank 36): env:// -| distributed init (rank 40): env:// -| distributed init (rank 37): env:// -| distributed init (rank 41): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 1500785) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python -Traceback (most recent call last): - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in - sys.exit(main()) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper - return f(*args, **kwargs) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main - run(args) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run - elastic_launch( - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -======================================================== -train_cycling_gen.py FAILED --------------------------------------------------------- -Failures: -[1]: - time : 2023-09-21_12:38:12 - host : sc9 - rank : 37 (local_rank: 1) - exitcode : -6 (pid: 1500786) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 1500786 -[2]: - time : 2023-09-21_12:38:12 - host : sc9 - rank : 38 (local_rank: 2) - exitcode : -6 (pid: 1500787) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 1500787 -[3]: - time : 2023-09-21_12:38:12 - host : sc9 - rank : 39 (local_rank: 3) - exitcode : -6 (pid: 1500788) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 1500788 -[4]: - time : 2023-09-21_12:38:12 - host : sc9 - rank : 40 (local_rank: 4) - exitcode : -6 (pid: 1500789) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 1500789 -[5]: - time : 2023-09-21_12:38:12 - host : sc9 - rank : 41 (local_rank: 5) - exitcode : -6 (pid: 1500790) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 1500790 --------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2023-09-21_12:38:12 - host : sc9 - rank : 36 (local_rank: 0) - exitcode : -6 (pid: 1500785) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 1500785 -======================================================== diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_7.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_7.txt deleted file mode 100644 index fb26dabd..00000000 --- a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_7.txt +++ /dev/null @@ -1,1310 +0,0 @@ -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 42): env:// -| distributed init (rank 47): env:// -| distributed init (rank 43): env:// -| distributed init (rank 45): env:// -| distributed init (rank 44): env:// -| distributed init (rank 46): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 43): env:// -| distributed init (rank 44): env:// -| distributed init (rank 46): env:// -| distributed init (rank 42): env:// -| distributed init (rank 47): env:// -| distributed init (rank 45): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 47): env:// -| distributed init (rank 44): env:// -| distributed init (rank 42): env:// -| distributed init (rank 46): env:// -| distributed init (rank 43): env:// -| distributed init (rank 45): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 44): env:// -| distributed init (rank 45): env:// -| distributed init (rank 43): env:// -| distributed init (rank 46): env:// -| distributed init (rank 47): env:// -| distributed init (rank 42): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 46): env:// -| distributed init (rank 47): env:// -| distributed init (rank 44): env:// -| distributed init (rank 43): env:// -| distributed init (rank 42): env:// -| distributed init (rank 45): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 42): env:// -| distributed init (rank 44): env:// -| distributed init (rank 46): env:// -| distributed init (rank 45): env:// -| distributed init (rank 47): env:// -| distributed init (rank 43): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 45): env:// -| distributed init (rank 47): env:// -| distributed init (rank 42): env:// -| distributed init (rank 44): env:// -| distributed init (rank 46): env:// -| distributed init (rank 43): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 44): env:// -| distributed init (rank 43): env:// -| distributed init (rank 47): env:// -| distributed init (rank 45): env:// -| distributed init (rank 42): env:// -| distributed init (rank 46): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 46): env:// -| distributed init (rank 44): env:// -| distributed init (rank 42): env:// -| distributed init (rank 45): env:// -| distributed init (rank 47): env:// -| distributed init (rank 43): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 46): env:// -| distributed init (rank 45): env:// -| distributed init (rank 47): env:// -| distributed init (rank 44): env:// -| distributed init (rank 42): env:// -| distributed init (rank 43): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 43): env:// -| distributed init (rank 45): env:// -| distributed init (rank 42): env:// -| distributed init (rank 47): env:// -| distributed init (rank 44): env:// -| distributed init (rank 46): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 45): env:// -| distributed init (rank 44): env:// -| distributed init (rank 47): env:// -| distributed init (rank 42): env:// -| distributed init (rank 46): env:// -| distributed init (rank 43): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 45): env:// -| distributed init (rank 47): env:// -| distributed init (rank 46): env:// -| distributed init (rank 42): env:// -| distributed init (rank 44): env:// -| distributed init (rank 43): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 47): env:// -| distributed init (rank 43): env:// -| distributed init (rank 44): env:// -| distributed init (rank 45): env:// -| distributed init (rank 42): env:// -| distributed init (rank 46): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 44): env:// -| distributed init (rank 43): env:// -| distributed init (rank 46): env:// -| distributed init (rank 47): env:// -| distributed init (rank 42): env:// -| distributed init (rank 45): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 42): env:// -| distributed init (rank 47): env:// -| distributed init (rank 44): env:// -| distributed init (rank 46): env:// -| distributed init (rank 45): env:// -| distributed init (rank 43): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 45): env:// -| distributed init (rank 43): env:// -| distributed init (rank 46): env:// -| distributed init (rank 44): env:// -| distributed init (rank 42): env:// -| distributed init (rank 47): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 44): env:// -| distributed init (rank 45): env:// -| distributed init (rank 46): env:// -| distributed init (rank 42): env:// -| distributed init (rank 47): env:// -| distributed init (rank 43): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 42): env:// -| distributed init (rank 47): env:// -| distributed init (rank 45): env:// -| distributed init (rank 43): env:// -| distributed init (rank 44): env:// -| distributed init (rank 46): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 46): env:// -| distributed init (rank 44): env:// -| distributed init (rank 42): env:// -| distributed init (rank 47): env:// -| distributed init (rank 43): env:// -| distributed init (rank 45): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 46): env:// -| distributed init (rank 43): env:// -| distributed init (rank 45): env:// -| distributed init (rank 47): env:// -| distributed init (rank 44): env:// -| distributed init (rank 42): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 44): env:// -| distributed init (rank 43): env:// -| distributed init (rank 45): env:// -| distributed init (rank 42): env:// -| distributed init (rank 47): env:// -| distributed init (rank 46): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 43): env:// -| distributed init (rank 47): env:// -| distributed init (rank 45): env:// -| distributed init (rank 42): env:// -| distributed init (rank 44): env:// -| distributed init (rank 46): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 43): env:// -| distributed init (rank 46): env:// -| distributed init (rank 47): env:// -| distributed init (rank 44): env:// -| distributed init (rank 42): env:// -| distributed init (rank 45): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 44): env:// -| distributed init (rank 47): env:// -| distributed init (rank 43): env:// -| distributed init (rank 45): env:// -| distributed init (rank 46): env:// -| distributed init (rank 42): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 47): env:// -| distributed init (rank 42): env:// -| distributed init (rank 45): env:// -| distributed init (rank 46): env:// -| distributed init (rank 44): env:// -| distributed init (rank 43): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 47): env:// -| distributed init (rank 46): env:// -| distributed init (rank 44): env:// -| distributed init (rank 43): env:// -| distributed init (rank 45): env:// -| distributed init (rank 42): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 47): env:// -| distributed init (rank 43): env:// -| distributed init (rank 42): env:// -| distributed init (rank 46): env:// -| distributed init (rank 45): env:// -| distributed init (rank 44): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 42): env:// -| distributed init (rank 43): env:// -| distributed init (rank 45): env:// -| distributed init (rank 44): env:// -| distributed init (rank 46): env:// -| distributed init (rank 47): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 43): env:// -| distributed init (rank 47): env:// -| distributed init (rank 45): env:// -| distributed init (rank 46): env:// -| distributed init (rank 42): env:// -| distributed init (rank 44): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 42): env:// -| distributed init (rank 43): env:// -| distributed init (rank 45): env:// -| distributed init (rank 44): env:// -| distributed init (rank 46): env:// -| distributed init (rank 47): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 3214610) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python -Traceback (most recent call last): - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in - sys.exit(main()) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper - return f(*args, **kwargs) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main - run(args) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run - elastic_launch( - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -======================================================== -train_cycling_gen.py FAILED --------------------------------------------------------- -Failures: -[1]: - time : 2023-09-21_12:38:12 - host : sc19 - rank : 43 (local_rank: 1) - exitcode : -6 (pid: 3214611) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 3214611 -[2]: - time : 2023-09-21_12:38:12 - host : sc19 - rank : 44 (local_rank: 2) - exitcode : -6 (pid: 3214612) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 3214612 -[3]: - time : 2023-09-21_12:38:12 - host : sc19 - rank : 45 (local_rank: 3) - exitcode : -6 (pid: 3214613) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 3214613 -[4]: - time : 2023-09-21_12:38:12 - host : sc19 - rank : 46 (local_rank: 4) - exitcode : -6 (pid: 3214614) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 3214614 -[5]: - time : 2023-09-21_12:38:12 - host : sc19 - rank : 47 (local_rank: 5) - exitcode : -6 (pid: 3214615) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 3214615 --------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2023-09-21_12:38:12 - host : sc19 - rank : 42 (local_rank: 0) - exitcode : -6 (pid: 3214610) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 3214610 -======================================================== diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_8.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_8.txt deleted file mode 100644 index 3461fba9..00000000 --- a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_8.txt +++ /dev/null @@ -1,1310 +0,0 @@ -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 53): env:// -| distributed init (rank 48): env:// -| distributed init (rank 49): env:// -| distributed init (rank 52): env:// -| distributed init (rank 50): env:// -| distributed init (rank 51): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 53): env:// -| distributed init (rank 51): env:// -| distributed init (rank 48): env:// -| distributed init (rank 50): env:// -| distributed init (rank 49): env:// -| distributed init (rank 52): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 50): env:// -| distributed init (rank 52): env:// -| distributed init (rank 48): env:// -| distributed init (rank 49): env:// -| distributed init (rank 53): env:// -| distributed init (rank 51): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 53): env:// -| distributed init (rank 49): env:// -| distributed init (rank 52): env:// -| distributed init (rank 50): env:// -| distributed init (rank 51): env:// -| distributed init (rank 48): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 48): env:// -| distributed init (rank 50): env:// -| distributed init (rank 52): env:// -| distributed init (rank 53): env:// -| distributed init (rank 51): env:// -| distributed init (rank 49): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 51): env:// -| distributed init (rank 52): env:// -| distributed init (rank 48): env:// -| distributed init (rank 53): env:// -| distributed init (rank 49): env:// -| distributed init (rank 50): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 52): env:// -| distributed init (rank 48): env:// -| distributed init (rank 51): env:// -| distributed init (rank 49): env:// -| distributed init (rank 50): env:// -| distributed init (rank 53): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 52): env:// -| distributed init (rank 50): env:// -| distributed init (rank 48): env:// -| distributed init (rank 53): env:// -| distributed init (rank 49): env:// -| distributed init (rank 51): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 52): env:// -| distributed init (rank 48): env:// -| distributed init (rank 49): env:// -| distributed init (rank 50): env:// -| distributed init (rank 51): env:// -| distributed init (rank 53): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 48): env:// -| distributed init (rank 49): env:// -| distributed init (rank 52): env:// -| distributed init (rank 50): env:// -| distributed init (rank 51): env:// -| distributed init (rank 53): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 50): env:// -| distributed init (rank 53): env:// -| distributed init (rank 52): env:// -| distributed init (rank 49): env:// -| distributed init (rank 51): env:// -| distributed init (rank 48): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 50): env:// -| distributed init (rank 51): env:// -| distributed init (rank 48): env:// -| distributed init (rank 53): env:// -| distributed init (rank 52): env:// -| distributed init (rank 49): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 48): env:// -| distributed init (rank 53): env:// -| distributed init (rank 52): env:// -| distributed init (rank 51): env:// -| distributed init (rank 49): env:// -| distributed init (rank 50): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 52): env:// -| distributed init (rank 53): env:// -| distributed init (rank 48): env:// -| distributed init (rank 49): env:// -| distributed init (rank 51): env:// -| distributed init (rank 50): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 50): env:// -| distributed init (rank 53): env:// -| distributed init (rank 51): env:// -| distributed init (rank 52): env:// -| distributed init (rank 49): env:// -| distributed init (rank 48): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 51): env:// -| distributed init (rank 52): env:// -| distributed init (rank 50): env:// -| distributed init (rank 48): env:// -| distributed init (rank 49): env:// -| distributed init (rank 53): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 52): env:// -| distributed init (rank 53): env:// -| distributed init (rank 51): env:// -| distributed init (rank 49): env:// -| distributed init (rank 48): env:// -| distributed init (rank 50): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 50): env:// -| distributed init (rank 51): env:// -| distributed init (rank 49): env:// -| distributed init (rank 48): env:// -| distributed init (rank 53): env:// -| distributed init (rank 52): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 51): env:// -| distributed init (rank 49): env:// -| distributed init (rank 52): env:// -| distributed init (rank 53): env:// -| distributed init (rank 50): env:// -| distributed init (rank 48): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 53): env:// -| distributed init (rank 48): env:// -| distributed init (rank 51): env:// -| distributed init (rank 52): env:// -| distributed init (rank 50): env:// -| distributed init (rank 49): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 53): env:// -| distributed init (rank 48): env:// -| distributed init (rank 49): env:// -| distributed init (rank 52): env:// -| distributed init (rank 50): env:// -| distributed init (rank 51): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 52): env:// -| distributed init (rank 50): env:// -| distributed init (rank 53): env:// -| distributed init (rank 49): env:// -| distributed init (rank 51): env:// -| distributed init (rank 48): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 51): env:// -| distributed init (rank 53): env:// -| distributed init (rank 52): env:// -| distributed init (rank 50): env:// -| distributed init (rank 48): env:// -| distributed init (rank 49): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 50): env:// -| distributed init (rank 52): env:// -| distributed init (rank 49): env:// -| distributed init (rank 53): env:// -| distributed init (rank 48): env:// -| distributed init (rank 51): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 48): env:// -| distributed init (rank 49): env:// -| distributed init (rank 52): env:// -| distributed init (rank 50): env:// -| distributed init (rank 53): env:// -| distributed init (rank 51): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 49): env:// -| distributed init (rank 48): env:// -| distributed init (rank 53): env:// -| distributed init (rank 50): env:// -| distributed init (rank 51): env:// -| distributed init (rank 52): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 53): env:// -| distributed init (rank 51): env:// -| distributed init (rank 50): env:// -| distributed init (rank 48): env:// -| distributed init (rank 52): env:// -| distributed init (rank 49): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 53): env:// -| distributed init (rank 52): env:// -| distributed init (rank 49): env:// -| distributed init (rank 48): env:// -| distributed init (rank 50): env:// -| distributed init (rank 51): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 49): env:// -| distributed init (rank 51): env:// -| distributed init (rank 50): env:// -| distributed init (rank 48): env:// -| distributed init (rank 52): env:// -| distributed init (rank 53): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 51): env:// -| distributed init (rank 53): env:// -| distributed init (rank 50): env:// -| distributed init (rank 48): env:// -| distributed init (rank 49): env:// -| distributed init (rank 52): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 53): env:// -| distributed init (rank 52): env:// -| distributed init (rank 48): env:// -| distributed init (rank 51): env:// -| distributed init (rank 49): env:// -| distributed init (rank 50): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 2065295) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python -Traceback (most recent call last): - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in - sys.exit(main()) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper - return f(*args, **kwargs) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main - run(args) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run - elastic_launch( - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -======================================================== -train_cycling_gen.py FAILED --------------------------------------------------------- -Failures: -[1]: - time : 2023-09-21_12:38:12 - host : sc10 - rank : 49 (local_rank: 1) - exitcode : -6 (pid: 2065296) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2065296 -[2]: - time : 2023-09-21_12:38:12 - host : sc10 - rank : 50 (local_rank: 2) - exitcode : -6 (pid: 2065297) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2065297 -[3]: - time : 2023-09-21_12:38:12 - host : sc10 - rank : 51 (local_rank: 3) - exitcode : -6 (pid: 2065298) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2065298 -[4]: - time : 2023-09-21_12:38:12 - host : sc10 - rank : 52 (local_rank: 4) - exitcode : -6 (pid: 2065299) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2065299 -[5]: - time : 2023-09-21_12:38:12 - host : sc10 - rank : 53 (local_rank: 5) - exitcode : -6 (pid: 2065300) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2065300 --------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2023-09-21_12:38:12 - host : sc10 - rank : 48 (local_rank: 0) - exitcode : -6 (pid: 2065295) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2065295 -======================================================== diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_9.txt b/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_9.txt deleted file mode 100644 index b16f4d1c..00000000 --- a/monai_brats_mri_2d/exp_1855 (autoencoder)/rank_9.txt +++ /dev/null @@ -1,1310 +0,0 @@ -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 54): env:// -| distributed init (rank 56): env:// -| distributed init (rank 57): env:// -| distributed init (rank 55): env:// -| distributed init (rank 59): env:// -| distributed init (rank 58): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 55): env:// -| distributed init (rank 59): env:// -| distributed init (rank 54): env:// -| distributed init (rank 56): env:// -| distributed init (rank 57): env:// -| distributed init (rank 58): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 55): env:// -| distributed init (rank 59): env:// -| distributed init (rank 57): env:// -| distributed init (rank 54): env:// -| distributed init (rank 56): env:// -| distributed init (rank 58): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 56): env:// -| distributed init (rank 58): env:// -| distributed init (rank 59): env:// -| distributed init (rank 55): env:// -| distributed init (rank 57): env:// -| distributed init (rank 54): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 55): env:// -| distributed init (rank 54): env:// -| distributed init (rank 58): env:// -| distributed init (rank 57): env:// -| distributed init (rank 56): env:// -| distributed init (rank 59): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 59): env:// -| distributed init (rank 57): env:// -| distributed init (rank 58): env:// -| distributed init (rank 55): env:// -| distributed init (rank 56): env:// -| distributed init (rank 54): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 54): env:// -| distributed init (rank 59): env:// -| distributed init (rank 58): env:// -| distributed init (rank 57): env:// -| distributed init (rank 55): env:// -| distributed init (rank 56): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 59): env:// -| distributed init (rank 58): env:// -| distributed init (rank 57): env:// -| distributed init (rank 55): env:// -| distributed init (rank 56): env:// -| distributed init (rank 54): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 58): env:// -| distributed init (rank 56): env:// -| distributed init (rank 55): env:// -| distributed init (rank 59): env:// -| distributed init (rank 54): env:// -| distributed init (rank 57): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 57): env:// -| distributed init (rank 56): env:// -| distributed init (rank 58): env:// -| distributed init (rank 54): env:// -| distributed init (rank 55): env:// -| distributed init (rank 59): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 59): env:// -| distributed init (rank 56): env:// -| distributed init (rank 57): env:// -| distributed init (rank 55): env:// -| distributed init (rank 54): env:// -| distributed init (rank 58): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 54): env:// -| distributed init (rank 58): env:// -| distributed init (rank 59): env:// -| distributed init (rank 55): env:// -| distributed init (rank 56): env:// -| distributed init (rank 57): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 58): env:// -| distributed init (rank 56): env:// -| distributed init (rank 55): env:// -| distributed init (rank 59): env:// -| distributed init (rank 54): env:// -| distributed init (rank 57): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 58): env:// -| distributed init (rank 55): env:// -| distributed init (rank 57): env:// -| distributed init (rank 54): env:// -| distributed init (rank 56): env:// -| distributed init (rank 59): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 56): env:// -| distributed init (rank 58): env:// -| distributed init (rank 59): env:// -| distributed init (rank 55): env:// -| distributed init (rank 57): env:// -| distributed init (rank 54): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 56): env:// -| distributed init (rank 57): env:// -| distributed init (rank 59): env:// -| distributed init (rank 58): env:// -| distributed init (rank 55): env:// -| distributed init (rank 54): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 54): env:// -| distributed init (rank 56): env:// -| distributed init (rank 55): env:// -| distributed init (rank 57): env:// -| distributed init (rank 58): env:// -| distributed init (rank 59): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 59): env:// -| distributed init (rank 56): env:// -| distributed init (rank 55): env:// -| distributed init (rank 54): env:// -| distributed init (rank 58): env:// -| distributed init (rank 57): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 55): env:// -| distributed init (rank 56): env:// -| distributed init (rank 59): env:// -| distributed init (rank 57): env:// -| distributed init (rank 54): env:// -| distributed init (rank 58): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 55): env:// -| distributed init (rank 58): env:// -| distributed init (rank 57): env:// -| distributed init (rank 56): env:// -| distributed init (rank 54): env:// -| distributed init (rank 59): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 55): env:// -| distributed init (rank 57): env:// -| distributed init (rank 56): env:// -| distributed init (rank 59): env:// -| distributed init (rank 54): env:// -| distributed init (rank 58): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 56): env:// -| distributed init (rank 59): env:// -| distributed init (rank 58): env:// -| distributed init (rank 57): env:// -| distributed init (rank 55): env:// -| distributed init (rank 54): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 54): env:// -| distributed init (rank 57): env:// -| distributed init (rank 58): env:// -| distributed init (rank 59): env:// -| distributed init (rank 56): env:// -| distributed init (rank 55): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 58): env:// -| distributed init (rank 57): env:// -| distributed init (rank 59): env:// -| distributed init (rank 56): env:// -| distributed init (rank 54): env:// -| distributed init (rank 55): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 59): env:// -| distributed init (rank 58): env:// -| distributed init (rank 56): env:// -| distributed init (rank 54): env:// -| distributed init (rank 57): env:// -| distributed init (rank 55): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 57): env:// -| distributed init (rank 54): env:// -| distributed init (rank 55): env:// -| distributed init (rank 59): env:// -| distributed init (rank 58): env:// -| distributed init (rank 56): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 57): env:// -| distributed init (rank 56): env:// -| distributed init (rank 58): env:// -| distributed init (rank 59): env:// -| distributed init (rank 54): env:// -| distributed init (rank 55): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 58): env:// -| distributed init (rank 55): env:// -| distributed init (rank 56): env:// -| distributed init (rank 57): env:// -| distributed init (rank 59): env:// -| distributed init (rank 54): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 57): env:// -| distributed init (rank 58): env:// -| distributed init (rank 59): env:// -| distributed init (rank 56): env:// -| distributed init (rank 54): env:// -| distributed init (rank 55): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 57): env:// -| distributed init (rank 54): env:// -| distributed init (rank 58): env:// -| distributed init (rank 59): env:// -| distributed init (rank 55): env:// -| distributed init (rank 56): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -WARNING:torch.distributed.run: -***************************************** -Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -***************************************** -| distributed init (rank 54): env:// -| distributed init (rank 58): env:// -| distributed init (rank 56): env:// -| distributed init (rank 59): env:// -| distributed init (rank 57): env:// -| distributed init (rank 55): env:// -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -: Class `AddChanneld` has been deprecated since version 0.8. It will be removed in version 1.3. please use MetaTensor data type and monai.transforms.EnsureChannelFirstd instead with `channel_dim='no_channel'`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -monai.transforms.utility.dictionary EnsureChannelFirstd.__init__:meta_keys: Argument `meta_keys` has been deprecated since version 0.9. not needed if image is type `MetaTensor`. -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -Grad strides do not match bucket view strides. This may indicate grad was not created according to the gradient layout contract, or that the param's strides changed since DDP was constructed. This is not an error, but may impair performance. -grad.sizes() = [64, 128, 1, 1], strides() = [128, 1, 128, 128] -bucket_view.sizes() = [64, 128, 1, 1], strides() = [128, 1, 1, 1] (Triggered internally at ../torch/csrc/distributed/c10d/reducer.cpp:325.) -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -free(): invalid pointer -ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 2917693) of binary: /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/python -Traceback (most recent call last): - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/bin/torchrun", line 8, in - sys.exit(main()) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper - return f(*args, **kwargs) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main - run(args) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run - elastic_launch( - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - File "/mnt/Client/StrongUniversity/USYD-04/usyd04_adam/.venv/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent - raise ChildFailedError( -torch.distributed.elastic.multiprocessing.errors.ChildFailedError: -======================================================== -train_cycling_gen.py FAILED --------------------------------------------------------- -Failures: -[1]: - time : 2023-09-21_12:38:12 - host : sc16 - rank : 55 (local_rank: 1) - exitcode : -6 (pid: 2917694) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2917694 -[2]: - time : 2023-09-21_12:38:12 - host : sc16 - rank : 56 (local_rank: 2) - exitcode : -6 (pid: 2917695) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2917695 -[3]: - time : 2023-09-21_12:38:12 - host : sc16 - rank : 57 (local_rank: 3) - exitcode : -6 (pid: 2917696) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2917696 -[4]: - time : 2023-09-21_12:38:12 - host : sc16 - rank : 58 (local_rank: 4) - exitcode : -6 (pid: 2917697) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2917697 -[5]: - time : 2023-09-21_12:38:12 - host : sc16 - rank : 59 (local_rank: 5) - exitcode : -6 (pid: 2917698) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2917698 --------------------------------------------------------- -Root Cause (first observed failure): -[0]: - time : 2023-09-21_12:38:12 - host : sc16 - rank : 54 (local_rank: 0) - exitcode : -6 (pid: 2917693) - error_file: - traceback : Signal 6 (SIGABRT) received by PID 2917693 -======================================================== diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275223.sc14.2163737.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275223.sc14.2163737.0 deleted file mode 100644 index 040bf46aca2d09c27264c109779502f2bcbc466d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzv5Bs@3$=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(s5s9g$mSMC+!Ab&dj3_a|IZ+#JNPd_(DLY z>!+vY#pmP~7mG3+2)5sUH{4Jas^`@nv(KET5qc!J#JKn&dQvirlcAa%UfSN3a_t2G DzyLwZ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275225.sc14.2163737.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275225.sc14.2163737.1 deleted file mode 100644 index e581300283f6e964e9ba4ddca7e4a690a5876dab..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxl2#h!T=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@-$PLK?Q2AlXinIXYp2uxdKdD;#{I!d?6sy z_0v=H;&bwgi$xh4{Osi~?-y2u>Up)t>@(*%gdPblF)n_Ho|MeuWT+;F7TZO6?WzFs Cu0DML diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275228.sc14.2163737.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275228.sc14.2163737.2 deleted file mode 100644 index 06d9dc5853e0eed150ac06e9521e28edc3445b10..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 132 zcmeZZfPjCKJmzxxi*lHMbGqp$#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZF$?udm4TXj;^0(S&d5@Txmto;{9GJii8=Z? R`NhSe9P;1nSQfIX008!UDdqqG diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275232.sc14.2163737.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275232.sc14.2163737.3 deleted file mode 100644 index f88073939410d3fe6dda4ee238539f53d9f6a53e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzvHn8;dubGqp$#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z*%^DQO$BPMlXinI=c}m@a|M{S#JNPd_(DLY z>!+vY#pmP~7mG4DgxOaVG%Qzz>Up)t>@#QdG`JoKE-@~Ch@O|u Dd<#C$ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275238.sc14.2163737.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275238.sc14.2163737.5 deleted file mode 100644 index e674ec3582a516a63457190a6b1cd71f875f4309..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzw86fLy;=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-62rggq#V@Tn|FkaIKN+km@B}jCCJ6k#Sxa6 Tqo0#sTrB#q`J~-7O>tHL=N~Q! diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275242.sc14.2163737.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275242.sc14.2163737.6 deleted file mode 100644 index 3d6496d920a7affb58e6216ca869f77f44d47c63..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw~D%x)K&FQA26mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWFCj@RTZeYPTCE=oN2r}z~&0DYKe1+a`A_zC2;1c8Fhv-SkEKY`MTEJ!dbotg6 E0R7EELI3~& diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275249.sc14.2163737.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275249.sc14.2163737.8 deleted file mode 100644 index 30b175c21f9fa4f4c88889202876340fd9851691..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzw~^Yyd&=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^42J^Tn=jP%{xLyoE{VW!R87uX$f-ib8&NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZxxiWLssc6FNxQ+9^IHnUTmg11aV}9Vz7UY< z`st~8@j3a$#i9%k{@0g!Ym}-&^}O0+_Lrk&FQA26mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#OhJPY!#@vPTCE=oQVp)U~>gHw8XhYx%fgr zrt7Dt=Edja7Z-~%JosO)wEIw{Dpb#_J!YRdA0zZgaEWp8L-eF%7AHeBEx2gIxAfIY E0DV?LTL1t6 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275259.sc14.2163737.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275259.sc14.2163737.11 deleted file mode 100644 index c81b730e48981f8508854fe891c1481d596549c4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzv%=>N0(=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;=AC{6FI24H}41;aXwuGF;{?DOOT77iz6&C UM?WXOxLEXI^HIC$pBKge09bP{_W%F@ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275264.sc14.2163737.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275264.sc14.2163737.12 deleted file mode 100644 index 1622a6e86d60bf0c7407ec540347aed899a2be1b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw~my>q*=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a%urXr3%zsC+!AbPNn$}a|JlH#JNPd_(DLY z>!+vY#pmP~7mG4H_+P)#Y2hzbsGe7Q%szAOMd*>>664~B=t;>ePKIhau*N2!==EFx DIu1cy diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275267.sc14.2163737.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275267.sc14.2163737.13 deleted file mode 100644 index 6911450c440a67a8cba6bbbbf69c89a87692e4a1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzv*3f^}3=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!X17qUj=HelXinI=j{%NxdL2T;#{I!d?6sy z_0v=H;&bwgi$xh8{I7SpwK!B2s^`@nv(KE4op3!8Tw+}O5IrfG#mP`j2O4b}kIk6_ E0EinxMgRZ+ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275270.sc14.2163737.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275270.sc14.2163737.14 deleted file mode 100644 index 5adc0af9062672b2ae64dd3631d531359a58ce85..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwW2K76BbGqp$#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZdF(GNAO|(~<{cp;&fhyB<_fTA33BmsafBu2 T=;!1Y7mJo$+;1oJBO(9*yF)Fj diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275274.sc14.2163737.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275274.sc14.2163737.15 deleted file mode 100644 index 802b5ea4c1d67e9a61e2f492f2156d2dda1fbb0f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxtY+LU1&FQA26mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWLcl@Srw?cPTCE=oLiMvfXx-)))MCu<>Ctg znXaFnnirpwUtBE8@Zf)awypk0Rj8gNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZNf*BLNCj%HlXinI=Yeg_U~>g{w8XhYx%fgr zrt7Dt=Edja7Z-~%JosNfZS9>!s!%^4Bf%xc#ShVwl3AP#)zsi@Q^Apc F7XZKxLBIe2 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275280.sc14.2163737.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275280.sc14.2163737.17 deleted file mode 100644 index 722e5a56c328830b94bff2acaf0642a0a7605918..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvX&9rp+=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GU=n`d^xDOH}41;ahi%*fz1_Q)e_|5=i&%U U%+b%uFD@3f`?1ST=RoCR0O8^-ng9R* diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275285.sc14.2163737.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275285.sc14.2163737.18 deleted file mode 100644 index 240ef112ac7e28c9be85a705f053cbe95047b8e7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw0t_*Vh=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-QmE9ct^zgJNxQ+9Q(Z0?Y_0&WmN=Iv7hed- zbp7idw8XhYx%fgr zrt7Dt=Edja7Z-~%D1_SwKZsnd3f1#!kJ)EVMFF@T2`(`%eu$ow%;IFIrUULaAJ+<+ F005j6J;eY3 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275975.sc12.3473404.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275975.sc12.3473404.0 deleted file mode 100644 index a70d87afcfdffe4070b2f2b132cb1839d7680142..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvXZ2Iu=o6}84Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$Oofq$#PJ0Z{86y;=CX85p1pio0cFKKNm+> VVvc@JesQtr>Q!xaJzV>y0RYdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vwe@4 E012c)-~a#s diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275983.sc12.3473404.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275983.sc12.3473404.2 deleted file mode 100644 index defc00125ca2bd9f1a7a163c249c142e22e5fec2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxdrTzQ-&FQA26mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL~>!FlnT^bC+!Ab&Z0XIa|Hyn#JNPd_(DLY z>!+vY#pmP~7mG4Hu(w~j?dw8SsGe7Q%sz8ILFkd-664~B=t;>ePKIhaaKcva^qo}z D(=kQ- diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275986.sc12.3473404.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275986.sc12.3473404.3 deleted file mode 100644 index c347673b66b2a6a8005a2fed3121dcd40a6e84e1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzv9wcqjOo6}84Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$c4b9m2yyXZ{86y;+(e#Vy*zYmLL~D7e`oP Tj($#lak1#^oH{!;<)0S;njJ7G diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275991.sc12.3473404.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275991.sc12.3473404.4 deleted file mode 100644 index 6e9f780b521f759ad424bc31690064d16d26dc20..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzv{{J!(`o6}84Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th~xM3-YQUYowOT#Ij`s31)D1%s3p!N%EcD~ zGF?ACH7`CVzqnYG;enO?1~Ip{s!%C>J;)m!-$t+HWYC5peHstE{ FE&w5?MJWIP diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275995.sc12.3473404.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275995.sc12.3473404.5 deleted file mode 100644 index b0fc9296c722bea47b4e602f9696725246f36f87..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzvvv^eqYo6}84Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th|ROIZz@o8owOT#IZIYR%oPyQ66X@-;tK(p zuAiQo7oU?~TrA4q;9>v7ePKIh~XtO<`)w~n{ D)0;%D diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275997.sc12.3473404.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695275997.sc12.3473404.6 deleted file mode 100644 index 0852b3bc5e380ca5bbc453cfe8b1f03b4876e4cc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzx#dQt!Vo6}84Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNL1OYNI9svH}41;aefPH0Glhop(V)0&&3gz Un4_PQUtBD@det2}6Ylcq0OL3?l>h($ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276002.sc12.3473404.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276002.sc12.3473404.7 deleted file mode 100644 index a101cabba3835239a5fd70feacd87bc2f43557bd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxR-dXK*QfC|)HC+!AbPFIf=U~>h8wZyqZx%fgr zrt7Dt=Edja7Z-~%D0tgnUCv~t3f1#!kJ)F=3kW?DTw+}O5IrfG#mP`j4w<%lylQ#@ DrOiM? diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276006.sc12.3473404.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276006.sc12.3473404.8 deleted file mode 100644 index 1f6d6d52033a9eac04c377cd4ac66091829522e7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw8==A^m=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;a|J}S#JNPd_(DLY z>!+vY#pmP~7mG3|c-#MUo;X7ls^`@nv(KCd5PBrI#JKn&dQvirlcAap1lro4GWP=j D;L1VO diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276009.sc12.3473404.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276009.sc12.3473404.9 deleted file mode 100644 index a2298a98b2fbb0b82770b7c6447eb49b4e468ad9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzxVQgi(E&FQA26mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#Al|(FFB~WH}41;aUPlkF;{?7OOT77iz6&C TM?WXOxL9;{&IG%c$2|lASEVm& diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276013.sc12.3473404.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276013.sc12.3473404.10 deleted file mode 100644 index 62cba7a56b270b1e529cb229cc1c0691e19e41da..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw4mrD8l&FQA26mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq&fXspbFGnC+!AbPM-x3a|J}T#JNPd_(DLY z>!+vY#pmP~7mG4D*xP4(K6PCcs^`@nv(KD25PBrI#JKn&dQvirlcAazqHLx3-x>k{ D9$`Ur diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276016.sc12.3473404.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276016.sc12.3473404.11 deleted file mode 100644 index e5846474fc8bf8e67ac5642a5fa38a791d0eda6a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw~op1K%o6}84Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th~$k= diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276019.sc12.3473404.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276019.sc12.3473404.12 deleted file mode 100644 index e7016000a765230de8095f3b1925c69de60e1e21..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwm{CWB3o6}84Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNR+bX1v#j>H}41;ajpn|1vXcJOG}W8pNk_b UF-JcqzqnY`?njbcz{@wH0H<0oEdT%j diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276024.sc12.3473404.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276024.sc12.3473404.13 deleted file mode 100644 index 7361d3ba11b89a9d159fa6b008333c8c1973908d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw0XY%~}=5*6hiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!V+<%Lj`KClXinI=e8pda|Oh;#JNPd_(DLY z>!+vY#pmP~7mG4D*xPFaFQ`_9>Up)t>@%nDQMeunE-@~Ch@OdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vgyZB3K^3UEPTCE=oTYCe<_bt?iF1i^@r8g) z*H2H)i_gg~E*522U|=sYIYCkts^`@nv(KDw5PBrI#JKn&dQvirlcAaz#BApX-G~JM DsG>rk diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276030.sc12.3473404.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276030.sc12.3473404.15 deleted file mode 100644 index 66e8dae95f2f35527163949b317f5950219c8efc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzxFzP{l9H>aD9QoKn;iJ5tNu4SotC00g3dR#gssd>fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjw5%UwFb#hR1Z{86y;uQB<2sT%MTT76OpNk_b UF-JcqzqnX5wxrzddVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL_TGqk_yyZC+!AbPTA8Ca|I-|#JNPd_(DLY z>!+vY#pmP~7mG4D7}=Y?ow`yLs^`@nv(KCp5PBrI#JKn&dQvirlcAapsM$t@ObiDA D{e?dU diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276038.sc12.3473404.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276038.sc12.3473404.17 deleted file mode 100644 index 1fba33e94f25bdff4671abe139bc2543500671ad..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzwq-!_f$yVFfaDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T2w(o8KPpgjowOT#Ii+oCtg znXaFnnirpwUtBE8aKPC9)B3~4s!%<{9 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276041.sc12.3473404.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276041.sc12.3473404.18 deleted file mode 100644 index d8368e46d2dd88b19e9ef6e4b72629b7e23a90f7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzxln&rdv-RY*I6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vgjY!^Uk+;S%{xLyoIkpJ!R89^XbE!hb8&dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWOm5;W)-NpPTCE=oY%ty!R87`Yl(A-a`AdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#Hj7rE)}S`PTCE=oTgzwZyqZx%fgr zrt7Dt=Edja7Z-~%BuLv&nvoKt3f1#!kJ)F=I|w}zTw+}O5IrfG#mP`j2V!iGGlkj# E000R<=l}o! diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276769.sc13.1777137.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276769.sc13.1777137.4 deleted file mode 100644 index 73d7945622111aa025f468c9b92ac79f00c85477..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxVdK9|jyVFfaDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNdM%W3ss=zI%zlfa=N>Pfz1_=(-P+r<>Ctg znXaFnnirpwUtBE8z#wV=%BMbD6{_de9<$G!ClGohxWu^lA$n3Wi<6<68e(jNJ#yy& E02~HDs{jB1 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276772.sc13.1777137.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276772.sc13.1777137.5 deleted file mode 100644 index 2d085310e12e8335f23a35cff7b7733c0218e00e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzw~f6BJ=yVFfaDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$c;PSv*n=X-n=7Z#95ib4mMYSPfL)CpNk_b UF-JcqzqnX5cT=X_e-D!c0F^*5ng9R* diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276776.sc13.1777137.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276776.sc13.1777137.6 deleted file mode 100644 index 5ccba276188ea821e69f947be380f4e8b32ddcb4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxdtyJ6f-RY*I6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vB(+$sMFncElXinI=T&`mu(<;ATH;)yTznxQ z)AiF+^Wt;zi;G1W6m;!%6!I!mp?Y5JG5gHvZ2;FJ!6nAU57Cp7S)2^jw7}n%>*bd+ E0Qf;Zi2wiq diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276780.sc13.1777137.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276780.sc13.1777137.7 deleted file mode 100644 index bc1933dc35a49cf825248efa6ceafee21b28de2e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzwKzazN&yVFfaDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNRPxkZWXAxPTCE=oc5VQU~>f&w8XhYx%fgr zrt7Dt=Edja7Z-~%DCpWJHcF_fLiN1bWA>SIK0=QKmlzj6L{Ca)aWYg>gQe}WST{QW DLeoAl diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276783.sc13.1777137.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276783.sc13.1777137.8 deleted file mode 100644 index 28c8b764e8b01b12b61c6782f3a48f21b110cbd3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvrBp%-V-RY*I6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vB)G_aq8!xRn|FkaIPWh$0ybBGUrUgSpNk_b UF-JcqzqnZRW=f--Y^lHw04vxomjD0& diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276787.sc13.1777137.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276787.sc13.1777137.9 deleted file mode 100644 index f0875e7e0af7b8ebcdfb50f0493c615a2df8d99b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxF`TBIvcc+_fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwk*QlF-BqCGI%zlfa{eiK1~ykfQA?all#4F} zWV(KOYF>O!esQrVLxYvQz1oils!%C>J;)m!-$t+HWY0|T0UB_hz E0O>?RQvd(} diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276791.sc13.1777137.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276791.sc13.1777137.10 deleted file mode 100644 index 1438796be89b77d7246db1b97f6be4044ba27879..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw~d318`cc+_fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwk*Y4^Y!#@vPTCE=oT>h&z~%}lX^C@*a`AdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq>t;)L^-IrH}41;asHnIF;_r9OOT77iz6&C UM?WXOxL9;?_aeIuVh{EJ0BI{QMF0Q* diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276798.sc13.1777137.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276798.sc13.1777137.12 deleted file mode 100644 index 087afcb87f8c1a67027ea140aed26a82a2f44fc6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzv{&ziaayVFfaDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNSsl_1Qn>cPTCE=oZHxDfz1_A))MCu<>Ctg znXaFnnirpwUtBE8aKO!8!SX?lDpb#_J!YRdli1;UB)G)5_#t{yGK-U;nih!ITJCMS F006RDK#u?b diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276801.sc13.1777137.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276801.sc13.1777137.13 deleted file mode 100644 index 3831e675cda7e8450f766baa8cf97182f6668b87..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzv9Dkwkj-RY*I6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vr2BhUtqRm!C+!Ab&P%@`<_f52iF1i^@r8g) z*H2H)i_gg~E*52IaIt5t6FH&^)$?kP*=J7mKX5$~Tw+}O5IrfG#mP`j3wUiS4$V;q E0Bcf0F8}}l diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276804.sc13.1777137.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276804.sc13.1777137.14 deleted file mode 100644 index 3956faaba01ea53b67f1df7f1b56c9f04ed3f556..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzv*s%akl?sU^piZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^3K0lP7Z4B%{xLyoc|X>%oPyS66E6N;s{I3 T(a*^*E*6!qnPI2fd`=Jm)xIrP diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276809.sc13.1777137.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276809.sc13.1777137.15 deleted file mode 100644 index a7d1ba887a0d9020078ce93700b3f9889b46b891..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzvvEAu$?-RY*I6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vgp-jWP6cYNlXinI=S>bzu(<-NTH;)yTznxQ z)AiF+^Wt;zi;G1W9@yCb&3Toj3f1#!kJ)EVcTTt-2`(`%eu$ow%;IF2CMnxBAx|y? E04Lo*X8-^I diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276812.sc13.1777137.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276812.sc13.1777137.16 deleted file mode 100644 index 6a10508a050dcde2a495a71a09c6448331086b4e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzwqIVFDhyVFfaDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$f@6D1u9T;owOT#IS)rmfXx+9(-P+r<>Ctg znXaFnnirpwUtBE8u)xCp^TH43RH1ra?J@hzX%++5Bf%xc#ShVwl3AP#)ufn+a diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276819.sc13.1777137.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276819.sc13.1777137.18 deleted file mode 100644 index 798956e2ec95d026400ebf79e59eacf3578afe08..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxV+wttkcc+_fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwksYTW*Q!9xb<%F|<+Rs(4mMXnT}zxxl#4F} zWV(KOYF>O!esQrVgMyL0Z0Lu_s!%B<^TWy diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276822.sc13.1777137.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695276822.sc13.1777137.19 deleted file mode 100644 index 111c084925b95f5f368ce3d213687601d949ea7a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzvvY1(u2yVFfaDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th=l2NV-={mPTCE=oO$;k<_c(NiF1i^@r8g) z*H2H)i_gg~E*51-(6ZnAx+zE%s^`@nv(KED5qc!J#JKn&dQvirlVO@dY>lr3i2wje C)j~1= diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277599.sc12.4183255.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277599.sc12.4183255.0 deleted file mode 100644 index d14004f38543e266d1ae2c2693f725e2a122455d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvb82)Pi;dIkciZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vhmyd4mqf~H}41;ahhC#m@6QxCCJ6k#Sxa6 Tqo0#sTr9e{ThA^?%;6^hWD+lq diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277604.sc12.4183255.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277604.sc12.4183255.1 deleted file mode 100644 index 0fa8f4a3fb3b7780a75d42124a903e8d9dca7879..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzvvF4FA$;dIkciZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@>KRguL{&$C+!Ab&b${8a|JZD#JNPd_(DLY z>!+vY#pmP~7mG49$k?x{U;j%Ls^`@nv(KD65qc!J#JKn&dQvirlcAapB-)<)`+PG1 D-5x`j diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277608.sc12.4183255.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277608.sc12.4183255.2 deleted file mode 100644 index 4d1a10823e5a6bb80ec9099293b8111423731837..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw8Rd96uaJuOz#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZNxG;ipaM15NxQ+9)9ydSTmdaDaV}9Vz7UY< z`st~8@j3a$#i9%jJoXAV7s#qY^}O0+_L=hdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL@wdZKNYCCPTCE=oB@n4z~%~QYl(A-a`AdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#J;@goC?%jC+!AbP8G&WU~>g@w8XhYx%fgr zrt7Dt=Edja7Z-~%ByihH7j&Oeh3a{=$LurbG=v@rE-@~Ch@OdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vBw22=w;a^mn|FkaIQz{z!R88xY6)`jb8&dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWKPPwG8L$~PTCE=oJt{GU~>iZw8XhYx%fgr zrt7Dt=Edja7Z-~%G;rH{2J|SI2SSeomlzj6L{Ca)aWYJkz3qyE9|-^- C4M9=> diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277632.sc12.4183255.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277632.sc12.4183255.9 deleted file mode 100644 index c66473406a5b8f8c4e80ae75b00e9a896219b5f3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwKOt+cv!|A4@6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq%dvbUpc6`H}41;ajv;&3pQ6kOiPf9pNk_b UF-JcqzqnX5H1w~n$-@)L09Au8v;Y7A diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277637.sc12.4183255.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277637.sc12.4183255.10 deleted file mode 100644 index 261a1970a3617963d4017014339133c6a7e2916c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzu=q$W@N;dIkciZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GF#(ahzitPC+!Ab&X1ufU~>iZwZyqZx%fgr zrt7Dt=Edja7Z-~%FmT(SykqCD3f1#!kJ)F=A7OAk5?o?j{181UnZ?O4O-{BP37ang E06lv@hX4Qo diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277641.sc12.4183255.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277641.sc12.4183255.11 deleted file mode 100644 index d8bf3abfac2f7acbbc9c1f9c7cf90192ec4ecd3d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzwS1^Z6=;dIkciZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(vX}XpaM15NxQ+9bNUQFu(<*TTH;)yTznxQ z)AiF+^Wt;zi;G1W4zSq^T>bV@6{_de9<$G!jx*tUB)G)5_#t{yGK-U;njYBND#m3` F0{}QHKnwr? diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277644.sc12.4183255.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277644.sc12.4183255.12 deleted file mode 100644 index 65a1fcd0b664c46f9826abfca88a6ef926abc433..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvvyTLN~hto|*Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$SvLPeR5E9Z{86y;=JR|1~ykfTuYFPpNk_b UF-JcqzqnX**9!r=q}gv+0Y6tR=>Px# diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277648.sc12.4183255.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277648.sc12.4183255.13 deleted file mode 100644 index cf809ea3135577598819b470148aa6996cf2c6d4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzv9jntd+!|A4@6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#H2mYO$BPMlXinIr^#%6u(<+;TH;)yTznxQ z)AiF+^Wt;zi;G1W4oKRUa3t?jh3a{=$Lurb0fZh2E-@~Ch@OCtg znXaFnnirpwUtBE8z#wAZB)p?Y6{_de9<$G!5+!gw5?o?j{181UnZ?O4O-8oI5+k1i E081M_o&W#< diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277655.sc12.4183255.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277655.sc12.4183255.15 deleted file mode 100644 index 0fae31448eea83aeb67896ee172a879dbb2b0b7c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwSZoD=1hto|*Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNS`{>S2?J;H}41;ao*i@8*HwCgq9!|KNm+> UVvc@JesQtrt`}T(pEZ-?0dHw9rvLx| diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277659.sc12.4183255.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695277659.sc12.4183255.16 deleted file mode 100644 index 2d94dfba14ec149c2c1acee620cc92c045c98161..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxBt^POdhto|*Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th~WRmz*tM1OO%T* z1Z28?dTL&LPJVH*D1!sHeWB5sv#L-%ulAUI=FHNC>yh9RdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#IJVlKRKwmH}41;aTdwt<8 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278445.sc20.758883.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278445.sc20.758883.0 deleted file mode 100644 index 83e43f53e1f5248bcd76ac00713aa0c02a180155..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzv<{#OdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq{ZfalnT^bC+!Ab&Z5K`u(<-JTH;)yTznxQ z)AiF+^Wt;zi;G1W61eOe&*?8xh3a{=$Lurbb%Y)XE-@~Ch@OdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq#(h|LIrBBlXinIXZ8KxU~>h`w8XhYx%fgr zrt7Dt=Edja7Z-~%FmT#`Ie5}p6{_de9<$G!ClGohxWu^lA$n3Wi<6<660~d?8ji0A E09}4UU;qFB diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278450.sc20.758883.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278450.sc20.758883.2 deleted file mode 100644 index 07ac8c2a98b50d5197f0af43a1edd55bd911187d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwC{ag|A)9I$86mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq)({vupHFfn|FkaI0M$M1e+@$r6tJ4&&3gz Un4_PQUtBEeCjQj+aOuVc0AWurH~;_u diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278455.sc20.758883.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278455.sc20.758883.3 deleted file mode 100644 index 1a5c1e2348fc87b80c9d87bf4c6e06a45f326a59..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzv{8C;0{>2%XkiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GWF9j4;84nPTCE=oV98f!R88>Yl(A-a`A2%XkiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^5#hJc@?O+PTCE=oD()|1e+^hp(V~G%EcD~ zGF?ACH7`CVzqnYG;Q_n-`huP(s!%Ppz diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278461.sc20.758883.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278461.sc20.758883.5 deleted file mode 100644 index 6a67c1475d0386bd052e1f951ed782866532b1a2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvL7Nx}hbh_y%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z+5L9iM>(juH}41;asDn$1)D1%ttH6C&&3gz Un4_PQUtBC28v4{$SXKE702&r9i~s-t diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278466.sc20.758883.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278466.sc20.758883.6 deleted file mode 100644 index 6882d44bbb102ccee6455c0acea111aab5ba9738..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw$i?t^Fbh_y%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zxv`yhsS4CwC+!Ab&J8VXU~>g5wZyqZx%fgr zrt7Dt=Edja7Z-~%FmT$x_dWSY6{_de9<$G!hOKZt5?o?j{181UnZ?OaO$~asYZLT+ E0WXU`h5!Hn diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278469.sc20.758883.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278469.sc20.758883.7 deleted file mode 100644 index 1e4989ece3547ef4334cae9523af0788c6d8b341..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzvXZB0r1>2%XkiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-V!n8(lnT^bC+!AbPJi|^u(<+OTH;)yTznxQ z)AiF+^Wt;zi;G1W64>o`{oEa)3f1#!kJ)F=R|q{4Tw+}O5IrfG#mP`j3c9w@!hf{@ DIW9iI diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278472.sc20.758883.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278472.sc20.758883.8 deleted file mode 100644 index c1637fb0d5358ca29c278e70fae065fbaa165b20..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwiX^ABLbh_y%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZseQ9{gB;Y{n|FkaI1MjC%oUK)66E6N;s{I3 T(a*^*E*9<4dt$p#w^kYe&%Z5N diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278476.sc20.758883.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278476.sc20.758883.9 deleted file mode 100644 index bd8c73864d637b45fc5cb715188d33c93262c302..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzvvone{$)9I$86mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#5SW+PX%hOlXinI=PhY#u(<-(TH;)yTznxQ z)AiF+^Wt;zi;G1W5;*MVF8r9T3f1#!kJ)EV9~rnF2`(`%eu$ow%;IFIrUyE<{PSBl E0RE#sJOBUy diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278480.sc20.758883.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278480.sc20.758883.10 deleted file mode 100644 index 44868675702fe4f26fc7d519d67955c8d9bcb88f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw0Bnzhebh_y%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zx#0b)S_NvZlXinIr~4iuu(<*@TH;)yTznxQ z)AiF+^Wt;zi;G1W8rbanY_qPbLiN1bWA>SIBSMb^mlzj6L{Ca)aWYg>f`)CJ6oVlE D_Mkoc diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278483.sc20.758883.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278483.sc20.758883.11 deleted file mode 100644 index a1a5cc8f3e65000470444b9e0f99c34f312ca91a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzxVO+AwG)9I$86mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vBelCu% U#2o#c{NiHK#oZTd<9oxI0hm}YE&u=k diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278487.sc20.758883.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278487.sc20.758883.12 deleted file mode 100644 index 6e6eb7f0566981897a2bb99e99b9dc701a727fee..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxd>%U3;>2%XkiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!raK|rUEtBNxQ+9GvzSETmf4xaV}9Vz7UY< z`st~8@j3a$#i9%h9QM39Q)N}5dS2}@`^@2%XkiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;?)}*rvf$CNxQ+9Q=jV`*jxcSEpaYUF1`?u z>H6uZdGR^<#l@lw3~ctdlzUdFLiN1bWA>Tz2||wqmlzj6L{Ca)aWYg>gPN_`Keapn D3UfZ& diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278493.sc20.758883.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278493.sc20.758883.14 deleted file mode 100644 index acc36f9d5b90b15a9e07f92d9ef796497786b253..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwC{i{v?>2%XkiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-64{uTCiJv;?{Mxj4cS UbM$lai;G1k`|h)~x8XDe08IWZlmGw# diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278498.sc20.758883.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278498.sc20.758883.15 deleted file mode 100644 index bf638679d3298c41c56137d0e5d9e318a2b78e86..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzwK30j=-)9I$86mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWb3!J8Y)n8owOT#IaSU;%oVWL66X@-;tK(p zuAiQo7oU?~TrA4)fYrWBp*2nws^`@nv(KFK5qc!J#JKn&dQvirlVO_lY=u@fu>k-f Ce?eCO diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278501.sc20.758883.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278501.sc20.758883.16 deleted file mode 100644 index 7cf8728e09a3bcba1bcba120a6ca1999fda9a85f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzvvKAWHU)9I$86mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq$(#@N(E}JlXinIXQ@>I*jxbzEpaYUF1`?u z>H6uZdGR^<#l@lw4UG1o3njLyLiN1bWA>SIKSGZLmlzj6L{Ca)aWYiXfneMDlWnvB DP{BV! diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278504.sc20.758883.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278504.sc20.758883.17 deleted file mode 100644 index 156717289aeab9502938524618d25414e8689cbe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzw$Z2%XkiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-lJ_*rLk?>0%{xLyoSP(M!R89cYYB4kb8&2%XkiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a$V%1tP0dzC+!Ab&U6nuu(<+`TH;)yTznxQ z)AiF+^Wt;zi;G2pO6~n(wyCH>^}O0+_L=h_LXQNO7#BZ8PfBKSGE7sat=V+Z835d# BJ$wKF diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278512.sc20.758883.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695278512.sc20.758883.19 deleted file mode 100644 index 20f7790b1d94eafc7cc5e48e76c5cd046a93fc72..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxBO=ivc>2%XkiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-lH%JwM+IuGlXinIr`tO=u(<+GTH;)yTznxQ z)AiF+^Wt;zi;G1W7W}gFSatM=Dpb#_J!YRd?;-R^aEWp8L-eF%7AHeB9hhXxalCsU E07*STdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v!+vY#pmP~7mG3|{I$F7Tx_HY)$?kP*=NoR2t5*9VqE+XJt>*R$xuxVlWjYHJdy$c D!)8JI diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279213.sc10.1505340.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279213.sc10.1505340.2 deleted file mode 100644 index 464d18c50bfc5a3eb474dc9bad4d73dde2d120a6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzv{{_>Ol<#f|giZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;&<-eLKUdFPTCE=oSS3)!R89MXo+))a`A|X<#f|giZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@^4>~stVLxC+!Ab&Z2n`a|K+r#JNPd_(DLY z>!+vY#pmP~7mG3^Fxp!@syU{a=Pg##hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZF%vBLtO7OHNxQ+9bIEszxdLul;#{I!d?6sy z_0v=H;&bwgi$xg{80>2Tv<+3EdS2}@`^+iw1FlDcON@&jq9-M@I2o$xL5;0|u(SgJ Df8RWt diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279227.sc10.1505340.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279227.sc10.1505340.6 deleted file mode 100644 index 2aab0a06a60bd48fe16171f1035d7912bc4b4c6b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzw`Xr7S!<#f|giZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@{N79wj9*ln|FkaIMdr9<_aij33BmsafBu2 T=;!1Y7mNCt&9yz{@nI(b%Dye& diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279231.sc10.1505340.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279231.sc10.1505340.7 deleted file mode 100644 index 2a9a3f3af6a9302379846a9ffc019f3eda1419d5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzvT8NZPK<#f|giZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a@uA8G!>}1PTCE=oHOP?%oT9g66X@-;tK(p zuAiQo7oU?~TrA4az-WK-Aj3mdsGe7Q%sz9P%!TWb;1c8Fhv-SkEKY`MT99L_QGDYN E0CUAbRR910 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279234.sc10.1505340.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279234.sc10.1505340.8 deleted file mode 100644 index 96a42acf53c14bf9225b364a8d0459a4123842db..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzvPJGNWlm(xv0Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$clHBdMZ$JowOT#IWODn0h=q}p(V~G%EcD~ zGF?ACH7`CVzqnYGfq~K9>M!GERj8gdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vBv4beKn`l|%{xLyoM!&{U~>hOwFJ5Nxj4cS UbM$lai;G2fy_jSB=sDvN00p2fI{*Lx diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279242.sc10.1505340.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279242.sc10.1505340.10 deleted file mode 100644 index ef846efab0515a381de0fd159f20bf5c46bfdd6b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw)tm;gLwZyqZx%fgr zrt7Dt=Edja7Z-~%G%(uFn{|Oj6{_de9<$G!hY)%sxWu^lA$n3Wi<6<66vAzJ>y8Tn E07`W}ng9R* diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279245.sc10.1505340.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279245.sc10.1505340.11 deleted file mode 100644 index e6db4c024647226cb9917862b1203de2630b8c0a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxpzQ|Jk<#f|giZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GPlO=lnT^bC+!Ab&h6gWU~>h$w8XhYx%fgr zrt7Dt=Edja7Z-~%9AL2jzGsEMDpb#_J!YRdd41q|B)G)5_#t{yGK-U8ngVT`m$L8! E07D}`I{*Lx diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279248.sc10.1505340.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279248.sc10.1505340.12 deleted file mode 100644 index 996b06014b2ecf9766ca11105c46ee5eb6a0f2ef..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzxVoh7OA%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWZ}FG`EpQmZ{86y;+!EV1vXbeMN5#2pNk_b UF-JcqzqnZRl-gw5?%a9&0Ql!EH2?qr diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279252.sc10.1505340.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279252.sc10.1505340.13 deleted file mode 100644 index 89cc23b986e32607049df83f6098d97dcea0a693..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzvfY%*8<<#f|giZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(v$zjSOsdXlXinI=coS=a|OJ$#JNPd_(DLY z>!+vY#pmP~7mG4HV6wM7|EEqBs^`@nv(KC*3>ILw0QE?4iE;5m^rU1KCqp$kgxNYA HEiwZDs7pW# diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279255.sc10.1505340.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279255.sc10.1505340.14 deleted file mode 100644 index fc539ed40f0cc226e22900e4253d089ecef34e7b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzvPb@-?H%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v*R59-s^`@nv(KCf``~&cxWu^lA$n3Wi<4oR!fcm3x*rPw D0qsG> diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279258.sc10.1505340.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279258.sc10.1505340.15 deleted file mode 100644 index 1f294ce7b492bc6584a062897c82079b586bfa45..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzv9?%J&O%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vgtz!prySJWn|FkaIQ_S70h=qJswK$9&&3gz Un4_PQUtBEO!rx~5KWgqK05#h#%>V!Z diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279263.sc10.1505340.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279263.sc10.1505340.16 deleted file mode 100644 index 6ec550057696ab53f30b5f4ce3e9165ef9b463c9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw?Y`Cue%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#4cM`UIl8dlXinI=Uv?!U~>h0wZyqZx%fgr zrt7Dt=Edja7Z-~%G%(nE#{W613f1#!kJ)EVPd&IE2`(`%eu$ow%;IFIrU#+6n?C!; F0|01MK#%|c diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279266.sc10.1505340.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279266.sc10.1505340.17 deleted file mode 100644 index 27813e4f5e00511b800f2780d5a750208ca1881f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxF&R?nV%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#4L;PlnT^bC+!Ab&N}iF1i^@r8g) z*H2H)i_gg~E*52I_-(g5^q!I`RL`qDW}i8^pTqS?aEWp8L-eF%7AHeBCAizFJ&{%f E0G+WwG5`Po diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279269.sc10.1505340.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279269.sc10.1505340.18 deleted file mode 100644 index d6baa2d1c3006a30c40ee10b815098e1166f7c0e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvPIuWP&%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL~*~oxE$2nn|FkaI9YAu!R896X$f-ib8&dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWR1@1(<)GNowOT#Iirjkz~&10Yl(A-a`AR diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279943.sc18.3937612.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279943.sc18.3937612.0 deleted file mode 100644 index f27fa1d7e91e98b07943d19354419b5cdcb43f06..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw4u6c0rm(xv0Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNW;xj-&COHI%zlfa-RJKF;^f!OPouTi!TIZ zx_)|UUVKh|aj_`F0S5btdU}Ucp?Y5JG5gG^`x~xDf=i5xAEGBEvp5;5$-%?+VqlXw E09^D$lK=n! diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279946.sc18.3937612.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279946.sc18.3937612.1 deleted file mode 100644 index b9d08a432a83b7c16656627abd3b17dd9e0ff986..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvP-P(WYm(xv0Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$kuOOf^txEZ{86y;=E=&0c@^-x|SdpKNm+> UVvc@JesQsAk6w!{#~M3L0F1vblmGw# diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279950.sc18.3937612.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279950.sc18.3937612.2 deleted file mode 100644 index 635b79adf58c6850d243a6eabbe08315095550ce..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxBr>wdB%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWVXtRyDCs~owOT#IRl?U%oPaK66X@-;tK(p zuAiQo7oU?~TrA3v@ZU~z)1*38sGe7Q%sz8ILFkd-664~B=t;>ePKIiF;9+Z6v*0fP DXvsu( diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279954.sc18.3937612.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279954.sc18.3937612.3 deleted file mode 100644 index c42f79bb335a952571f702403179b3b0c8d1bbad..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzvTZ*RQv%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#P2{$lnT^bC+!Ab&W~D6U~>h6w8XhYx%fgr zrt7Dt=Edja7Z-~%Jost1y*=)=Dpb#_J!YRdJ+$F^B)G)5_#t{yGK-U;niAY?+2)9= F001=9L6ZOg diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279956.sc18.3937612.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279956.sc18.3937612.4 deleted file mode 100644 index febab17fa85bc9cdcd161dbe99095b8efe31b419..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWSRDbZ7NW6owOT#Ii(DP!R87CYl(A-a`AFFt E037Q=_5c6? diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279964.sc18.3937612.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279964.sc18.3937612.6 deleted file mode 100644 index bb72332bba90005c21a220c026a430c8e8996642..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzwie$>DI%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq}n$H6uZdGR^<#l@lw4ZrLXW4TLJp?Y5JG5gH<2BAlSON@&jq9-M@I2o!*!O_;$`&=^s D$SpvI diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279967.sc18.3937612.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279967.sc18.3937612.7 deleted file mode 100644 index 2b4ebe85da39c70e25558b7cd0d2d65bb5e925eb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwqocrMVFQ=Q1QoKn;iJ5tNu4SotC00g3dR#gssd>fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwk#J4pb8=8~Z{86y;=C^K5Nxi1rj{TVKNm+> UVvc@JesQs=-H#GmA)$Sr0jP2>%>V!Z diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279971.sc18.3937612.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279971.sc18.3937612.8 deleted file mode 100644 index 649d05ae87b99c6434508612172b18e99504a5d0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw~a{qVZm(xv0Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$P9;P>s6rUI%zlfawZ;zm@5#fCC(+v#TNoH zT|YfFFFq%~xLA~70h9gng(4SKp?Y5JG5gGU9HB>oON@&jq9-M@I2oqN(Du&-Payz6 Cl|@zn diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279975.sc18.3937612.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279975.sc18.3937612.9 deleted file mode 100644 index f6ffcff25c15635856c490f9aa34a47f7c4b4bc9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzwS9DI25m(xv0Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$ai7>_bO0xowOT#IoDb}0-GxkrX|iL%EcD~ zGF?ACH7`CVzqnYG!GY1f=i_H7Rj8gz6qjFGlZ{86y;ymX#8EmeAmX;tFKNm+> VVvc@JesQtr?3{JBO5wLZ0{~PIFPZ=V diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279982.sc18.3937612.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279982.sc18.3937612.11 deleted file mode 100644 index a348476937cd00cd789edda0e56a9b81bc1da2a9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw0uitR{m(xv0Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th?LFCQWdDVPTCE=oLAL1g3T2O*AnLv<>Ctg znXaFnnirpwUtBE8knq<|Tc`DbDpb#_J!YRdjWytUB)G)5_#t{yGK-U;niR}!1+E{P F4gei7LBIe2 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279985.sc18.3937612.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279985.sc18.3937612.12 deleted file mode 100644 index 16840a34db1a4deba716ad36c4dbebfc3f20b821..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw)U0r|Ym(xv0Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Ti1<==6&0wtPTCE=oNweBz~%}>Xo+))a`Aid{P~u3f1#!kJ)F=GI_Wj2`(`%eu$ow%;IFICI>xRf3}^E E0m@B4G5`Po diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279988.sc18.3937612.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279988.sc18.3937612.13 deleted file mode 100644 index 31b807474043a204203193c7aa59e1bc6110642b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzxFZPvg0%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vMDF@@RynA-H}41;ai01MF;_raOOT77iz6&C TM?WXOxLEYZgT=OHmDjTYO7bud diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279993.sc18.3937612.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279993.sc18.3937612.14 deleted file mode 100644 index 163a115f77b2c7eac447bc1f3afa1b8e0606a7fc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzv%Uh%*8%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWPM<2mI~BdC+!AbPVSxnu(<+}TH;)yTznxQ z)AiF+^Wt;zi;G1W99Zoav}bgxLiN1bWA>S|6ro3gON@&jq9-M@I2o$xfV%DM^Y@wn D+UG&k diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279996.sc18.3937612.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695279996.sc18.3937612.15 deleted file mode 100644 index 9067dee7fd4b4509bca683bcaeb9115c203a8d7d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxdNyy#*<#f|giZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-B5W=&MFncElXinIr`;5YxdKsI;#{I!d?6sy z_0v=H;&bwgi$xh8Fxk&*R$xuxSX0}ekH8KGB C5if%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWX1icC2~-6Z{86y;#_y_9N1g|9W6mFelCu% U#2o#c{NiHK7XE3r?^M!70hmTF?EnA( diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280003.sc18.3937612.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280003.sc18.3937612.17 deleted file mode 100644 index 4f1b0f91353191a20ad97bf6831857b80b6259d4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzwyaz1dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vaa|NQc#JNPd_(DLY z>!+vY#pmP~7mG49FxubPq;X6Ys^`@nv(KCt5PBrI#JKn&dQvirlcAazENyw&Id%X5 DOR7PN diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280006.sc18.3937612.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280006.sc18.3937612.18 deleted file mode 100644 index abac888facdd17ec5b94da3e7390b7d7f562c8a2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw)irxS4m(xv0Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Ti2Wsh0~M&bPTCE=oPoC?<_g4UiF1i^@r8g) z*H2H)i_gg~E*52Y@Xzj_?@V4*sGe7Q%sz8&L+Fv<664~B=t;>ePKIhaU}k%yn8O(W Dy0JtF diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280009.sc18.3937612.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280009.sc18.3937612.19 deleted file mode 100644 index f347621f0bb33178089389dbdd9f93d89f022a37..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwKwaIz(%ju@06mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vBwoxL1#oQ! E0HeA<#sB~S diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280682.sc10.1562555.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280682.sc10.1562555.1 deleted file mode 100644 index 6c62314224cb662fcdc2c48e6bd6c419e03e9ab4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzwC`p@h7?R3*oiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-qRsfbTm@>blXinIr{`6OxdL%o;#{I!d?6sy z_0v=H;&bwgi$xh6nCt@-!_rludS2}@`^?R3*oiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;@54^qyjb9NxQ+9Q$4X1Y_34OmN=Iv7hed- zbp7NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIo0PdPX%hOlXinIr_-81u(<*WTH;)yTznxQ z)AiF+^Wt;zi;G1W8vfZ`T`F}$6{_de9<$G!D-e1lxWu^lA$n3Wi<6<67MR%X(&UN< E0GH80DgXcg diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280696.sc10.1562555.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280696.sc10.1562555.5 deleted file mode 100644 index c578d9c94694ff0e73ea806dabe1e0921a4ea4e7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzxxzhUkB?R3*oiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vfZ|3f*jP`n|FkaI4?P|gUuDt*AnF7=i&%U U%+b%uFD@3n=-O_Za$|lO01ON+egFUf diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280700.sc10.1562555.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280700.sc10.1562555.6 deleted file mode 100644 index 075312964c11ef5b62beb903da344eafbe7b3c92..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxlI&IMZ+v%pG6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vB!w}_M+IuGlXinIC!@F#*j#}`EpaYUF1`?u z>H6uZdGR^<#l@lw5B}I4h&||}3f1#!kJ)F=7K9!NE-@~Ch@OdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vhMw8XhYx%fgr zrt7Dt=Edja7Z-~%EckEdXnV(86{_de9<$G!=MZ`%xWu^lA$n3Wi<6<64k+4Ap2>3# E09N-w2mk;8 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280707.sc10.1562555.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280707.sc10.1562555.8 deleted file mode 100644 index 0972361c85788955b98d4ad7dfc502687b263a5a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzxlzkOiBZ>O7%QoKn;iJ5tNu4SotC00g3dR#gssd>fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwk*T*W&dEW|y?IB-h;!bTgJ5$7473Ef__;X3 V5_9x(@{5Z_{mi;;Yi@6i1OUm7Fwp=2 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280711.sc10.1562555.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280711.sc10.1562555.9 deleted file mode 100644 index 178a49c5c22efb0cb90213d0b30229d5d84c2ca1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzu=ta&`~x6@5WDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$S)D|L=~vHPTCE=oJ)Ctg znXaFnnirpwUtBE8aDc&H{7B{nRj8gdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vB-{J3yb9D@C+!Ab&Zboma|Keg#JNPd_(DLY z>!+vY#pmP~7mG3^{I!!&dBmd%)$?kP*=Npo2t5*9VqE+XJt>*R$xuxP6l~M^KhFgK D UVvc@JesQs=cx<0-BxlJ?0H>WVga7~l diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280722.sc10.1562555.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280722.sc10.1562555.12 deleted file mode 100644 index 8911109a68d6d0b0000e17e3b01b5e9fa995e7a5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxF`O-1vx6@5WDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNY@#Iy(&<1owOT#IVJPDz~&01YKe1+a`A diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280725.sc10.1562555.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280725.sc10.1562555.13 deleted file mode 100644 index 14a5eeb33035bc95afc2dd2c11f18c4d8723fde1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxFTaz>Ox6@5WDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNYuXBd@4|LowOT#IrEtFz~&01X^C@*a`AdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vgptiTQ3YzQlXinI=lz8)U~>i1wZyqZx%fgr zrt7Dt=Edja7Z-~%DEziF7q#F~h3a{=$Lup_;Uc&m2`(`%eu$ow%;IFIrURU|5ubEk F0{{%#KVAR; diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280736.sc10.1562555.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280736.sc10.1562555.16 deleted file mode 100644 index 3aabc5519307ed8eb6508d74658462ed6e396052..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzx#f2=s;x6@5WDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNKG%Rp$gPoC+!Ab&S|wuU~>gBw8XhYx%fgr zrt7Dt=Edja7Z-~%9Qb2*JH0ql6{_de9<$G!PZ4?~xWu^lA$n3Wi<6<67I51>x!TO7%QoKn;iJ5tNu4SotC00g3dR#gssd>fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwk*Ujq{p6tL-n=7Z#M#b&8*HwCv6dhgKNm+> UVvc@JesQtrWZw$gCEs(I0f@#gHvj+t diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280743.sc10.1562555.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695280743.sc10.1562555.18 deleted file mode 100644 index 6ff89d311e6faa523dee3037bd980d343e8e5105..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzvP=KM4Bx6@5WDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th!+vY#pmP~7mG3^{I)ykzv#ItRL`qDW}i8iAoNIZiE;5m^rU1KC&M%e*_LejZw>(P CZbF*? diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281501.sc13.2274570.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281501.sc13.2274570.0 deleted file mode 100644 index b0aaad48d3470a7f38b11313ca40b6bd7f422ce2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw8wiJi|ak}X!#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZNxPWwPz7qPlXinI=b3LMU~>hsw8XhYx%fgr zrt7Dt=Edja7Z-~%9Qa}PoVjbhDpb#_J!YRdv%kajNN|a9@k8{aWELkwH9g?AeN*^M F2mk?6Lreew diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281504.sc13.2274570.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281504.sc13.2274570.1 deleted file mode 100644 index 27047580ef6ba166e36b1cb4e1b638ae20460c58..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzv*6stu1ak}X!#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZNjv}ji5%42n|FkaIOQKe%oQ-v66E6N;s{I3 T(a*^*E*7o%P;T40|B(*>?WZp% diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281509.sc13.2274570.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281509.sc13.2274570.2 deleted file mode 100644 index 85ab7c93d8b94d8f980ab310233245c71e10bc1c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxRT=tFp<8;$eiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-k~V3Qr3%zsC+!Ab&PBm~U~>hswZyqZx%fgr zrt7Dt=Edja7Z-~%Ecj~IzrJOGDpb#_J!YRd*+SrYB)G)5_#t{yGK-U;ni?2wgY>+U E0CQA7YybcN diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281512.sc13.2274570.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281512.sc13.2274570.3 deleted file mode 100644 index 618b37813050d77c685940af4aaaf2bcaa78dc39..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxt>Z(Wmak}X!#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Znd$vpTm@>blXinIr}I*XxdJ&_;#{I!d?6sy z_0v=H;&bwgi$xh4e%smg3G7mZ>Up)t>@(+DgdPblF)n_Ho|MeuWT>VG_iRq+_dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWb(vaTq;m=owOT#IhPx7g3T4k)e`3t<>Ctg znXaFnnirpwUtBE8knqVa(5%K<6{_de9<$G!f`)KC5?o?j{181UnZ?O4OdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWRiMvpbFGnC+!AbPM@fcU~>iXw8XhYx%fgr zrt7Dt=Edja7Z-~%9QbB;F?{iTRj8g6en|FkaIPdRW2sT&1OiPf9pNk_b UF-JcqzqnZRVRO4J)1@d~02!Yyy#N3J diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281530.sc13.2274570.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281530.sc13.2274570.8 deleted file mode 100644 index 6957b181e38adfbb02a18a8020a876d01d649404..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzvvoVh>lkJC*@Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNW>|_D=JWPowOT#Irn`(05(@3UrU@zl#4F} zWV(KOYF>O!esQrV!-G$D&o>#JQ-$hzwa4r;r`Zp<9tkcnE`Erfl+5B}sHOw~+ZTE3 GZvp_rD@6bR diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281534.sc13.2274570.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281534.sc13.2274570.9 deleted file mode 100644 index ed115ef93290c0a1ebd01fa6eceb742f9ef8256b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzvbHOz_s<8;$eiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-qITVNmkQKeC+!Ab&baS$!R87SXo+))a`Ae0!p;Dpb#_J!YRd&miCr(} diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281537.sc13.2274570.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281537.sc13.2274570.10 deleted file mode 100644 index eca17a5e6054fb954b867e360fc3a3dc8928f116..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzu=&GSq6<8;$eiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-VwoYPDF-$8<{cp;&J!;FU~>h`wFJ5Nxj4cS TbM$lai;G1C>>6zAt@|7SoM0<+ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281541.sc13.2274570.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281541.sc13.2274570.11 deleted file mode 100644 index 30038c671a70b78a275f946ae27d4191c2223e96..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzvXT*^=U<8;$eiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@>$(FS_NvZlXinIC!1Lz*j#}^EpaYUF1`?u z>H6uZdGR^<#l@lw55C&jpV6JG3f1#!kJ)F=H3&TtTw+}O5IrfG#mP`j2N-ShUIz68 E0GK5}^#A|> diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281545.sc13.2274570.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281545.sc13.2274570.12 deleted file mode 100644 index 75c05f34ce752e7ab8e9cc6ddad86f5e67b29ba4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzu+Cj}(^ak}X!#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zarnr4Pz7qPlXinIr`bb@xdKI6;#{I!d?6sy z_0v=H;&bwgi$xh0e6_1Q-?>Q@s^`@nv(KCl5qc!J#JKn&dQvirlVO@Z*|a|CJPQD_ CqeII8 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281548.sc13.2274570.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281548.sc13.2274570.13 deleted file mode 100644 index 649524917a932c481edf1058148bb9571ac6dde0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvPWnxbL<8;$eiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-BGG&JuN>6en|FkaI45|qfXx-K&=Tb0=i&%U U%+b%uFD@3<-_>ASeeIVr0Ji%rt^fc4 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281552.sc13.2274570.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281552.sc13.2274570.14 deleted file mode 100644 index 5668e28094b385b5e3169ee6731e84a61cf9bb3a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzv{_A90Qak}X!#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZG2IcOqyjb9NxQ+9Q{=8P*j#~PEpaYUF1`?u z>H6uZdGR^<#l@lw2R_+-V9yj+h3a{=$Lup_3qp?smlzj6L{Ca)aWYhs!W|pqY17XF E02mxVQ~&?~ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281555.sc13.2274570.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281555.sc13.2274570.15 deleted file mode 100644 index d0ae549115936bafb5c2c475682a387724549d5a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxlIP@#!kJC*@Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th?KyMDix@?PTCE=oX0HwfXx*s(Guqp<>Ctg znXaFnnirpwUtBE8!0^?MVH?8_Rj8g-as$LXe{6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v_p0H}41;aY{R_0-GyfsU^t8&&3gz Un4_PQUtBDjyQ$Uo>$ALx0AxNd{Qv*} diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281563.sc13.2274570.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695281563.sc13.2274570.17 deleted file mode 100644 index 0f40b4a47b01452ce1695ec8231d99827083f0c3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzv9T75R{kJC*@Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th`i&R4i%`mPTCE=oHJO?gUuBv)e`3t<>Ctg znXaFnnirpwUtBE8@ZhJN)4{trs!%C>J;)m!-$t+HWYEsx{O!esQrVL&FEVn^jiRRiS!b?J@hzsTK&=Bf%xc#ShVwl3AP#({$4&evb1Z E0M(2`(f|Me diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282323.sc13.2296022.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282323.sc13.2296022.0 deleted file mode 100644 index 18f728d79d7532db9417515e47796d4e9d943e42..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzxd1w2;y>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;=N~KogCEMn|FkaIG{2LJ#7 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282328.sc13.2296022.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282328.sc13.2296022.1 deleted file mode 100644 index ac51c9f4c20c88baa9473985ce4fdc4093bf099a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw)d?BRv*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL?->5t_sv#C+!AbPD>*Zu(<-|TH;)yTznxQ z)AiF+^Wt;zi;G1W9(=X?pYerD6{_de9<$G!QxJM2xWu^lA$n3Wi<6<66mHqvxmmd! E04&%*!2kdN diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282331.sc13.2296022.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282331.sc13.2296022.2 deleted file mode 100644 index 55acf9de289b0e89253c00fdf11ca3d6e906c061..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzw~`}k4quhUIODc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$gj-Cr7BQ!owOT#Ip^Dd2AeBTp(V~G%EcD~ zGF?ACH7`CVzqnYG;lNM3oGiBOs!%C>J;)m!-$t+HWYEpP=b13?W FJpjIuLpJ~b diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282334.sc13.2296022.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282334.sc13.2296022.3 deleted file mode 100644 index a110068f7bcdb00d75c3200eea580381453ecb9b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwy99*OR*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vqepjn|FkaIM+^E2R2v0T1$|NpNk_b VF-JcqzqnZR!@-HRf1dXm000}7FNgpD diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282339.sc13.2296022.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282339.sc13.2296022.4 deleted file mode 100644 index 04d00749a72cb5874a6eb3a0e369150972eeef9c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxVxO7?LuhUIODc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$nn_^-m5^(b<%F|l3N}}uQcIjml#4F} zWV(KOYF>O!esQrV!-Fq&o9`$0sY3O<+GF;avl*dBf=i5xAEGBEvp5;5iQ$&bQw`%+ E0LH*W(*OVf diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282342.sc13.2296022.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282342.sc13.2296022.5 deleted file mode 100644 index 65fd4ba7d04b5bdb94c8c9b4dbc24362efae5f49..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzvnaPHRp>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-Qq)mwtpYXINxQ+9)5?Dj*j#}sEpaYUF1`?u z>H6uZdGR^<#l@lw2R_;DePw=46{_de9<$G!vk`hExWu^lA$n3Wi<6<68t&PYTD-~t E08p(#zW@LL diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282345.sc13.2296022.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282345.sc13.2296022.6 deleted file mode 100644 index c1339e2c43802263b3fcee8af6fce71d39dbc42e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzw;oy*et>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@;Tf0sT|bYn|FkaIL~Y6fXx-K(Guk1=i&%U U%+b%uFD@3f`_X61Zq)x10Q9IXHvj+t diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282350.sc13.2296022.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282350.sc13.2296022.7 deleted file mode 100644 index af8559b86a31561693b20d180a17dedd758df343..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzwK4(QeX>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a%9zt$tqBDowOT#IoF(pm@81NCC(+v#TNoH zT|YfFFFq%~xLB0oz$d#v=}mJ~p?Y5JG5gFZdk(Hgf=i5xAEGBEvp5;5so|vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a#W#wrwY_uC+!Ab&aNMsU~>g(w8XhYx%fgr zrt7Dt=Edja7Z-~%D15Z*TlKO^6{_de9<$G!FA;hqxWu^lA$n3Wi<6<67M!;6EYaKz E0NZCmyZ`_I diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282356.sc13.2296022.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282356.sc13.2296022.9 deleted file mode 100644 index f42a128a806c5307b52f13bd50d80248f21b5371..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzxlbCcEm>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-qFA}vP7Z4B%{xLyoZma;z~&0rY6)`jb8&NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zk=|wUO$BPMlXinIr#FWi*j#~HEpaYUF1`?u z>H6uZdGR^<#l@lw4xj99ziWQ03f1#!kJ)F=?Fc;*Tw+}O5IrfG#mP`j3Hxn0kNX<| E04?@F1^@s6 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282364.sc13.2296022.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282364.sc13.2296022.11 deleted file mode 100644 index 44bbc250ae475cfc6fec3275046802e067d73f94..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxF^_0^8>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-5~mz`R0V3TlXinIr_fw!u(<+tTH;)yTznxQ z)AiF+^Wt;zi;G1W8a~=xQCzTJ6{_de9<$G!nFu`+Tw+}O5IrfG#mP`j4u@?dr$z1u E0NGYQd;kCd diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282369.sc13.2296022.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282369.sc13.2296022.12 deleted file mode 100644 index a0ad77e62192a7e9329818f3f19a66639fee7e36..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwq-0EoX*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v@WhY`~=0XzCHH~;_u diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282374.sc13.2296022.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282374.sc13.2296022.13 deleted file mode 100644 index 1274e599a75986b812a038073c307068ed3684e7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 193 zcmeZZfPjCKJmzxp&7W@g*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWV#}&jSAFUC+!Ab&XdeD!R89oYl(A-a`AdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#M=GqdsV2pulAUI=CrVP1)D3-z^En8CCbGY z0y17dJvA>rC%?E@ltJN>oxPj*S~aMqV*kesoYRmrNpOjA@k2DFWELkwbusL+i9Y=8 FC;$`#K(YV; diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282381.sc13.2296022.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282381.sc13.2296022.15 deleted file mode 100644 index 105a139e593f97b6ba6efe6008d796f6dd06f459..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvPTk355*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWNq-Ta5<>CH}41;an|#=fXx-K*AnF7=i&%U U%+b%uFD@4KGn;68=IOM>00FfwKmY&$ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282385.sc13.2296022.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282385.sc13.2296022.16 deleted file mode 100644 index 56b09cb22c46856f541a3ba040e710ee9bbb2eb3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvvu*fj^>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vTIjxxGL1#S9{DpbFOL01e+_+$fzaGCCbGY z0y17dJvA>rC%?E@lwrXayLyw-Wol4O#r}^OIPF>ynk2ZyxcDKOQZkE^VY-gm$SACv F001}2K|cTh diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282389.sc13.2296022.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282389.sc13.2296022.17 deleted file mode 100644 index c8552c650ce4778972b972a3fc9c9526585a255c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwKo8oQy*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#5ZcnI#sB-ulAUI=5!JD0h=q(#Hc0CCCbGY z0y17dJvA>rC%?E@l%e6Bopj1R9W|(?V*kesoO6*hNpOjA@k2DFWELkwbt&AiQ7TkY F0ss^fKKB3s diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282391.sc13.2296022.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695282391.sc13.2296022.18 deleted file mode 100644 index 6cf980a4e80607038986a92e05e79cb52e853d9e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzv{`2RQk>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vPPA;SPp9L%{xLyoLmPO%s}P}IA{rS@pEy6 VCFbbofuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjw5s|_pS*lQTU+ppb%(=B?1=w7HW=1V>E>SMN z5Rmcu>8W|~Ir+uKq6`n-*r`l?HAxMsso4K91E)kQLX!lS7#BZ8Q%YuWGECPEo9EFY F4FKiHMWFxy diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283064.sc16.2520987.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283064.sc16.2520987.1 deleted file mode 100644 index f1198c1fb2e77db23abca0c8a8cff2cf45c0d623..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxhJudVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vUwb3MqI@0 FHUQWRLWTeU diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283067.sc16.2520987.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283067.sc16.2520987.2 deleted file mode 100644 index a97be4377ad0b76a7aca4e3caf112759a8c81451..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzxNZhd_3uhUIODc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$n%CmV>zg~H}41;aY_n51)D42s3pk7&&3gz Un4_PQUtBEuu({sWPq6X>0H6df#{d8T diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283072.sc16.2520987.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283072.sc16.2520987.3 deleted file mode 100644 index c4b095ed3edfd2df3d184d0c89849b99de6a64b2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwGbMig->vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-va;C!ohsDaS9{Dpb6R%ugUuCaWz-Vq66N9x z0U58Ko|+e*lV4mc%CO+M-LwhrjcQO$#r}^OIAPom_v%j&9 F3jj{nLM#9P diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283075.sc16.2520987.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283075.sc16.2520987.4 deleted file mode 100644 index 7a64fc88fab3631ce17448e1e14707d8b8b48583..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvb9{T>^uhUIODc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Th5yumnaus z2*`N-^whlgoc!WqQHF+Bc2a$!J!()*#r}^OIDI`4nk2ZyxcDKOQZkE^p}G=I+K4Tc G;sF4S7(+Aw diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283078.sc16.2520987.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283078.sc16.2520987.5 deleted file mode 100644 index c09e0ea1053ec75e0cb3029375dbece32144732e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzw4JG%1WU#FXnQoKn;iJ5tNu4SotC00g3dR#gssd>fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwk>^}GH|3z_-n=7Z#JNdsHP~DMCoMrPelCu% V#2o#c{NiHK9}hZh@5Yab|*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#DpV!mMYZTS9{DpbG})70c@^7JEN93mnaus z2*`N-^whlgoc!WqQHBK%?HH?^AE`k#75hJC;B;7r&?Lbn#>EfOl#*GT4Aphuk4>8N Gg**T!JVavv diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283086.sc16.2520987.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283086.sc16.2520987.7 deleted file mode 100644 index 3eafaac0c89d27a9ba904e48ac9ed465178aa229..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw$q%3{>*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq%%KJL=|f8t376)Ij^i)1~ylqgHcPIOO%T* z1Z2E^dTL&LPJVH*C75hJC;0#!c&?Lbn#>EfOl#*GT4AteJZyQ&# GDF^@%DMMob diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283089.sc16.2520987.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283089.sc16.2520987.8 deleted file mode 100644 index 70f74ea364e1793b8fdc14b4d265e5c390435f9f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvbmd8E$>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-60q9-oE+5Ln|FkaIJLjWgUuCi))M66=i&%U U%+b%uFD@3{baIOAm6-1@0X?-aw*UYD diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283094.sc16.2520987.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283094.sc16.2520987.9 deleted file mode 100644 index 6164b2e215f1411e2296ceb2ae3a7cd63e862295..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxVi)?!O*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vB(_*ATor2Wt376)Idhp>z~%~cGHQu)iE{CU zfQ;8qPtA+Z$uBMzWng$vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-($idUOciSGt376)Im_B(!R88dF=~l(iE{CU zfQ;8qPtA+Z$uBMzWk`5s*ZeTfK@F;@*#9vDCuavjlLVI-7e7Q(N@j5~RF?yv?IOVw F{s56fL8brz diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283100.sc16.2520987.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283100.sc16.2520987.11 deleted file mode 100644 index 654f5991b8efb7e34dac76b593d21641c3162140..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvP$>M$f*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vM7QtLF*&HYH}41;aemn@05(^^MN5#2pNk_b UF-JcqzqnY`O}yE*^y`uL09EZUMgRZ+ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283104.sc16.2520987.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283104.sc16.2520987.12 deleted file mode 100644 index dcbce7d315ffb17180d4d3d1b458f35708a15d12..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxpz0-Q}*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWTEV~2dYqWU+ppb%=y7w2W+lDH=~v~mnaus z2*`N-^whlgoc!WqQ3i)6cDAvA?BCCbGY z0y17dJvA>rC%?E@l!4)u-SXaRN@`F|#r}^OI4>e;lHd~K;)iHT$t+HW>RMo68?bSr FHUP~^LGJ(n diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283110.sc16.2520987.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283110.sc16.2520987.14 deleted file mode 100644 index 862497b4949ab4b630661fc7aa09acc2ef1bb6cb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzxxo!$TPuhUIODc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$oac2o^nugZ{86y;?%b|2sT&1RZEbIpNk_b UF-JcqzqnXbz^={qq`7V?0FT%%rT_o{ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283115.sc16.2520987.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283115.sc16.2520987.15 deleted file mode 100644 index 3fa9391711a3bf02a51fcc7c59cd979554b95942..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv<;(7S$uhUIODc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$kAPDYN}9kU+ppb%z0ztBe1ywy^LDoT%ufj zAt2-R(^K=}bMlLeMHw94+6CmAA diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283118.sc16.2520987.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283118.sc16.2520987.16 deleted file mode 100644 index 84b2dae01e10ead0717275c47ceefda9171a971e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvfxW4-JU#FXnQoKn;iJ5tNu4SotC00g3dR#gssd>fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjw5mm7ljH*y`U+ppb%-J8h7HqCSAETBymnaus z2*`N-^whlgoc!WqQ3i+ScKEfOl#*GT4Aqq&Vmtrv G_0<5N4n(g2 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283121.sc16.2520987.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283121.sc16.2520987.17 deleted file mode 100644 index eda1296c82f45df8d79a601dd09e0edb6e54d7f8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzv1mc+jK>vYpmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-qRMHuLk?>0%{xLyoQ7ZHz~&0LX$f-ib8&dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vBr$Cbt18soS9{Dpb6&nv1vXcppHWMkOO%T* z1Z2E^dTL&LPJVH*C75hJC;B>o-&?Lbn#>EfOl#*GT4ArF|U>k7p G$!-9H2t=U( diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283129.sc16.2520987.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283129.sc16.2520987.19 deleted file mode 100644 index 69597aaaf049fb15376f8da7de456ca822971f66..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 141 zcmeZZfPjCKJmzw)XN!OL*XgFC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#4zVcqbk(gS9{DpbKY%A0Glf?fl*7GOO%T* Z1Z2E^dTL&LPJVH*D1*auyV{Pm_W-?PGfMyf diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283892.sc11.2190319.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283892.sc11.2190319.0 deleted file mode 100644 index 100039dd648445731124bcd78a1e44632811321f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwGTsK<#&*`S46mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL|kTTtt!;qS9{Dpb4pA#0h=o@fl*7GOO%T* z1Z2E^dTL&LPJVH*D8qvncJE!q*Q!A^75hJC;A}wBB*7)d#ShVxl3AP#)pdZ&Hf7O% FNdRvILBIe2 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283895.sc11.2190319.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283895.sc11.2190319.1 deleted file mode 100644 index 405a78cf24829bbf049e329cf43ee64842bf08d0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvfzxQbEKc}0HQoKn;iJ5tNu4SotC00g3dR#gssd>fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwk?ZvjMdhI8-n=7Z#90&h7;LV9yOtmqKNm+> VVvc@JesQtrMc3uFcer`x0|4sSFpB^H diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283900.sc11.2190319.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283900.sc11.2190319.2 deleted file mode 100644 index a0d4fd7e1bf4332c05bd19796fa4df61841ed06d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxF@#kFs&*`S46mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vLJ{ z$awwq)V%nd{NiF!28Jhg_NiMBs6jOq`#)yj;B*7)d#ShVxl3AP#)z!deo8aO7 F4**?hK_&nI diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283903.sc11.2190319.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283903.sc11.2190319.3 deleted file mode 100644 index 23f51cc4fb3b03118ecf513f44852353623c5995..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwSPI|lkpVLi8Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$j0WCv8qsWU+ppb%(+?g9oSrfNsL&!qU+ppb%=s>TC)iwp$&6a!T%ufj zAt2-R(^K=}bMlLeMHv=6vMX4!ae*3CQ?dVJ2F{cWgeD0tF)n_Hrj*R$WT-9%c3b7; G_i_Nh(L`kc diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283914.sc11.2190319.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283914.sc11.2190319.6 deleted file mode 100644 index 5ae294397f04ddc0eea0fa5e2d50fdc9c325621b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwqtDd>(pVLi8Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$j`_Z)v8c)U+ppb%-Qq}Vy?gxMlEqJQ7*m^ zkn#HIsd@1^`NhSe3=WU%X3Iqvt3fpt`#)yjyo;nsf=i5xAEGHGvp5;5>jA5+cvCC~ E0LTkOF8}}l diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283917.sc11.2190319.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283917.sc11.2190319.7 deleted file mode 100644 index 4c150b8e53ccd4f29e01beb09a2de323d12cb223..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwGO%B-n&*`S46mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vgx4rlMGk82%{xLyoST#a!R88hY6)`jb8&_*Zg!6nAU57Cs8S)2^jwcxi+?!u(I E0GQE4YXATM diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283925.sc11.2190319.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283925.sc11.2190319.9 deleted file mode 100644 index 7ff19659296befe4e40fcde41e4caa8f18d09b14..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwiNcnI5=XBFiiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GEe9ZmnziUS9{DpbKdBOm@6=iQA?all#4F} zWW0WQYF>O!esQrV1H%J5rq^q}sX;Xr`#)yjG@O9YB*7)d#ShVxl3AP#)ur&s=1`vU F4*+gXLZAQu diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283927.sc11.2190319.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283927.sc11.2190319.10 deleted file mode 100644 index 9b7ad02ba0be030e061b40b5601864f9c59fca9f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwK+V*$rKc}0HQoKn;iJ5tNu4SotC00g3dR#gssd>fuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjwk%^r4{BlrpZ{86y;uN#}2R2v0OG}W8pNk_b UF-JcqzqnY`>}H#7O~!Q<0Gz`v_5c6? diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283932.sc11.2190319.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283932.sc11.2190319.11 deleted file mode 100644 index a6f1f18d7293566520fae180b716f9fa2ff4682a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxd_b6@u=XBFiiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(k*UxRuyXQt376)Ip4oh2AeA|ol#4iOO%T* z1Z2E^dTL&LPJVH*CKAJm|liv1rmaE81_Xp-O(O!esQrV!-FSw(GNU#szEgs`#)yjWX?folHd~K;)iHT$t+HW>N;@8=J`*# FlK^GSLx%tW diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283938.sc11.2190319.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283938.sc11.2190319.13 deleted file mode 100644 index 3e170181e8c9b4b1ff56b51fa2e5292292376ddb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwq?%%WHpVLi8Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$nDp*E99W&-n=7Z#JQS(FW6iGZ!JMCelCu% V#2o#c{NiF!yB{^STUQ+V2mrazF{JfuMM?RIMJam4 zrMbC@MU{HxMVTe3MS7_qRq<(=IjQjw5gi$&PF1M6ulAUI<_z|~2{u|V~&4-J6 FsQ@0eLs0+# diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283946.sc11.2190319.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283946.sc11.2190319.15 deleted file mode 100644 index d0bdf7cc2419390e1ad8da474438c523a4967a8a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvr3$NSt&*`S46mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v)O!esQrVgTp&Jadl2nHK?Xy|HllRSxA~BxWu^lA(~P$i<4ox?%EtrmX!qn DFIGUE diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283949.sc11.2190319.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283949.sc11.2190319.16 deleted file mode 100644 index ab749924505884cd4a2a8e15dd1548e8ab4ac22e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzw~JruwDpVLi8Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)T$c}Gwx5`1yy?IB-h%+oW0c@^-kCq@8KNm+> VVvc@JesQs=e9a_Vrsu~m006XAFm?a{ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283953.sc11.2190319.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283953.sc11.2190319.17 deleted file mode 100644 index ea067c01547388263a094bc0ccb995ee2d0c263b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwiSl8_N=XBFiiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vQZ}Fsw&jnS9{DpbNWfvfz1_|&8Q{LCCbGY z0y17dJvA>rC%?E@l%e5{-MUKeQ)*C6#r}^OI8P#JlHd~K;)iHT$t+HW>SExvJyJH~ FApoDaLU{lH diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283957.sc11.2190319.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283957.sc11.2190319.18 deleted file mode 100644 index 94171004779b914dfaff40222ffd7af1a7cf4d6f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv5>4oh5=XBFiiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vM_y_iYnCHS9{DpbEa>Gm@6=cQA?all#4F} zWW0WQYF>O!esQrVgTf=bD^?m(YEVtZ{*M_rnYJJ_NpOjA@k2DFWELkwbukFpTKr;q F0{|GMKtccj diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283960.sc11.2190319.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695283960.sc11.2190319.19 deleted file mode 100644 index 57cc84743fb53df5293373f66e52c9ee2bde0791..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzu+?qc2d&*`S46mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vO!esQrVL&9yl)8*yI)S#M*{U0-Ms@y|plHd~K;)iHT$t+HW>SFk0QxWA} F3;=kRLAU?_ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284630.sc12.184661.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284630.sc12.184661.1 deleted file mode 100644 index 68652ed61f481c0b246f45f8f35e65c0d601ba93..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvX+*p$O-|42K6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#CqRWPgSV7ulAUI=8W-L1~yk<9;22xmnaus z2*`N-^whlgoc!WqQ3i*Hc0w;Fd8Hy`OLRA0& diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284633.sc12.184661.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284633.sc12.184661.2 deleted file mode 100644 index e3eb554bebec83aae6aad5b5f4374a3794c3143a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvvi?2!h?{w2qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-VtrKogB;Y{n|FkaIG3G+m@D9?CCJ6k#Sxa6 Tqo0#sTrAqc-)C#Qv-cqYDbX(f diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284638.sc12.184661.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284638.sc12.184661.3 deleted file mode 100644 index 38350d3010e53ff9d53f3cb51e8fd2f9852f1b6e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxdM$AnA?{w2qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a{urxX;rAXulAUI=Bz#gF;`$dqn0?AC>LJ{ z$awwq)V%nd{NiF!h64}nE`QoMPYtT6*#9vD=Qkux5?o?j{18nknZ?OaT@FudHkDnt F2>@`&MxX!y diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284641.sc12.184661.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284641.sc12.184661.4 deleted file mode 100644 index 4c7975eda3114e5dcf4d55ee7e803ef6820426f2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv%yBB5rce?2)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zd8U47mnziUS9{DpbLJn0m@BY=QA?all#4F} zWW0WQYF>O!esQrV1H((ZB_FnbQiEzL_J7R4c@#;L1eX{WKSWbXW^po9*Me&{`#;zv F0RRW3MG*i1 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284645.sc12.184661.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284645.sc12.184661.5 deleted file mode 100644 index bf72efc6fe14ebccdc221203a561e9288d7f786e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwq2=mMQ?{w2qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a{Jf!3^}N|H}41;ah}=@F;~D}OOT77iz6&C TM?WXOxL7oG@>E+*B^^@$T|X|r diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284650.sc12.184661.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284650.sc12.184661.6 deleted file mode 100644 index c236b45dcb6eafbaac447d0a38cf6035b7c2c97b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvPzSE!e-|42K6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#IA)?Tor2Wt376)Ik)*w0GlhYkWovVOO%T* z1Z2E^dTL&LPJVH*C_}NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZiTvCXr3y9o)gH6YoSPaU<_aug)Dq_s<>Ctg z8Lyw7nirpwUtBE8(D2MIp}xpl4XUZw|1krnOcO$r1eX{WKSWbXW^po9m%?rvb@yq; E0NVmT_5c6? diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284656.sc12.184661.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284656.sc12.184661.8 deleted file mode 100644 index 138d443b8684fd959951bbdb15edd5d40e698686..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwK4ra^w?{w2qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GIhe!J91ESZ{86y;=FtjVy-}dmLL~D7e`oP Tj($#lak1zj_T{#YwhEpAJK8Qy diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284660.sc12.184661.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284660.sc12.184661.9 deleted file mode 100644 index 801f2c478f6cc23c6a6d61618e741fb5be3be046..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvbty9YV?{w2qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^5l7stSZ#pS9{DpbDmI92AeCem{CidOO%T* z1Z2E^dTL&LPJVH*CO!esQrV!-J=GY9S$IYEVtZ{*M_r6OlAYaEWp8Lo}si7AHe>IqbEOdC_+i E00=5XApigX diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284666.sc12.184661.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284666.sc12.184661.11 deleted file mode 100644 index f381907f4dc342ee5a673283b4d80c0180c33459..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzxlxV$>=ztc@eDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Ti12xx<#JGSZ{86y; UVvc@JesQtr;_g1%I{9T;0CQt6A^-pY diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284671.sc12.184661.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284671.sc12.184661.12 deleted file mode 100644 index b5f686749e8023a5a291cf5483ae591ab94006c9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv%?K_tL-|42K6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vUyxtCeomW F9RQjfL4g1O diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284674.sc12.184661.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284674.sc12.184661.13 deleted file mode 100644 index bbaeb513e01695c862edfd5211b0b50f70f48755..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvH)hsRe?{w2qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vf4m5L=|f8t376)Ie8hEfz1_I#;7IECCbGY z0y17dJvA>rC%?E@l;OZ5JH;7l!fH@W#r}^OIHx0NlHd~K;)iHT$t+HW>QdNl^WMeJ F1OWSGKi~iW diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284677.sc12.184661.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284677.sc12.184661.14 deleted file mode 100644 index 9c4f8aa01f10a4995e779ecdd010e7497396e05a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvTHis1cce?2)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zx%VRWh8)z~n|FkaIOU#0%oPaI66E6N;s{I3 T(a*^*E*AaqpwTw#NYOC>3mz~L diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284681.sc12.184661.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284681.sc12.184661.15 deleted file mode 100644 index 486dd711d731876f9947bc74692b2e13c1f0c256..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwq%gZnN?{w2qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-5;s*nNfm1Dt376)Ir-j0%oSM9s3p!N%EcD~ zGG0GDH7`CVzqnYGLE(kntvCGt)u5V+{U0-M&PLKC!6nAU57Cs8S)2^jwP2f#VCpIf E0MD*M?EnA( diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284684.sc12.184661.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284684.sc12.184661.16 deleted file mode 100644 index 4c221fa154d389ee5e6e34328ff368c046432d24..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw;*4P#Qce?2)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZNtfBlt_n5x)gH6YoQJvX!R89AVAK-l66N9x z0U58Ko|+e*lV4mc%HVLvE-gnSM-8f}*#9vD=U*gE5?o?j{18nknZ?OaT@2f7zMuV~ F4FH8nKv@6) diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284687.sc12.184661.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284687.sc12.184661.17 deleted file mode 100644 index 221aee63048d6486d972f8b61173e18796e4d3b5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzw)jC@i2-|42K6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vMEcvLH*!#OZ{86y;#6XI2{uHq)$ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284692.sc12.184661.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695284692.sc12.184661.18 deleted file mode 100644 index 0b2e719ada9905e0b1015c33525a2557e7307e74..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwS*s_-Xce?2)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z(T!bnUlnTZt376)IYSRZ%oSM4s3p!N%EcD~ zGG0GDH7`CVzqnYGf#IHAi2CH6YEVtZ{*M_r_abSM;1c8FhiFR4EKY{$+GLYc`R^J4 D)uTaE diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285450.sc9.1183018.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285450.sc9.1183018.0 deleted file mode 100644 index 71260a87bcc59ec5c5d4d41ffb4f22db7946cdbb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxBt=nkC;C$0jiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-lEO7xRuyXQt376)IqMlWgUuCK#i%9DCCbGY z0y17dJvA>rC%?E@lp*1so#IxLJ8Do(#r}^OIIkmVlHd~K;)iHT$t+HW>DpwIobyQw E0Km6B#Q*>R diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285453.sc9.1183018.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285453.sc9.1183018.1 deleted file mode 100644 index e63f33de2038169154d4b1e0542a54dac5f992e0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwi-H0`2aK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZF+1W~D+e|A<{cp;&Th4Mu(<*uT7q2sTpVGE UIr=&I#l@l*U3+c6SA6~h0CABmC;$Ke diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285457.sc9.1183018.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285457.sc9.1183018.2 deleted file mode 100644 index 86b171ef074e7d9564b2c785e5041757a77aeefe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw)bS*PsaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zsr9yMQ-zxQYLD4xPQ`-|a|Kp2YKe1+a`ANT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZaXoy@MHOo9t376)IWw6UyxtW-iYq FO#t?PKB)iz diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285463.sc9.1183018.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285463.sc9.1183018.4 deleted file mode 100644 index 2ce3d1c9755c5df58c6e4bb8bffc7afd945bef84..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vrC%?E@lp*1k-J)++g=$bu#r}^OIMbdXG)ZuYaq&YmrDPT-!*p%7$ujz! F3;^buKC=J- diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285471.sc9.1183018.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285471.sc9.1183018.6 deleted file mode 100644 index 9174e87eefb57285e85e6981fb78c35e6cd3647a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzu=wth8daK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZdFZrqx+>J%S9{Dpa|U&O1Dh+bj!{dTOO%T* z1Z2E^dTL&LPJVH*C_}<^JL#M_4mGHzV*kesoC}aNNpOjA@k2DFWELkwbsgAblfU!F FM*wzhK;HlW diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285474.sc9.1183018.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285474.sc9.1183018.7 deleted file mode 100644 index 4a69636566dbf881c82e495d09f350ff4fde39d6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwi-h+v;?{Mxj4cS UbM$lai;G25Cnwr&TEBGz0Ciq0CjbBd diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285478.sc9.1183018.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285478.sc9.1183018.8 deleted file mode 100644 index f08baf63ce29d61f0fd16ee6537fba4b71926687..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw~dcMh$!TF}66mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vF9;#4tU+ppb%vo)-1#GUsdPXgAE>SMN z5Rmcu>8W|~Ir+uKq6`eT?KnL;EY+Zziv1rma9%*tB*7)d#ShVxl3AP#)pcNxjfs`$ FUI6D=KOz7C diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285482.sc9.1183018.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285482.sc9.1183018.9 deleted file mode 100644 index 7bb059aa143b90ac896c1fb56c576e499d5df987..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvv59qOCaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zd6f2WjVjdKS9{Dpb0(jJm@BY>QA?all#4F} zWW0WQYF>O!esQrV!-30omEnbfYEVtZ{*M_r_aSML;1c8FhiFR4EKY{%YB*^lQRaIa E0LL;wo&W#< diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285484.sc9.1183018.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285484.sc9.1183018.10 deleted file mode 100644 index ed657a70a46215af13e771f9e57a4120fdc047fd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzv<6}PfxaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZkqijYlY^Rj^Nx@ar$95rT!C;cK`wqSj!JYz*(v1! diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285489.sc9.1183018.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285489.sc9.1183018.11 deleted file mode 100644 index ef9ff28bf249dd5a12bbb89b5394ac293ea0e72e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw)Kjvw};C$0jiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;wNzExhmA$S9{DpbEdd?fz1`z$fzaGCCbGY z0y17dJvA>rC%?E@l!4*8-OcwZKh&U_iv1rma9&5!B*7)d#ShVxl3AP#)#Y&5W=4Q2 FBLHCCK3f0) diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285492.sc9.1183018.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285492.sc9.1183018.12 deleted file mode 100644 index 8b0d2a03716139ab06b96ed568d896b73ef69104..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw)KA>RB;C$0jiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a#h?!R~2gRt376)Irqvbfz1`z#Hc0CCCbGY z0y17dJvA>rC%?E@l;OcmyD)Pz5jCi$V*kesoPqKPO%hyUT>KDCDVfE|P+bmtZA6y` GnE(Kk(mSdE diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285495.sc9.1183018.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285495.sc9.1183018.13 deleted file mode 100644 index 90d392ddf988278b36675140a11d1515c214a7bd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzw~xqr@dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v|Vo&E*o~ FRsiBzK+*sJ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285502.sc9.1183018.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285502.sc9.1183018.15 deleted file mode 100644 index 9f413d564c8963b58e22568ebfd641c637d8d72f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxxyxM2a;C$0jiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;>?-Ks0uar)gH6YoPS&ofXx-y!l)(ACCbGY z0y17dJvA>rC%?E@ltJN+-S%tN-fB=y#r}^OI8)sank2ZyxcDKOQZkE^VY)8cC@qj_ F1OPU}J@Ehl diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285505.sc9.1183018.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285505.sc9.1183018.16 deleted file mode 100644 index 0be0ab24105746022b9439d773f2bbfd59588b78..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwmeav!TaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIqlD}T@GsQ%{xLyoX?DNz~%}>Y6)`jb8&Qvd(} diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285510.sc9.1183018.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285510.sc9.1183018.17 deleted file mode 100644 index 19a4f2de07eaad2fdd7d4da4c30b208eef539938..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvTyEHg5INx-X;!P?_%*@ksElbTSu`&wMhwGHQu)iE{CU zfQ;8qPtA+Z$uBMzWk|SU$Gb?yObx23*#9vD=W`@Y5?o?j{18nknZ?O4U5{;&`R)Y) E0M1Q5EC2ui diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285513.sc9.1183018.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285513.sc9.1183018.18 deleted file mode 100644 index decc880d7ce5d5effb0b008837e4456d343773c9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvfUli@c;C$0jiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-lC1GdPZetJt376)Iio(tfXx-y#;7IECCbGY z0y17dJvA>rC%?E@l;OcGyLEC8H>g2175hJC;5>??NrFp^iyxvXC9^mgsw?5L%`JxQ Fp8?r3L8$-$ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285516.sc9.1183018.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695285516.sc9.1183018.19 deleted file mode 100644 index 161b723e16d79010b1f5af1829506dc0f75a8aa8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvnbFn!yINx-X;!P?_%*@ksElbTSu`&wMhcv;?{Mxj4cS UbM$lai;G2TJ~Y}szM8lh0Qu4^IsgCw diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286186.sc10.1717960.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286186.sc10.1717960.0 deleted file mode 100644 index e3e2ebc1ff3a93e9aaa57f82c32f67614cb8c1cb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxt9&LZc;C$0jiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-V&$jAsR}js)gH6YoErr?!R88VXVens66N9x z0U58Ko|+e*lV4mc%8+o~E;c=My&6dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL^mL^OBHJFt376)Ie%HOfXx-y!Kfw9CCbGY z0y17dJvA>rC%?E@l;OZlyWe|n>!?9B75hJC;H_GGY diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286190.sc10.1717960.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286190.sc10.1717960.2 deleted file mode 100644 index 6ced3221dc72a6b83a710b95fa6dd2123d52548d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvH?p^(w!TF}66mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#M{cGL=I~1%{xLyoSWO$g3T3())M66=i&%U U%+b%uFD@3%-89`+;dy-p0OnOKw*UYD diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286195.sc10.1717960.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286195.sc10.1717960.3 deleted file mode 100644 index 888b9885695acca5932923e27c756d2b3645a569..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvf$UO9h!TF}66mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vBuO!esQrV!-MN~M@-c&0C3g F4gjIpLjeE) diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286198.sc10.1717960.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286198.sc10.1717960.4 deleted file mode 100644 index 2b3a1bd4b22adeea6da3e56683c14d87c4136a02..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwKv!3~u!TF}66mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#N&3zK2@l>ulAUI=G5<=1vXb;7o(Oqmnaus z2*`N-^whlgoc!WqQHBRM?aI^h@2NpG75hJC;GB=7NrFp^iyxvXC9^mgs_Vc88{2Q2 F7y+0iLgfGe diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286201.sc10.1717960.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286201.sc10.1717960.5 deleted file mode 100644 index 69a23837aff35fa39fabc3fbc1721cf2d72451ce..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzv!iJBaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIq;*eUKMKYt376)ISXsk!R89=X4Der66N9x z0U58Ko|+e*lV4mc%HVLrZt=F*7&WM-V*kesoIG_1O%hyUT>KDCDVfE|P+bpJ+1%W* Ga47)enL-Bu diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286209.sc10.1717960.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286209.sc10.1717960.7 deleted file mode 100644 index 4462e9b6b211a6ce3034523ed9a79d976e5b1ffa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwqo9+C8!TF}66mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vSMN z5Rmcu>8W|~Ir+uKq6`YR?AX?OajQW!75hJC;QWoGNrFp^iyxvXC9^mgs;gnCjr@6$ FCIC^uKz#rJ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286211.sc10.1717960.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286211.sc10.1717960.8 deleted file mode 100644 index 2f50d286171512b3000588cc40f0e1a5e22b8a6b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvn%76I4;C$0jiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vSe<@O*yE!H}41;aenal2sT$BR!fkJpNk_b UF-JcqzqnYGLw=TR`Uj~=00bB=DgXcg diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286216.sc10.1717960.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286216.sc10.1717960.9 deleted file mode 100644 index e7d1a79c57fd97f040b0d7d2536885be316ef7bc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxhXE1+aaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z$^W5Zs0uar)gH6YoPV!D%oW(ns3p!N%EcD~ zGG0GDH7`CVzqnYG!Qr~yjkRHG)S#M*{U0-MI$lR;lHd~K;)iHT$t+HW>QY!>lO)L% F0sxaeKPLbH diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286219.sc10.1717960.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286219.sc10.1717960.10 deleted file mode 100644 index ebbface214443507949c421af93b69d312088e1a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxVO~3t#!TF}66mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWQ*;dBdSnyU+ppb%(-yG9k96q`xv#vxkS16 zLO{mrr>Ew{=j0a`i!w0Wwp04K?zS3KQ?dVJ2F~9|nk2ZyxcDKOQZkE^p}G|2+boaS Gk_7-n9Yjt5 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286222.sc10.1717960.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286222.sc10.1717960.11 deleted file mode 100644 index 03c4d8f52716895c268d8da147a5abae474bb545..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzw){nYxI!TF}66mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vAEt376)IXmJkz~%}ZWYiMp66N9x z0U58Ko|+e*lV4mc%8+o~?!?k>XVjpYiv1rmaQ;KmB*7)d#ShVxl3AP#)#cDNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zkvq9-l`7QSS9{DpbGFSA0Glgth*3+NOO%T* z1Z2E^dTL&LPJVH*D8qppc1zr9eAJ+tiv1rmaO%uPXp-O(0B5}~7ytkO diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286247.sc10.1717960.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286247.sc10.1717960.18 deleted file mode 100644 index a6fa682d503a46da478bfcf0d8af1303830aeb47..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvvG`#YY!TF}66mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWLfOvqpDDIU+ppb%qcJhVy?hpMlEqJQ7*m^ zkn#HIsd@1^`NhSe3<}rn(zJQDt3fpt`#)yj3`No;!6nAU57Cs8S)2^jl`z@nLEOUU E0Ar;>L;wH) diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286250.sc10.1717960.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286250.sc10.1717960.19 deleted file mode 100644 index 3af39c699347727dde55563bbfc29d8ecf650681..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwmhphX>;C$0jiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!ZqXbY*nbaulAUI=Da_3J=k1!-a*Fz diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286253.sc10.1717960.20 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695286253.sc10.1717960.20 deleted file mode 100644 index 0a5a67895fd906edfac90cd2ebf7962765ad84af..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwGmxlaiaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z5nS@6O%7`A%{xLyoZrueg3T33)Dq<4=i&%U U%+b%uFD@26rB-d5r=^|+0MC6bJpcdz diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287096.sc9.1214828.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287096.sc9.1214828.0 deleted file mode 100644 index df4e7496f5eddddf047b45cebd9680fe873912b5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzx#)|ETQ=zP;riZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-lA`?RoE+5Ln|FkaI2U%ygUuC4)Dq<4=i&%U U%+b%uFD@2c++AX8@?+aQ0J>8y9{>OV diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287101.sc9.1214828.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287101.sc9.1214828.1 deleted file mode 100644 index d1c32b33bd6d7d656e58ae733c6972056ef01245..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxVd+l_d(fOvM6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?LJ{ z$awwq)V%nd{NiF!h68u(*t@M}t3fpt`#)yj{DY)Pf=i5xAEGHGvp5;5OJRzQiOHfY E06028JOBUy diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287104.sc9.1214828.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287104.sc9.1214828.2 deleted file mode 100644 index 0ca8549a5c7422a938efc5c7a401bb715ef60040..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv*DM?;nbiU~*#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZFhIF=~l(iE{CU zfQ;8qPtA+Z$uBMzWpKD|_c>uU#q6#(l)gH6YoHp^Vz~%}ZXVens66N9x z0U58Ko|+e*lV4mc%JAT(-NwW#GHOsw#r}^OIHw|MlHd~K;)iHT$t+HW>Qb0(!?C1m FBLLkuK*s<8 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287114.sc9.1214828.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287114.sc9.1214828.5 deleted file mode 100644 index 5fa171c737d4735153ed441f406ff03b901f8faa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw?@$b6C=zP;riZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vfz#OCsnAqulAUI=3LtfF<0OOqn0?AC>LJ{ z$awwq)V%nd{NiF!28TO#b&P+nszEgs`#)yj)Ney*lHd~K;)iHT$t+HW>6&Bn(<4b4 E0Ag%G!~g&Q diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287117.sc9.1214828.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287117.sc9.1214828.6 deleted file mode 100644 index e5bb49218c85a683caaff557644063895273f890..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvrC%?E@l%e64-5I{gx7DDUiv1rmaIQttB*7)d#ShVxl3AP#)#b3pW_Q&I FSpX9DL5TnW diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287125.sc9.1214828.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287125.sc9.1214828.8 deleted file mode 100644 index 69d7ac53809ab4743b9198489fce027b5d5d13c1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWV@{6PF1M6ulAUI<_!9n3pQ8a6r+|nmnaus z2*`N-^whlgoc!WqQHF*mc4rd>eyTw=75hJC;GBh|NrFp^iyxvXC9^mgrfZE&!yJ`p E0D*QwEC2ui diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287128.sc9.1214828.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287128.sc9.1214828.9 deleted file mode 100644 index 2b2e931bf0aa9039ed168c137dea2ab3cc911da7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzxVHkG@^=zP;riZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-Vzf24M-FQ4%{xLyoV-iq!R892XbE!hb8&RPbWrr@JM FD*#FXKH>lX diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287139.sc9.1214828.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287139.sc9.1214828.12 deleted file mode 100644 index eac65453214c8cbdd8fb8d79705d0a85f0f1183d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzv*%3i<0=zP;riZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vfnf~UJh#R%{xLyoUdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vB+Id}T@`BXt376)IXkxA2b(K!j!{dTOO%T* z1Z2E^dTL&LPJVH*D8qq!c52$MwA7%Qiv1rmaDGD4B*7)d#ShVxl3AP#)wN)mjh!=p FF94doLMi|N diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287150.sc9.1214828.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287150.sc9.1214828.15 deleted file mode 100644 index 6499b554d956ca88b7f5ec36e335f98c25d4f69a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzw4x-{W7qw`HiDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNMlDtjU3e6n|FkaI6sC>1e+_6rX|S5&&3gz Un4_PQUtBDDh`rl(Luj-O03cT_%>V!Z diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287154.sc9.1214828.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287154.sc9.1214828.16 deleted file mode 100644 index 20dfadcad00ef1ca5a7e717c9fca559322716eb5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvLT-dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL}7CXn<~`YS9{DpbDAER2{u>Y0;85Vmnaus z2*`N-^whlgoc!WqQ3i+WcJo&ri&2AWD)xWOz}bqVNrFp^iyxvXC9^mgs*B;8O=zRa FVE_?!K>+{& diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287161.sc9.1214828.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287161.sc9.1214828.18 deleted file mode 100644 index ee1ab992546b7f5a4db5ac37db367e2de87a32e1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvfdhdOY(fOvM6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vBwc^TF*&HYH}41;arTw^g3T33*AnF7=i&%U U%+b%uFD@2c++Ajy(7NL$02_xcl>h($ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287165.sc9.1214828.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287165.sc9.1214828.19 deleted file mode 100644 index 6b54f82e25dc017af74bc886f80a1ec100925a01..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxp9?!kc=zP;riZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;yiVqvntfwS9{DpbIz{J2b(K!kx@&WOO%T* z1Z2E^dTL&LPJVH*D8qtVc6TOJ38_If75hJC;1sGtXp-O(NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z3EzL@kt)>OS9{DpbC&;ym@9CRQA?all#4F} zWW0WQYF>O!esQrVgTh@q1s2YWYEVtZ{*M_rA0TOx;1c8FhiFR4EKY{%N?^1-V&OCc E0P@p73;+NC diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287835.sc20.1362611.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287835.sc20.1362611.1 deleted file mode 100644 index aeb15a8513dfceaec153ef9801f8ce5aa12e8808..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwqD|_3<rC%?E@l!4)iof$`#vKmxVvHxQR&H^M&5?o?j{18nknZ?OaT@To8CzRzr F2LPQRKM?=` diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287838.sc20.1362611.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287838.sc20.1362611.2 deleted file mode 100644 index 6276772fc821fab9f08632a69755f46805c2301d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwizn#&}NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZNi?)@RfU@SYLD4x&f9w-<_cV9)Dq_s<>Ctg z8Lyw7nirpwUtBE8&~V!>;O&l+YEVtZ{*M_r_4Xk&NpOjA@k2DFWELkwbsacv(|vEf F2>^>NL6ZOg diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287846.sc20.1362611.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287846.sc20.1362611.4 deleted file mode 100644 index 8c721510c6562dd9d7635352fafd130185363dbe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxpxJ~F}a=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z(abrlqY5?m)gH6YoB=x~g3T4U!l)(ACCbGY z0y17dJvA>rC%?E@l%e6FT|y7TQ8lQhV*kesoKKK6NpOjA@k2DFWELkwbv@W*Bj6&Y F2mmz%KK%dy diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287849.sc20.1362611.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287849.sc20.1362611.5 deleted file mode 100644 index d5c2374d48b0e198e5b1b811ceb3db54284c5068..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzxBR5*1pIp1`Y;!P?_%*@ksElbTSu`&wMfl#r}^OIKLxllHd~K;)iHT$t+HW>SEY#BX#rN FQ2^TzKt%um diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287857.sc20.1362611.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287857.sc20.1362611.7 deleted file mode 100644 index 18306978f6086596fa352792c50f8f1462c31bf4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw8tO@I3a=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zk&d+BSB0AUYLD4x&Q8Gyu(<-)7`4Q?M7j7v zK*sB*r{=}yNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z36rh*CkHk6<{cp;&P_#ZU~>hsv;?{Mxj4cS UbM$lai;G3YW6NxPw^wZi0AU3yx&QzG diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287864.sc20.1362611.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287864.sc20.1362611.9 deleted file mode 100644 index 5afaf507b060f7557b65859594b504430c116006..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv9YE$lGa=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZSzj0`tO_;v)gH6YoHJQf!R88FXVens66N9x z0U58Ko|+e*lV4mc%FuAr?#aDX9%@ic#r}^OI0e`cnk2ZyxcDKOQZkE^p}G_{+Z;|T GUjqO=**?zz diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287867.sc20.1362611.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287867.sc20.1362611.10 deleted file mode 100644 index ba3227d221cf1389e0450c09d3c3dd225d5eef20..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw0`Tp!NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z>51OIT@`BXt376)IVWy{m@9CDQA?all#4F} zWW0WQYF>O!esQrVL&7~fj_BX1YEVtZ{*M_rpCW0J;1c8FhiFR4EKY{%IdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vEw{=j0a`i!wa8WoPDUmZk>PRP6tlf%7$zCJ8PvE`Ernl+5B}sIG)nHl?{& F3IV6DLd^gG diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287878.sc20.1362611.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287878.sc20.1362611.13 deleted file mode 100644 index cb208c12418f4933f7097d5a9fcf1c53845e9a78..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzx#ZCpK($@!+E6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL@h2uP8DkIt376)IcMEj12$LS7NeFpmnaus z2*`N-^whlgoc!WqQ3i*bcJs}OZPlQfiv1rmaB|#5Xp-O(NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zxwda>nH<#In|FkaI8#4E%oWJd66E6N;s{I3 T(a*^*E*4dv-DNwm;p1umjWsRR diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287886.sc20.1362611.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287886.sc20.1362611.15 deleted file mode 100644 index 31f0b913265f8e8ebfad0611e04cc7ddb7918808..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxd_BBjqa=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZaY>QuP=%WNYLD4xPOICEU~>g-Gir%*iE{CU zfQ;8qPtA+Z$uBMzWk|ScH|K@&RW+!lV*kesoZFE!NpOjA@k2DFWELkwbv>AEGky0g FCjh%cL7D&n diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287889.sc20.1362611.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287889.sc20.1362611.16 deleted file mode 100644 index 996675f5dd6b59c6b34a5a3cd38f375ee84d6042..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvT*rZQka=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z$^G_sohsDaS9{Dpb8^hi0GlgthfzzMOO%T* z1Z2E^dTL&LPJVH*D8qxhcJn#ce^rBOD)xWOz_}DjlLVI-7e7Q(N@j5~R2Rd1o1JeS Gy#oNTOF~ot diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287892.sc20.1362611.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287892.sc20.1362611.17 deleted file mode 100644 index dcf0e8464a0ed69f63d085320d9241c989ebfe63..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwqWDuLmNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Znb{murV2Ip)gH6YoXIT)U~>iTGHQu)iE{CU zfQ;8qPtA+Z$uBMzWms_CPHusjt{PNRvHxQRPVrWRCJ8PvE`Ernl+5B}s4j&?HZF@= G)&KyyUp=e< diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287900.sc20.1362611.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695287900.sc20.1362611.19 deleted file mode 100644 index fc94aafbc72dfff745c70eac40a31c02f838cd25..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvvc)~uN$@!+E6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vM6u|Tnkv-XS9{Dpb51nn1e+^xk5NmUOO%T* z1Z2E^dTL&LPJVH*C_}<+I~&^~Mm4CWV*kesoG+0yNpOjA@k2DFWELkwburAcX%x5o F3;-VWJ%IoK diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288568.sc10.1768728.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288568.sc10.1768728.0 deleted file mode 100644 index afcd98fafd9c80bd300a6e46c3f9c84d3b8a3777..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzx#U88Kr?0nNviZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^6rD_c{!-LH}41;aUK*>0h=q3rzOb6&&3gz Un4_PQUtBD@>12X!Ui#$k0KkDQ#Q*>R diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288573.sc10.1768728.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288573.sc10.1768728.1 deleted file mode 100644 index 394f242887ef9e96c0166871e5b4cd5d60e09095..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxFE%LQzcE0H-#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIi%4psR}js)gH6YoQ9tv<_g?r)Dq_s<>Ctg z8Lyw7nirpwUtBE8@Zg@^imIg9YEVtZ{*M_rrz2^S;1c8FhiFR4EKY{%Ixxwmn=xM& E08kP?QUCw| diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288576.sc10.1768728.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288576.sc10.1768728.2 deleted file mode 100644 index 1cf8e78ed9807f8080cc222506ef4132db173b75..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv%zEgK#cE0H-#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZarnK>MHOo9t376)Ic>u=!R87)VAK-l66N9x z0U58Ko|+e*lV4mc%CO+R-3r}=nQBl?#r}^OI2R&mlHd~K;)iHT$t+HW>N?P8Go^gC F2mnCJKG*;N diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288579.sc10.1768728.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288579.sc10.1768728.3 deleted file mode 100644 index 8fc2e95b0ae548804cf9cb67968d0e3fd633d923..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwGeY@|#?0nNviZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GFdKQz8uutn|FkaI1{}eg3T4k*AnF7=i&%U U%+b%uFD@1}yE(@;>dNA)0J;(_hyVZp diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288584.sc10.1768728.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288584.sc10.1768728.4 deleted file mode 100644 index 4ffdba6683257dd2054c1462a2e41b6a95b34ca1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw~m*j9_cE0H-#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z>8q2jRfU@SYLD4x&LayU<_bJy)Dq_s<>Ctg z8Lyw7nirpwUtBE8kZ|8F&H8VN8dOuU|6>MD%S8xH5?o?j{18nknZ?OaT@Af9Yh5qO F0RZ8yJ}m$M diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288587.sc10.1768728.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288587.sc10.1768728.5 deleted file mode 100644 index d66db4a411b071af674763866ab716f867329cbf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvf*!0+m+4-iU6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v}pU=#r}^OIIkgTlHd~K;)iHT$t+HW>Qb0svt;6$ Fd;sE;KhgjI diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288590.sc10.1768728.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288590.sc10.1768728.6 deleted file mode 100644 index bc35c64637a546709a6e3f4bb1f1c02cf201c8dc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzxpOrGYdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#ENZlvMSWvS9{DpbH4br8*Hw?V@54;E>SMN z5Rmcu>8W|~Ir+uKq6`PF+ui5er=bSbRP6tlfiw0qLX!lS7#BZ8Q%YuWGE~=tsWzWd HCzt~O|13em diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288598.sc10.1768728.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288598.sc10.1768728.8 deleted file mode 100644 index 2127253200d422c8e685fab5edb54e64bdea756d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxt$V_x)cE0H-#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZvG|v>LKSN6t376)Ijh?yfz1_o!l)(ACCbGY z0y17dJvA>rC%?E@lp*1UU7Jpdof=eAvHxQR&L2pcB)G)5_#v87GK-U;x*klou|K1` F2moAwKl}gy diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288601.sc10.1768728.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288601.sc10.1768728.9 deleted file mode 100644 index 08cd800be07a93642147375af6806271bdc2d8a3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwGpLTL%cE0H-#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZSu^#Es~ptan|FkaIPWUEfXx*s)Dq<4=i&%U U%+b%uFD@2cy(+_Y@~@NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZStb5^vntfwS9{DpbL!rA2AeDJj8RLROO%T* z1Z2E^dTL&LPJVH*D1*a8J9YJ=&1z6h#r}^OI2RyklHd~K;)iHT$t+HW>6&M=@q2(c E0EZPnt^fc4 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288611.sc10.1768728.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288611.sc10.1768728.12 deleted file mode 100644 index 7c4cb8cd6f797841869d497d404766beb99d63dc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwKKl9Cl+4-iU6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWc6*OJ~^nlH}41;ac;2v4mMYyNK255pNk_b VF-JcqzqnXbeRhuRwynor0st;4FBt#; diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288616.sc10.1768728.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288616.sc10.1768728.13 deleted file mode 100644 index 4e582ae425b2d21ce6501b0ad3d4f1b695099900..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxps>*pWJKuDa;!P?_%*@ksElbTSu`&wMNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZS?e0fstPst)gH6YoOz3%gUuCq$*3jHCCbGY z0y17dJvA>rC%?E@lp*1^U3K0SeKn}2V*kesoX?RoNpOjA@k2DFWELkwbsbo2^CDtO F1ps7qK==Rv diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288631.sc10.1768728.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695288631.sc10.1768728.17 deleted file mode 100644 index e88ae1137a1d846f1a4d32dc12882e63a26cc75f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzx#zjoS}+4-iU6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vSMN z5Rmcu>8W|~Ir+uKq6`W5>TG{czrhOtLt`t| diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289303.sc10.1793187.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289303.sc10.1793187.0 deleted file mode 100644 index d37aca89930d7ff95f069a75a02f379fd0a34e95..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxVHa_*0+4-iU6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vu&**3+I GmJ0zkazrcu diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289305.sc10.1793187.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289305.sc10.1793187.1 deleted file mode 100644 index bdfc42161e5897e0e7265a18f71bff1e459e120b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw$9dr4{?0nNviZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-lKRkA?BCCbGY z0y17dJvA>rC%?E@l)>SW-A3MTo@!7{#r}^OI7J)~nk2ZyxcDKOQZkE^p}G!CvtgKU G>jeOwBtTyP diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289307.sc10.1793187.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289307.sc10.1793187.2 deleted file mode 100644 index 5a9da0c5ed56962b3d05251c60f2f84a8f78074d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwiHNXAF?0nNviZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-Qodl$B{`_MH}41;aqjyBF;}2eOOT77iz6&C TM?WXOxL9;{PJ=DKc+Xk@Qgbg$ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289311.sc10.1793187.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289311.sc10.1793187.3 deleted file mode 100644 index 27d5afc86bfafb752865ef449cb1c4b0cbfe9606..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvncK`j(?0nNviZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@?&aPjVjdKS9{DpbB1jG2R2vWEu)q=mnaus z2*`N-^whlgoc!WqQ3i#pcD8A)Z`Gihiv1rmaGpTYB*7)d#ShVxl3AP#)wN)n&G-3< Fw*U%LMDPFr diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289315.sc10.1793187.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289315.sc10.1793187.4 deleted file mode 100644 index 120cf572a2a203e3768915b44572dbf5baa22f9a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvX&p7>q+4-iU6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vSMN z5Rmcu>8W|~Ir+uKq6`UF>_p!DTA&8iRP6tlfwKiklLVI-7e7Q(N@j5~R9C}f8(H<= Fo&e}ELt_8{ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289317.sc10.1793187.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289317.sc10.1793187.5 deleted file mode 100644 index e327c377886fbd90c911937147be311e5b448a94..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzxpH&y;*cE0H-#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIks@2rX1AVn|FkaI3u~L!R88-X$f-ib8&SCB~V{UyS F1OPj>L0td< diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289325.sc10.1793187.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289325.sc10.1793187.7 deleted file mode 100644 index 4c5535f7ef7992bfb87465a29dd0b51d1df402af..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwmiYEVNcE0H-#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZS=7~Dp$awk)gH6YoaM7qz~%~kVAK-l66N9x z0U58Ko|+e*lV4mc%JAT_U1yg-i5gT>vHxQR&ihE3B)G)5_#v87GK-U8x~AL6e{QS> E0OztoE&u=k diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289328.sc10.1793187.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289328.sc10.1793187.8 deleted file mode 100644 index f4c90e0416c6d900087b5010d4c6e4ab51be55a4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvPy~z8A+4-iU6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vg!j4p6*;K6H}41;ahmG$gUuBv*AnF7=i&%U U%+b%uFD@1puq(0^+q&Nv0Kttb#Q*>R diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289332.sc10.1793187.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289332.sc10.1793187.9 deleted file mode 100644 index 5ffb28357a421ef1c559e9ff510261973f8c5f9b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw~{UrC7+4-iU6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq&GUwRuyXQt376)IcH@kfXx;7$fzaGCCbGY z0y17dJvA>rC%?E@l!4)j-KptIpQu4K75hJC;AF}~Xp-O(Qw+v$w5j0 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289335.sc10.1793187.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289335.sc10.1793187.10 deleted file mode 100644 index 21bc67a97cf5efee275c94e3230251ac505875c6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxdT7Un`?0nNviZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vSibtI#sB-ulAUI=H!|618lCqCq^xCE>SMN z5Rmcu>8W|~Ir+uKq6`UF?YcMLl~99fD)xWOz}bqVNrFp^iyxvXC9^mgs*7Q^P0elh F=K$EoL$3e; diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289338.sc10.1793187.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289338.sc10.1793187.11 deleted file mode 100644 index 3b3ee129e7abb08edea257902b7bea821bc89579..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzwWyk7E;+4-iU6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vr2ibt6gjB5H}41;asDx11~ylqLQ9Z~pNk_b UF-JcqzqnY`O}yFGxBIaN00S>A!~g&Q diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289342.sc10.1793187.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289342.sc10.1793187.12 deleted file mode 100644 index 5016b39b649cd5eb5df8e4b7b36a85140c07c5c8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv58}0bd?0nNviZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vMev=fGX77S9{DpbM9{52{u>YGozL`mnaus z2*`N-^whlgoc!WqQ3i#pcKvU?`_!PCiv1rmaB8+7G)ZuYaq&YmrDPT-!*tECae1R7 F1OQN8L0

diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289346.sc10.1793187.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289346.sc10.1793187.13 deleted file mode 100644 index a1ac76ede28e697dd6bf3a611864751fca5a2f58..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw$AD_s;;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vaP;orz+IkS9{Dpb50kZ3^rHb3!|1emnaus z2*`N-^whlgoc!WqQHBTC?fehQ%vFPGD)xWO!1)JBlLVI-7e7Q(N@j5~R9C`un}B_~ F!2nW|KNtW2 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289348.sc10.1793187.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289348.sc10.1793187.14 deleted file mode 100644 index 42ed8e23ef774a0a0ade347bcbf57043255746b5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzxVUuVO};(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-qCV{nyByTqn|FkaIIY?3z~%~6Y6)`jb8&NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zu~&=zE(bOD<{cp;&d!8;U~>hkv;?{Mxj4cS UbM$lai;G3|cXikNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZseZ4>qzX0n)gH6YoS{r#!R88lXVens66N9x z0U58Ko|+e*lV4mc%HVLtE|)PRUJa_L*#9vD=M5xH5?o?j{18nknZ?OaT?x}{wjbK~ F3;@GKKE?n5 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289367.sc10.1793187.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695289367.sc10.1793187.19 deleted file mode 100644 index 2812414ac9cd9a268768a4bba61147f6925d9189..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwqusY4k;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-Qo_OfL=|f8t376)IVa3G12$LS2cwoamnaus z2*`N-^whlgoc!WqQHF-AcKM+XE7hQyiv1rmaO%xOXp-O(NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZF^T={BL_A2<{cp;&StMFu(<-&T7q2sTpVGE TIr=&I#l@oHv0b*QuI9l2RnIDJ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290122.sc11.2375140.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290122.sc11.2375140.0 deleted file mode 100644 index 81b3e46fc284df33f4c5bb4d1d5e53f884b39fea..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvP4xVv{#rdYA6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#KfA}Kox54t376)IWKVU2b(MKlTk~YOO%T* z1Z2E^dTL&LPJVH*Civ1rma9Z&oG)ZuYaq&YmrDPT-!*tEEF&Ecd F3jm*4K3o6* diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290130.sc11.2375140.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290130.sc11.2375140.2 deleted file mode 100644 index 7f47a46ddcd420a5df5dea9189a849aba31443d7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzu=uAO^?#rdYA6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWL2q?u`1NuS9{Dpb8b?a2R2vW7o(Oqmnaus z2*`N-^whlgoc!WqQ3i)=c9Q$`9Mqtiiv1rmaQ;EkB*7)d#ShVxl3AP#)uk}Q#(~lI FDFC10KS=-p diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290133.sc11.2375140.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290133.sc11.2375140.3 deleted file mode 100644 index 403c803f6d6ca08aef6d55506c7dcc265d99f142..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzvPKkI*##rdYA6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL^^2GEjg&UH}41;ac=bt0GlgNqb112&&3gz Un4_PQUtBEuu({dR^Xb1m0O~(3-2eap diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290137.sc11.2375140.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290137.sc11.2375140.4 deleted file mode 100644 index 73664c2d83f9c369f7c70d105f312b281e48e0c5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxR{!2N=;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(v|G|P8DkIt376)IV18?!R8A5X4Der66N9x z0U58Ko|+e*lV4mc%CO*)UC_w|YHCnT#r}^OIKLxllHd~K;)iHT$t+HW>T;N5!}ak) F9{@3gLHz&# diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290140.sc11.2375140.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290140.sc11.2375140.5 deleted file mode 100644 index 789e36cd10ce57826a3a48fd1f8b2355eeaeb102..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvB#rdYA6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#3n)Dwkp)zS9{Dpb6zsE2AeDJhfzzMOO%T* z1Z2E^dTL&LPJVH*D8qxxb_-9igsDL_75hJC;M6ffXp-O(yr{ GgRKDm>p?UC diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290143.sc11.2375140.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290143.sc11.2375140.6 deleted file mode 100644 index 61a95602321badad1c2dc9cd4e3b413c62c5e1a2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzw`?0s{b#rdYA6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWa1XBGC8QZH}41;aaL%(1)D2St0lO!esQrV!-7k8+jAlps6jOq`#)yjHLltW7t376)Ip^O!esQrVgTpnu9iAeAYEVtZ{*M_r?RO$HNpOjA@k2DFWELkwbv@{_Nxf1& F5dhn{L9_q> diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290154.sc11.2375140.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290154.sc11.2375140.9 deleted file mode 100644 index 755c3be9badefb0314d4cd2e1d4b4d862f085d38..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 134 zcmeZZfPjCKJmzv9Ql5W`#rdYA6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWJZ+uS~;k>H}41;aVD;Tm@81HCCJ6k#Sxa6 Tqo0#sTr8S8xz+Yk;WZZk%kC}E diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290158.sc11.2375140.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290158.sc11.2375140.10 deleted file mode 100644 index 69efccb2c9df2e650211107a012938366b95e691..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv5>1;jC;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-((s(~mnziUS9{Dpb8<*+1Dh-GpHWMkOO%T* z1Z2E^dTL&LPJVH*D1*XPyF;=i9coZb#r}^OINOmlNpOjA@k2DFWELmGbamN$KmBPQ E0H`oQJOBUy diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290162.sc11.2375140.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290162.sc11.2375140.11 deleted file mode 100644 index c70f29eaa15026a26215b0bd4ff343f8edd16072..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvXyy!T?;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-5*rZ5r3y9o)gH6YoI2s1U~>f;n6$*XM7j7v zK*sB*r{=}yNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIj-N?EC)6B<{cp;&a<;2<_gqn33BmsafBu2 T=;!1Y7mLc*wAiKyx%vVCabhcu diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290169.sc11.2375140.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290169.sc11.2375140.13 deleted file mode 100644 index f4f1f838729c05108f12c713d33ccdbcf6d3479f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvf;|My(;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-QeMZhUlnTZt376)Id@h-%oS*4(h}zq<>Ctg z8Lyw7nirpwUtBE8@ZhrD8NY9f)u5V+{U0-Ms#hX3NpOjA@k2DFWELkwbuo0?TowCd F0RYd!Kt=!n diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290172.sc11.2375140.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290172.sc11.2375140.14 deleted file mode 100644 index d70c3a6325adfdc9ae4e648cfb9de117f846f0f9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvP&Qd(j;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-veUCgKox54t376)IfWRMz~%}xF=>f&iE{CU zfQ;8qPtA+Z$uBMzWjJud?$F9_|J0zGiv1rmaE2ghlHd~K;)iHT$t+HW>Pl$0N#4lv F1^`NTKPUhI diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290175.sc11.2375140.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290175.sc11.2375140.15 deleted file mode 100644 index bdcc82f0afb10234771be9fef98d9fb6bae0dbec..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzw`GhR8*;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a`adCDS4>5f!;0>oXxoqa|Ie0wFJ5Nxj4cS UbM$lai;G29ud1@;I4YD20OH^+EC2ui diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290179.sc11.2375140.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290179.sc11.2375140.16 deleted file mode 100644 index 1b95dd91eebe4ab3fd03a084b3ed77b1bb33c967..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv<$@+AG#rdYA6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq~y7tk1EvMS9{Dpb6UQIm@ClCq$SQJ%EcD~ zGG0GDH7`CVzqnYGVZkN4PyUPg)S#M*{U0-Mu0zr!!6nAU57Cs8S)2^jmC$NacQI%? E0ONK;rvLx| diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290183.sc11.2375140.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290183.sc11.2375140.17 deleted file mode 100644 index e54c00a3f9d194d75f552406312c8afd6a59bb09..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwG1)aah;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a)C`#R26FOt376)IYnCtg z8Lyw7nirpwUtBE8u;8kl&X36bYEVtZ{*M_r8<8|gaEWp8Lo}si7AM1Wwb-n@pK1UA D%|k&u diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290185.sc11.2375140.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290185.sc11.2375140.18 deleted file mode 100644 index c28d16751b5f52dcdafed7e618e54d8ab6438941..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzw0Ev&i3;(XIliZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@=9@At~}J-KyMcb&K1nHU~>f;8MOqt__;X3 V5_9x(@{5Z_1?*C6e@XTm006V1D@6bR diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290190.sc11.2375140.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290190.sc11.2375140.19 deleted file mode 100644 index 4ff69f5c48a205d1e1c92e6ccb9965abedde9f0d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxlW}A7L#rdYA6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vggIkchbq+ES9{DpbMC5vm@ClAq$SQJ%EcD~ zGG0GDH7`CVzqnYGA>p#!x2l2%YEVtZ{*M_rO=}UFB)G)5_#v87GK-U;x*oLINHHkP F0RZFlKo$T1 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290935.sc12.371669.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290935.sc12.371669.0 deleted file mode 100644 index 99effbbd1071ed1e8cdf6c3fce1ff55f2c780efa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvnx_szmb-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z;V|5ip$awk)gH6YoC$kAg3T3ZW6~1m66N9x z0U58Ko|+e*lV4mc%JAT--P)w@i`Af-iv1rmaBf4=B*7)d#ShVxl3AP#)751o5%TpI E0Q{#yUH||9 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290938.sc12.371669.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290938.sc12.371669.1 deleted file mode 100644 index f0d5612717277876ec4841d8e85529541d90b9b2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxRe4gLO>U`5tiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-k}$g=O&)4)ptp+zr(@Fsu(<+Fj9P+R{9GJi Vi8=Z?`NhSer_@qyTWu!E0076UD|G+> diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290942.sc12.371669.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290942.sc12.371669.2 deleted file mode 100644 index 53c9f129ea5e768ee2b4d3d479160186a58b43c9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvvmD}0R>U`5tiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-5}uUwQ59T2k-d30lG FDFF03LBRk3 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290946.sc12.371669.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290946.sc12.371669.3 deleted file mode 100644 index 45b79eb508f9135d19067bf179f1176c038163b0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxxdiPCWb-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZNveKvS`}*Ut376)IWxBQgUuD_VA2xj66N9x z0U58Ko|+e*lV4mc%JATlU5a_yBQ>a|V*kesoY#>wNpOjA@k2DFWELkwbv5+bB=+$p F0|4RcK&Su! diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290948.sc12.371669.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290948.sc12.371669.4 deleted file mode 100644 index b0b90478f230a2b5ee0c56b396672a1ba8df4770..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzwC^qWs)b-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZSy`X{O&)4)ptp+zXU2OAu(<-wj9P+R{9GJi Vi8=Z?`NhSe9}X7VdM}XD0|2;YEMEWs diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290953.sc12.371669.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290953.sc12.371669.5 deleted file mode 100644 index 3d0f20ac95834b90e9764893a43469a9c9b5b1d7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvH^af30b-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZnOgOvP!($Kt376)IiEZZ2AeC;$)qLDCCbGY z0y17dJvA>rC%?E@ltJOD-Ry)m0X3+mV*kesoQY2mnk2ZyxcDKOQZkE^p}H8lZB8@B G8v+2h7e4d= diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290956.sc12.371669.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290956.sc12.371669.6 deleted file mode 100644 index c1375149dacf1dea52b5a2921c9680d5fe41e2b1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvLDws`Xb-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZVf~-Wq6#(l)gH6YoPjgV!R88dF=>f&iE{CU zfQ;8qPtA+Z$uBMzWpKD+cWP5gvl>)WvHxQR&do@gB)G)5_#v87GK-U;x*FPS_P_YI F767H8Kb8Oh diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290959.sc12.371669.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290959.sc12.371669.7 deleted file mode 100644 index 9d98938a7f6682c1380e55ea2db158f6a1b9df02..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxB{CF{$)%m8Q6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#QFR&9(ky_f!;0>oKiDhfz1_YVbl`j;^*QB VOU%*F$uBMzZQ-x7bv_;V8vy2>ERFyG diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290964.sc12.371669.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290964.sc12.371669.8 deleted file mode 100644 index b0242234a9d287e3a52f6f13e4474f39fdb10a66..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxR>atH|b-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zsao%LN)>AEt376)IhB`kfXx-?X3`Sp66N9x z0U58Ko|+e*lV4mc%JAU2UE;|fkJX@>iv1rmaPCFYB*7)d#ShVxl3AP#)75GdcAj+; E03NPDAOHXW diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290967.sc12.371669.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290967.sc12.371669.9 deleted file mode 100644 index 8de84be76b57b77d183f70c1419289312d88e9e4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvnp1w1c)%m8Q6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#6W3#kSf&NS9{Dpb4uCW1Dh+*!=xq7CCbGY z0y17dJvA>rC%?E@l!4)@U4CusO*N>dV*kesoEwldNpOjA@k2DFWELkwbscE3ITLN) F2>_%aKgIw6 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290970.sc12.371669.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290970.sc12.371669.10 deleted file mode 100644 index 79a544dee3ab0373f43d720fa2061f8a3227b5cd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzw)J=s5v)%m8Q6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWSY6$40))zf!;0>oE$b2!R89IGHMBO@pEy6 VCFbbodVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vEfOl#*GT4Ar%u!{)2< GJaqsbv_TmF diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290978.sc12.371669.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290978.sc12.371669.12 deleted file mode 100644 index 30cb0dd5723c81ea0bb9064fac9cbe06de51edf7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw`xwXzNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZdFNMarV2Ip)gH6YoNlYzz~&0{F=>f&iE{CU zfQ;8qPtA+Z$uBMzWq5GeuEXluPc^8fV*kesoF9=iNpOjA@k2DFWELkwbv1O^_(d%% F2LQ1WK>7dx diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290980.sc12.371669.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290980.sc12.371669.13 deleted file mode 100644 index 1d99a3c38089876d888528dde2a62e44c06a0a89..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzv{95tWG>U`5tiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(&?I)AP+S+(A!0VvqaAlY_32Xqn02SKNm+> UVvc@JesQsA>f}J%Nv_QA0Dd?sr2qf` diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290985.sc12.371669.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290985.sc12.371669.14 deleted file mode 100644 index b60e3fd1ca4f71d0b94a54e661836ca4cd5870ad..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw$UJjYX>U`5tiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-G9x`wP!($Kt376)IeFc}z~&0{Giix)iE{CU zfQ;8qPtA+Z$uBMzWms^<&Ph)3yBbtevHxQR&Nd`X5?o?j{18nknZ?OaT?+j+o?U+e E0OD>vkN^Mx diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290988.sc12.371669.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290988.sc12.371669.15 deleted file mode 100644 index b940730e5e6ce5aee6e19801f2aa5db38136432c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxlO_Z3;>U`5tiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(#d!7yDHS&S9{Dpa|&IM1e+@`fk{i8OO%T* z1Z2E^dTL&LPJVH*C_}>qyN6fVnbe?~iv1rmaLz;0B*7)d#ShVxl3AP#)74|cF2E`S E0B>eJy#N3J diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290991.sc12.371669.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290991.sc12.371669.16 deleted file mode 100644 index b1ede97241752cd640f19fd137aeffe48ae7b282..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzwK&O0@m)%m8Q6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vMCnvtkUZ4fKyMcbPHo0BU~>i98MOqt__;X3 V5_9x(@{5Z_53wiP=5f7O1pw%%D`fxx diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290995.sc12.371669.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290995.sc12.371669.17 deleted file mode 100644 index c443a23f1c850ef92c99e7c2eca58be4a17d8d3c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxNt$#9y)%m8Q6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v*33N}|@B9oRlmnaus z2*`N-^whlgoc!WqQ3i#Jc0G4olhvS_iv1rmaK1y*B*7)d#ShVxl3AP#)75QL_2FA9 E06hOez5oCK diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290998.sc12.371669.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695290998.sc12.371669.18 deleted file mode 100644 index 09eeab9c8f3517eaafc24b33a3b788ddd51d6dec..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw?|6DVd)%m8Q6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWYc#3VpXWQulAUI=9KSW2R2t=5|frVmnaus z2*`N-^whlgoc!WqQHBK`CJ8PvE`Ernl+5B}s4j;tn-xbi GjspNxAwj?Z diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291826.sc16.2741235.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291826.sc16.2741235.0 deleted file mode 100644 index e76dffbdf5fc9cf29b99f61125592c1a2c27fe5a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzwCPoJH`=6uspiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^7hmDDtV~6f!;0>oEfv{fXx-?VAK-i;^*QB VOU%*F$uBMz4Gqn=-L<=W7XbClExG^z diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291831.sc16.2741235.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291831.sc16.2741235.1 deleted file mode 100644 index 114eba52f50b46236f260aa10a6c282f7593ed23..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvXmLJPybH3>)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZF?r0%q6#(l)gH6YoP5WQgUuC~%%mmGCCbGY z0y17dJvA>rC%?E@l;OZ7yZ26uiq)W+iv1rma5f`plHd~K;)iHT$t+HW>T;NC!zd-U F1OSo_KKTFu diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291834.sc16.2741235.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291834.sc16.2741235.2 deleted file mode 100644 index ca6638eb1115e5a658105b47e75010158692d3f5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzu=IM2&tbH3>)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZNo@{#uL?Ey)gH6YoR3aI%oUizq$SQJ%EcD~ zGG0GDH7`CVzqnYG;lKsEX>;6OszEgs`#)yjR6B*xB*7)d#ShVxl3AP#)s=AFM)Su1 FB><=1L7V^p diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291837.sc16.2741235.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291837.sc16.2741235.3 deleted file mode 100644 index baa25fb3148fa84c08a46a590bcabd38e89e0a9d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzwyX$0i6Ip1`Y;!P?_%*@ksElbTSu`&wM UVvc@JesQtr?3@PMs26NU0WKpdiU0rr diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291841.sc16.2741235.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291841.sc16.2741235.4 deleted file mode 100644 index ddb5ba4c0c3f8def7abd62e10bc1520fac021a2b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzx#+gwt>=6uspiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-lE~S+LKSN6t376)Ii-2Zz~%}}WzrJo66N9x z0U58Ko|+e*lV4mc%CO+P-R6SlN7bO3iv1rma85$fB*7)d#ShVxl3AP#)zz@gCh5h? Fg#fK+K%@Ww diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291845.sc16.2741235.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291845.sc16.2741235.5 deleted file mode 100644 index 46cf37a55c5cb379508ee6924060c767eeb364d5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw4SQlH!=6uspiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-QfejMstPst)gH6YoV@$vz~%}}W6~1m66N9x z0U58Ko|+e*lV4mc%5dPaUH;T;N9^FFUP F9sp`AKbQaj diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291848.sc16.2741235.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291848.sc16.2741235.6 deleted file mode 100644 index 87fbcc2987f9e432fa92b5abe9efc1529cb9d42e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxdu!|J2Ip1`Y;!P?_%*@ksElbTSu`&wM UVvc@JesQsAY)PkWS?0ED04EG8_W%F@ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291852.sc16.2741235.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291852.sc16.2741235.7 deleted file mode 100644 index 988cd371b49bea6131e1467e7053639b529dcdd3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxV+-F_P=6uspiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-5}>zaohsDaS9{DpbKcgr0h=o@ok>faOO%T* z1Z2E^dTL&LPJVH*D8qxRcJH}0XQ@Fo75hJC;B?VJXp-O()#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z5$AQ9rwTRq)gH6YoJ*bA!R89gVA2xj66N9x z0U58Ko|+e*lV4mc%JAU2-R#v9m#RTE75hJC;8b)$Xp-O()#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZxwLNnM0u#Wf!;0>oU`UY%oXTn)Dq<4=i&%U U%+b%uFD@1h4K20RlULjU0I)wRp#T5? diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291863.sc16.2741235.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291863.sc16.2741235.10 deleted file mode 100644 index 96f94139f07451040a1f19a0b366c9f784ffbdbc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvfQh!v+=6uspiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;*wyzUlnTZt376)Ib*g!%oUi)q$SQJ%EcD~ zGG0GDH7`CVzqnYG!Qrx9cmu~YHK?Xy|HllR!dnrVB)G)5_#v87GK-U;x(;;Ow8!bJ F0RXl@KdAr! diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291867.sc16.2741235.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291867.sc16.2741235.11 deleted file mode 100644 index 3dfcac1d40ff38abe01f0f2c68c608578cdb1359..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvfO*~b`=6uspiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^7bg-4OOVQulAUI=6v_^G}v5$Sxj2uT%ufj zAt2-R(^K=}bMlLeMHw6}*`0iHO;8Q0so4K9184FpgeD0tF)n_Hrj*R$WT>u&Zkt7{ G1;zl;`9fd- diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291870.sc16.2741235.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291870.sc16.2741235.12 deleted file mode 100644 index 37082ef3ec4e0539082dcc1a843646a6554f3787..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzv9+SFLi=6uspiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a#m1hnLO0oKyMcbPB-Reu(<*~j9P+R{9GJi Vi8=Z?`NhSer_?fSn?9IU0syo3D`x-z diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291874.sc16.2741235.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291874.sc16.2741235.13 deleted file mode 100644 index 023c074fcab1a2c7e3675567f4175600054212bb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw~__wTr&H1LI6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#CNjhS5>IFulAUI=6oN%0&K3pY$h#nE>SMN z5Rmcu>8W|~Ir+uKq6`Zz*!3RXA+841RP6tlfipbn$EH^= GC=dXLM?j+h diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291878.sc16.2741235.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291878.sc16.2741235.14 deleted file mode 100644 index dd422e2d054007f0a371367b7b0d8ffa7199d453..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwqJTak?&H1LI6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#94o0q${W{KJ)+p diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291881.sc16.2741235.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291881.sc16.2741235.15 deleted file mode 100644 index ac1efa200b32b7b8d7fe55e867e4401dc5eef5e0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzw)DfX&jbH3>)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZDQ>%HD-Sg{(A!0V^Wi0Lu(<-gj9P+R{9GJi Ui8=Z?`NhSeJ$iMvO5$tk0H}*AjQ{`u diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291885.sc16.2741235.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291885.sc16.2741235.16 deleted file mode 100644 index 76a3c6857467e11259ce37ae0de12cf3f0c7e884..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwCAIhs{bH3>)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZseY-oQWa|Mt376)IrX&)z~&0fWzrJo66N9x z0U58Ko|+e*lV4mc%8+o`?$6@S|J0zGiv1rma9%>vB*7)d#ShVxl3AP#)#cD;^N`b; F4*==dKl=ax diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291889.sc16.2741235.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291889.sc16.2741235.17 deleted file mode 100644 index c2faeeedf02b899e85f65ff2eb26a9bc9d89f5f7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxFTJKTA=6uspiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-q8@SOw<^@!S9{Dpb4Idxfz1_|$D}3BCCbGY z0y17dJvA>rC%?E@l)>S$-Q2#`TWU~E#r}^OICmmxlHd~K;)iHT$t+HW>QZR1$v78~ F2LO57KHmTU diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291891.sc16.2741235.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291891.sc16.2741235.18 deleted file mode 100644 index b6c2bac84732f0e31c3f573ba5acdaab01922932..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzwG#{92gbH3>)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZkzC7aB@Z<>(A!0VbILb}xdMHRT7q2sTpVGE UIr=&I#l@n#UNqQ#^Hl2t0Id})&;S4c diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291896.sc16.2741235.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695291896.sc16.2741235.19 deleted file mode 100644 index ce8cc39eced2835fe4771be1e58538d164712d59..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvXYs%NLIp1`Y;!P?_%*@ksElbTSu`&wMNW0WcfRQ;#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zkz?7vS`}*Ut376)IaO3lz~&0fXVMbq66N9x z0U58Ko|+e*lV4mc%D`~NuFdWE12w3oV*kesoc&0eB)G)5_#v87GK-U;x(-y@_;It9 F000;wJu3hJ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292975.sc9.1353776.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292975.sc9.1353776.1 deleted file mode 100644 index d41e9c4f0cf76d992cb2a8f5a826e4afa3879bb6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv%SfA=NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zv7VpVqY5?m)gH6YoO{=w2AeCefJsZ7OO%T* z1Z2E^dTL&LPJVH*C_}?JyNgGCFRDQ`75hJC;7rEfOl#*GT4AWI(lUn4t F1OOCKL302A diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292977.sc9.1353776.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292977.sc9.1353776.2 deleted file mode 100644 index c8ff3ce347193011e9606811d113164155d9eef3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzw4*;?Df?tIfxiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-5)^&+i#*iaKyMcb&L8acU~>if8MOqt__;X3 V5_9x(@{5Z_7kBsAK3Sgq6#)DZE;9fC diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292982.sc9.1353776.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292982.sc9.1353776.3 deleted file mode 100644 index bc60d89ab1323645ff40dc54b8608d53680ace89..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxB3C-(ecfRQ;#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z`S)N}t}4{rS9{Dpb4par2b(LfkV#9NOO%T* z1Z2E^dTL&LPJVH*D8qwGb~lXA?Nx(nD)xWOz*&Z*NrFp^iyxvXC9^mgs*9n~=2nP` FD*(14Kr8?N diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292986.sc9.1353776.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292986.sc9.1353776.4 deleted file mode 100644 index 1e2ab76130621d4ac6b1d1b91f9a261694e0d998..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwKi*D&-cfRQ;#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZId1lEvMSWvS9{DpbJ{7ifz1_I#H1z8CCbGY z0y17dJvA>rC%?E@lwrXYy9-HT@7181iv1rmaPC3UB*7)d#ShVxl3AP#)x}U{^OU)| F6aWTYKm-5) diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292988.sc9.1353776.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292988.sc9.1353776.5 deleted file mode 100644 index dfb085c1f62d670d6a9648f06ceb08b0871385be..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzw88=3dBJKuDa;!P?_%*@ksElbTSu`&wMNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZdCc?7O%-bHt376)IoZUbz~%}pX3`Sp66N9x z0U58Ko|+e*lV4mc%Aj!BZoO*yV>PIzV*kesoOMW=B)G)5_#v87GK-U;x*Do&0w*5j F0s!S^J)!^r diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292996.sc9.1353776.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292996.sc9.1353776.7 deleted file mode 100644 index 19ec8f7f0b776571572fc35fcc89f91ed98aded8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvbUeTY(?tIfxiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@-FiEb5*FhulAUI<}9-^1e+_cgh@-BOO%T* z1Z2E^dTL&LPJVH*D8qpZc5$&k=c_?A75hJC;Jk*UNrFp^iyxvXC9^mgs_Q|MjnT~$ F^8o8aL7V^p diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292999.sc9.1353776.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695292999.sc9.1353776.8 deleted file mode 100644 index 6a87dc4577b0692f462ceefc0e213ee972ad2719..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxV_q;ce-T9`Y6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq`p@wP#$V-ptp+zXWE(jU~>f~GHMBO@pEy6 VCFbbodVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vSMN z5Rmcu>8W|~Ir+uKq6`j~><+&dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v0JN(t8vpSMN z5Rmcu>8W|~Ir+uKq6`X`>|}35epZ8OD)xWO!0Ez*&?Lbn#>EfOl#*GT4Aqs;VsqBp G&mI8%bU%Xt diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293017.sc9.1353776.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293017.sc9.1353776.13 deleted file mode 100644 index 80d27cadc74ccb73be46a7a9cd091c8d993d6bde..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxt?X8){?tIfxiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(*5|2hbq+ES9{DpbJmE|fz1_I!K5Y5CCbGY z0y17dJvA>rC%?E@l;OY~I}JmbS87mA#r}^OIG-bFlHd~K;)iHT$t+HW>QY!}!`2sA F1pxgaKaBtY diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293020.sc9.1353776.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293020.sc9.1353776.14 deleted file mode 100644 index f4a855278f10f70075530351662f9fe8c0d8f73f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzwKlGU5e?tIfxiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-qM>NRArCb-(A!0V^GdNk*j$0hj9P+R{9GJi Vi8=Z?`NhSeJ$ebY&lo-l0RTdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWX0sm^{P;FU+ppb%qbWj1U6S-C6ksomnaus z2*`N-^whlgoc!WqQHBRs>>7ol;?SEYvvtv2q F9su)+KfM3| diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293028.sc9.1353776.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293028.sc9.1353776.16 deleted file mode 100644 index d145a85c8bc3c1df7672b2018f2c958cd5155f12..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw8tx%iE?tIfxiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-BK<-Bo+{MbS9{Dpb2eINfXx+H#iS+9CCbGY z0y17dJvA>rC%?E@ltJOPUH-%LEH$X6V*kesoXnO8O%hyUT>KDCDVfE|P+bj+ZP-Fq G+XDdM={+|9 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293031.sc9.1353776.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293031.sc9.1353776.17 deleted file mode 100644 index 7f3c830f8ecd3a918c1dd5711d94dc0ffc0e3a29..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzw4dvtpyyYo#)Dc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2 z(%js{qDsB;qRf)iBE3|Qs`#|boYZ)Ti1eaa4)RcQ1HD}&IA4X|1)D1{g;7h8i=T@l WEHOtvC%?E@bk~a{+Z}zomI44Us4koU diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293035.sc9.1353776.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293035.sc9.1353776.18 deleted file mode 100644 index 626b42460e453d28bd06df770b2ca02eedfe9ac9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxRuJ}2N-T9`Y6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v^IqKyMcb&Ut~HU~>hgGHMBO@pEy6 VCFbboNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZiJ8L`uL?Ey)gH6YoIAPKDCDVfE|P+bcq+q8LU G=K=r?I6bcb diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293876.sc14.2649097.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293876.sc14.2649097.3 deleted file mode 100644 index 8b9aa1fb7693f0be76ac861767153a5e38875617..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvTatP&eINx-X;!P?_%*@ksElbTSu`&wMi5F=>f&iE{CU zfQ;8qPtA+Z$uBMzWoWoy_cS2pp&C?EvHxQRPM&gvCJ8PvE`Ernl+5B}sICW-Y&=t+ G{09I>Q9elk diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293879.sc14.2649097.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293879.sc14.2649097.4 deleted file mode 100644 index 9d2dc5839a08ed53dde321f4d87feefcbf5874c9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxxW}eLDaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZdHzA}raaW#KyMcb&P`V!<_b(>)Dq<4=i&%U U%+b%uFD@1>x!7&1q`z||0O6l4mH+?% diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293883.sc14.2649097.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293883.sc14.2649097.5 deleted file mode 100644 index 7f82409cb263be0f3da1db99ea4ab04c330d1f45..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzx#x$!!W!}+G86mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v;m8n5B75hJC;QWH5NrFp^iyxvXC9^mgs>@-b&C5r1 F_W=7cL<0Z- diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293887.sc14.2649097.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293887.sc14.2649097.6 deleted file mode 100644 index 4d214fe6b5be39450e600f97da6052c229865c70..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvPV!o8m;e69kiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-Qg-0(1y!iIulAUI=KNK78Eme=1|}_WE>SMN z5Rmcu>8W|~Ir+uKq6`gJ?82Te%vXbID)xWOz&Wu9p-F;EjEf(lDJ8Qw8LEq+$A+&_ Ggb@H!zCd39 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293890.sc14.2649097.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293890.sc14.2649097.7 deleted file mode 100644 index 0069594ea466982b0ba7beb3d97e2b9f8d1854eb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxdzw9mGaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z34iCuBM&t<(A!0V^F?|e*j$0>j9P+R{9GJi Ui8=Z?`NhSelYN_POZ05o0m#TJrT_o{ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293894.sc14.2649097.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293894.sc14.2649097.8 deleted file mode 100644 index 86b3627175486bf95c56fc736d21eec4b98cbdbd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxVu3uZo;e69kiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;<9&hhbq+ES9{Dpb6P)J2R2t=Ba@ammnaus z2*`N-^whlgoc!WqQ3i+WcDm2DN2x(I75hJC;CzClNrFp^iyxvXC9^mgs;i;f=G&#S Ff&f@ILY@Es diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293897.sc14.2649097.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293897.sc14.2649097.9 deleted file mode 100644 index d36e1a17aadb92dcda079a0f9d8934d3bf44c2a3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxlU0GMe;e69kiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vV2E~zbe$+S9{Dpb9ys0fXx-y#H1z8CCbGY z0y17dJvA>rC%?E@lwrXYJB#DabJU=kiv1rma0)UaG)ZuYaq&YmrDPT-Lv=ND*f1ts GHwOTKFFrW{ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293900.sc14.2649097.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293900.sc14.2649097.10 deleted file mode 100644 index 6495ba6e3800cc9d916543e763f082eabf75c6b8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxBzBDN2aK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZsXzCAy*$+1KyMcb&h1-`z~&0fVAK-i;^*QB VOU%*F$uBMz?a|A#oqH{!8vw9tEW!W) diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293905.sc14.2649097.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293905.sc14.2649097.11 deleted file mode 100644 index 3940546c0458db72602003e7bd3692bb6b9a9819..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwW_=T5nINx-X;!P?_%*@ksElbTSu`&wMEnk2ZyxcDKOQZkE^VY+&3cD{Jl F1OUv5KWYE~ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293908.sc14.2649097.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293908.sc14.2649097.12 deleted file mode 100644 index 3c68c1853494ea73ce4d35cd6e7187b4bc13a3cc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvfw6rPZaK7m%#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z@%Z*EQx$6Nt376)IloP|1)D3dg-J`COO%T* z1Z2E^dTL&LPJVH*D1*WUJKd(=qH0i0#r}^OI4hNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZshTqJjy%-dKyMcb&dqZ^gUuC~$*3jB#m~hN VmYAcTlV4mcIyNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZxwZUHlPc8QS9{DpbH2YW3N}|@E0dNumnaus z2*`N-^whlgoc!WqQHBMV>>jOn`c@69so4K9182+)geD0tF)n_Hrj*R$WSFi=Hm8N& GO#%Q>HA43Q diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293919.sc14.2649097.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293919.sc14.2649097.15 deleted file mode 100644 index 2209c712db15ee5f0e5a5bb2c4f3e595fb7e7a72..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxNFZof<;e69kiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;=*>mRTXOPt376)IS;-52{udVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq;j{9g*?>UKyMcb&NZCtz~&0fV$>4k;^*QB VOU%*F$uBMzRiB+=J8A8W1OUd@EHwZC diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293927.sc14.2649097.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293927.sc14.2649097.17 deleted file mode 100644 index 371eee16d317cfef37b10833cbc1217ceefc61e1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw4=s#7-;e69kiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^8fYO2vw-LulAUI=G0?34K`O`JCl|;mnaus z2*`N-^whlgoc!WqQHF%GcKpw|WYnOViv1rma4tmBB*7)d#ShVxl3AP#)upi4=HQ#X FJOC8BKy3g3 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293930.sc14.2649097.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695293930.sc14.2649097.18 deleted file mode 100644 index 55ded12df7ca590140cb283eae6a110fb0a350c1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxFs9Ijd;e69kiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-ayv@gS`}*Ut376)Im>^p0GlhYgGo!AOO%T* z1Z2E^dTL&LPJVH*C_};(yVc9|_p3oQ75hJC;JkyRNrFp^iyxvXC9^mgs_VgKoA+Oq Glmh?{h(js> diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294796.sc14.2671192.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294796.sc14.2671192.0 deleted file mode 100644 index 60beba1a7b0ad839a2b26e30dc51a24c00c70958..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzv%m)Hk$I^T4Z;!P?_%*@ksElbTSu`&wMrC%?E@l!4)vomZMxs~S{OvHxQR&eU{-CJ8PvE`Ernl+5B}sIG=8n|m9! GPXqw16Fz$Y diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294807.sc14.2671192.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294807.sc14.2671192.3 deleted file mode 100644 index 67b9a072cd9983132a727e56080bbd999ee5a08d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzw~fB7kt)A^>O6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vMCbGNpYl+11HD}&I4{ro3^rF_4x^SJ7e5zA VSYnQTPJVH*Xpdf#tvNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZDG^zJR~2gRt376)Iav=%g3T4!&7>vHCCbGY z0y17dJvA>rC%?E@l%e6Oou#)^z8X|hvHxQR&hKDCDVfE|P+bfSHgnqg FG6B;tJ(>Uj diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294818.sc14.2671192.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294818.sc14.2671192.6 deleted file mode 100644 index 60fbabc9e8dc9135b7363fb588acb9582b802332..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxdSni16biU~*#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z;d=3Ir##f$KyMcb&Zkp$g3T3}%cv#D#m~hN VmYAcTlV4mcT5>VhmeEagIRLdZEHeNA diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294823.sc14.2671192.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294823.sc14.2671192.7 deleted file mode 100644 index 8ae2075e0190446f3e1fa1b1b0bcc60bb2ae4124..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzu=J3o%(biU~*#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z@k+Yes0uar)gH6YoGtsFfXx-y%cLdFCCbGY z0y17dJvA>rC%?E@lp*1g9V@R(z8X|hvHxQRPV@Z;O%hyUT>KDCDVfE|P+bg-Hf!aS GCj$UbdOkM* diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294827.sc14.2671192.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294827.sc14.2671192.8 deleted file mode 100644 index 4696dfb8aacd003b64cff2208a5e4ffd23bdc066..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvL-#8P+>3q{siZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-BGc^tSQTpSt376)IawLbfz1`z$D}3BCCbGY z0y17dJvA>rC%?E@l)>Se9ZP89LN%zSV*kesoTW&bB)G)5_#v87GK-U;x)_>l`ehoM E0U`Q5NB{r; diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294830.sc14.2671192.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294830.sc14.2671192.9 deleted file mode 100644 index 6e687df00f72eb2b373699bf91255a0b7a700f17..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzwK`_dTA>3q{siZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-V*0gekv!DgKyMcbP9~RTu(<;B7_|hs__;X3 V5_9x(@{5Z_V@uL)Z>3q{siZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!ZxRJyDHS&S9{DpbN>0Y3~a8zekLt(E>SMN z5Rmcu>8W|~Ir+uKq6`nN*zMp*S5<>*D)xWO!0GxMp-F;EjEf(lDJ8Qw8LF$H)F$+j G&IABDWI;3l diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294838.sc14.2671192.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294838.sc14.2671192.11 deleted file mode 100644 index 6c630cbef5ed4bceb99dfb16c1c6fd0c1e386a20..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv{Uu%lxbiU~*#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIrh|4RuyXQt376)Il~iLz~%}ZVA2xj66N9x z0U58Ko|+e*lV4mc%D`~bPM{@unHp46vHxQR&SOZLB)G)5_#v87GK-U;x(+njB$)a; F004MBK8*kX diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294840.sc14.2671192.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294840.sc14.2671192.12 deleted file mode 100644 index 2e74a67b779c7b234132cbf388af180cad36b5f5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzvf6*rCJbiU~*#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z;qv^xQyyw=ptp+zXTSr9xdQVUwFJ5Nxj4cS UbM$lai;G29ud1_MGDW-`0CAoxmH+?% diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294845.sc14.2671192.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294845.sc14.2671192.13 deleted file mode 100644 index 70eee611f0a384662cc5ddbf2c09916e65ceec06..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv{UJi}tbiU~*#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z`Ep#VQWa|Mt376)IlD~4!R87aWYQAn66N9x z0U58Ko|+e*lV4mc%D`~ZZr2T`WHqR!V*kesoV=z8O%hyUT>KDCDVfE|P+bq2Y)rb2 GgaQC;XFl@) diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294848.sc14.2671192.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294848.sc14.2671192.14 deleted file mode 100644 index 49ced80eb27b174543cde6f1ce8c588f92f5b4e4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw$-!V?$biU~*#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z*{>ddL=|f8t376)IbW%ng3T2;#H1z8CCbGY z0y17dJvA>rC%?E@l%e68onzlz2{ovuV*kesocZbqO%hyUT>KDCDVfE|P+bg7Hp@Am GF9QG+tUdw& diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294851.sc14.2671192.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294851.sc14.2671192.15 deleted file mode 100644 index 4ea3341d2f9123a9bf372335cc093cb96ecaef2b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzv%c|J?vbiU~*#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z3Efk&LLO>vptp+zr|Rf)Flq^M@pEy6 VCFbbo3q{siZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-QrCIrw<^@!S9{DpbG}Vx2b(K!m`O{VOO%T* z1Z2E^dTL&LPJVH*CNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zv6K#uSB0AUYLD4xPR8$#!R87aVbT)k66N9x z0U58Ko|+e*lV4mc%HVL>E=Fa~IyI=KV*kesoI8*-NpOjA@k2DFWELkwbv5MK2<_k4 F1OQz~Kmq^& diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294862.sc14.2671192.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294862.sc14.2671192.18 deleted file mode 100644 index ff747579f29594b79bc5e8016783f7d339133b0d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzw)VV#o9>3q{siZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!o$1$t~}J-KyMcb&c0TNxdICrwFJ5Nxj4cS UbM$lai;G2bH#ORF9W2}h0CJftp8x;= diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294866.sc14.2671192.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695294866.sc14.2671192.19 deleted file mode 100644 index efb95a3d32a39fd833ff9a81708d5dd92b078a0c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwytlyBr>3q{siZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(h{kgp$awk)gH6YoUGqBg3T2;%A_UECCbGY z0y17dJvA>rC%?E@l;OY)yEJRVYBi{)V*kesoS8_PB)G)5_#v87GK-U;x)}0o;@1eW F0sxjaK9c|d diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295534.sc14.2696291.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295534.sc14.2696291.0 deleted file mode 100644 index 57df03f7e946aa8a65ad6e2864535ce4dce5c535..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw~Q0U<1a=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zv32<{OBHJFt376)IqgqF%oRAsq$SQJ%EcD~ zGG0GDH7`CVzqnYG;lT|%3z^Gj)S#M*{U0-MENT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZDPxe!m4})e=rC%?E@l;Od3yNNpj>eQf`iv1rmaO$r>Xp-O(gdFlmW%iE{CU zfQ;8qPtA+Z$uBMzWl*?iw`+@UgBny*vHxQR&Pzy|B)G)5_#v87GK-U;x*Do%1UG6P F1pswIJzoF- diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295547.sc14.2696291.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295547.sc14.2696291.4 deleted file mode 100644 index ea6bd2535c757e4f343c21ec2179cc58252a72fc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzvXZoJON<$TjoiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-BKOVfhdk8WKyMcb&UC?>U~>f)GinKP@pEy6 VCFbboYB$Jjnmnaus z2*`N-^whlgoc!WqQ3i%z#UDcqPiv1rmaPl-GG)ZuYaq&YmrDPT-Lv=kUw&9dH G6Ab{+0zWeV diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295555.sc14.2696291.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295555.sc14.2696291.6 deleted file mode 100644 index 42964cb3ab0793c97e268af0e0cfffcd90f1cff6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxFWxFE4<$TjoiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!q+BtQWa|Mt376)IYUyeg3T2;#iS+9CCbGY z0y17dJvA>rC%?E@lwrXoyXRWxPpCmP75hJC;9QENNrFp^iyxvXC9^mgrYqm(lzE&s E05j-6MF0Q* diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295558.sc14.2696291.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295558.sc14.2696291.7 deleted file mode 100644 index fcd2523ae8ac7ae3da709c89b872d539756017ff..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxR+qMgGIp1`Y;!P?_%*@ksElbTSu`&wM(A!0VbNwNRxdKZVwFJ5Nxj4cS UbM$lai;G2nJgBt2+wHUj06q~cGynhq diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295562.sc14.2696291.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295562.sc14.2696291.8 deleted file mode 100644 index ecded2ae747e60d44ac599a5a6b22984df0ae74a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv9|F=?z%lW3G6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq`ft3y(-k)S9{DpbBgw_0-Gyvnn_EXOO%T* z1Z2E^dTL&LPJVH*D1*XvyF?E21U0CpV*kesoM(_UNpOjA@k2DFWELkwbvcyT{9@35 F1OTNeKH~rY diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295565.sc14.2696291.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295565.sc14.2696291.9 deleted file mode 100644 index e16d2df4816fee0f6160d396600523d2c98f55fe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv5Yu5;KIp1`Y;!P?_%*@ksElbTSu`&wMh|FlmW%iE{CU zfQ;8qPtA+Z$uBMzWpKD|_j{L@pc+(DvHxQR&iP21B)G)5_#v87GK-U;x)`c$B2LzC F0{}#RJh1=( diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295568.sc14.2696291.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295568.sc14.2696291.10 deleted file mode 100644 index 74ec7b990641a0b42d124698f3dbaaec1bc9c870..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzvXi|UJTIp1`Y;!P?_%*@ksElbTSu`&wMh|GHHo(iE{CU zfQ;8qPtA+Z$uBMzWoS5W*O1HhSPiPF*#9vD=PM*l5?o?j{18nknZ?OaT@EcaN|Blt E0C?3rivR!s diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295576.sc14.2696291.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295576.sc14.2696291.12 deleted file mode 100644 index 2a142b54484f85829ffef32becda3a93f4062b12..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvf`C%-^<$TjoiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-B3c%cs0uar)gH6YoTh|F=>f&iE{CU zfQ;8qPtA+Z$uBMzWpKD;7e7CvM-8f}*#9vD=Po2o5?o?j{18nknZ?OaT?d+MKA*U* F0RRZRKG6UG diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295579.sc14.2696291.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295579.sc14.2696291.13 deleted file mode 100644 index 78096f77f5816060991e5b1e5de4b5ce7bbbe908..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzvvEqWow<$TjoiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(zvYcqCC{xKyMcb&g~B%<_auh)Dq<4=i&%U U%+b%uFD@1}yBTLYaXYgV0NTMUn*aa+ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295584.sc14.2696291.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295584.sc14.2696291.14 deleted file mode 100644 index f3b092e7a47acfbedaf63258c05b1b14cb61d174..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvT|a=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z`Msp+wkp)zS9{Dpb3T0tF<0O`la@G_C>LJ{ z$awwq)V%nd{NiF!h6k7IEWb5>RD)_N_J7R4nfM5yNrFp^iyxvXC9^mgs!O5C=B4ew F3;@*>K|cTh diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295587.sc14.2696291.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295587.sc14.2696291.15 deleted file mode 100644 index f7cfb0f4f0989f797875792bcde013f94b249e2c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw$ay*mZa=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZG3fZBt_n5x)gH6YoWYZygUuDVz@#P4CCbGY z0y17dJvA>rC%?E@lp*1gT|uSs95tw>V*kesoX3$gNpOjA@k2DFWELmGbXD8<$8A^y E00VwM82|tP diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295590.sc14.2696291.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295590.sc14.2696291.16 deleted file mode 100644 index 4cc1f39fbb4793bbef5db9c3d717f2dbc5799dce..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxVpFUHP%lW3G6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWQOaN1bL{rf!;0>oD&#kgUuCK&Zs5G#m~hN VmYAcTlV4mcIy)!B_91_>J^-xFD?k7M diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295594.sc14.2696291.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295594.sc14.2696291.17 deleted file mode 100644 index 2b57a5a569cd5b27fb9eade43ad899d5c1fe94cb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv*sqd8Ha=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZkqhH&R)w1TYLD4x&i`w6fz1`T$fPCCCCbGY z0y17dJvA>rC%?E@lp*1go%jdQg=$bu#r}^OI47({Xp-O(EfOl#*GT4AsR@WwVvN GzzP7*fj^=E diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295601.sc14.2696291.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695295601.sc14.2696291.19 deleted file mode 100644 index 2ee15b81428fe65c4d5b53de880759261fa99ff6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxtaC*ydIp1`Y;!P?_%*@ksElbTSu`&wMrC%?E@l;OcSJGL!lacWRa#r}^OIPW28lHd~K;)iHT$t+HW>T;;Fne@KP F0RX;(KzjfH diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296482.sc14.2714192.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296482.sc14.2714192.1 deleted file mode 100644 index a98a26608951ae08b95d0846a4c3cca4a43ae40e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwC*iQMu<$TjoiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GP_OVqbk(gS9{Dpb2c5E3N}~Z3X_&Nmnaus z2*`N-^whlgoc!WqQHBHO?4(ajDN=)KD)xWOz$tJDp-F;EjEf(lDJ8Qw8LBIx!KU_4 G?L+`|v_iQ6 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296485.sc14.2714192.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296485.sc14.2714192.2 deleted file mode 100644 index 2cd2cfc66097545e3862ef1c8d0fdbcf1611be3d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzu=FY^4!<$TjoiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-BG#P6Cl56@(A!0V^DwtJ*j#~?j9P+R{9GJi Vi8=Z?`NhSe9P(+lFE;O)4ghzXD@gzV diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296489.sc14.2714192.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296489.sc14.2714192.3 deleted file mode 100644 index d410692bf96a1a5b3f65eaaa61f4aed2a51ae27f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw0br$^Ma=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z37yz~P!($Kt376)IVW%zg3T4U%A_UECCbGY z0y17dJvA>rC%?E@lp*1?-F4j>4mGHzV*kesod1zDNpOjA@k2DFWELkwbuDPJnfZQs FIsoAqL0kX; diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296493.sc14.2714192.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296493.sc14.2714192.4 deleted file mode 100644 index 50e7ca521ebf29b6e3b260cd32069630e464aff4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxVQ;Po0<$TjoiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GIjk-K2@l>ulAUI=6o?N25hdtH6|@_E>SMN z5Rmcu>8W|~Ir+uKq6`UV?R3`N^HzguD)xWOz!@+dp-F;EjEf(lDJ8Qw8LDeRiw)18 GMn3>XUO}V) diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296496.sc14.2714192.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296496.sc14.2714192.5 deleted file mode 100644 index 691c2d5d0f48b759f8ca57b20533cb935ad44c5e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxBZsGsK<$TjoiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@^Pt^t~}J-KyMcb&dJ<@U~>glF=`2N@pEy6 VCFbbodVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWU`Wvsw&jnS9{Dpb2_rC%?E@l;OcSyVKk4PpCmP75hJC;9P;ENrFp^iyxvXC9^mgs;i;hrt;(F F^8gbZLLC4A diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296504.sc14.2714192.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296504.sc14.2714192.7 deleted file mode 100644 index 36bb8537083d431db625982b90e56b829712becc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvb%P9Tha=z&(#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZiTLn-l`7QSS9{Dpb3Q0g2AeBzgGo!AOO%T* z1Z2E^dTL&LPJVH*D8qp>b`sU8wrWsK#r}^OI4cVgnk2ZyxcDKOQZkE^p}H74ZEANh G_yGXV^FP!8 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296507.sc14.2714192.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296507.sc14.2714192.8 deleted file mode 100644 index b790e251d720d0b081b58d55a32da34ed9b6ca95..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzwmuDtw@%lW3G6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vL`c@xS`}*Ut376)IX|5G0XA3QCX<#pmnaus z2*`N-^whlgoc!WqQHF$bb`yRt+o1;4RP6tlfivNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zx$e62u{_k=KyMcb&a{^ha|PBgY6)`jb8&dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v)WvHxQRPF53yCJ8PvE`Ernl+5B}sICR&HietR Fc>#ozJe2?d diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296528.sc14.2714192.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296528.sc14.2714192.14 deleted file mode 100644 index f24e19c207573ae60f1e39207bb8b935f810c537..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzw`*Qv2^JKuDa;!P?_%*@ksElbTSu`&wMNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZiMVBXMipxAt376)IX}(t1)D2ymq|;UOO%T* z1Z2E^dTL&LPJVH*CNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZQ9u5PM-^)Bt376)Ijvqn%oVuDq$SQJ%EcD~ zGG0GDH7`CVzqnYGA>oppPFT%PHK?Xy|HllRGm$h&aEWp8Lo}si7AHe>F=W~->N~$4 E0Q@CB)&Kwi diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296539.sc14.2714192.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296539.sc14.2714192.17 deleted file mode 100644 index f29744a2bac8196a8c2499e79aa9b23b038ebfe6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxlJ@A-~+xe!W6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?voUOc1!R89AW7HDl;^*QB VOU%*F$uBMzUA?NxHpko54gk&3E0_QP diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296544.sc14.2714192.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695296544.sc14.2714192.18 deleted file mode 100644 index 92be09368e75b38c1993390da1dfd49394c1cbd1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwSbF*-8JKuDa;!P?_%*@ksElbTSu`&wMdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWcw_?8LCioU+ppb%qg%7Vy?ggCM|I;Q7*m^ zkn#HIsd@1^`NhSe3=b~b#b1#vQG;qK_J7R4S%RcVf=i5xAEGHGvp5;5DdVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vB$mVZk}A~PS9{DpbH0o|3N}~ZA(NIkmnaus z2*`N-^whlgoc!WqQHBGT?J5`Q>`{YiD)xWOz!@Hc&?Lbn#>EfOl#*GT4Ate3X!9ei GfD-^(C_!8R diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297326.sc14.2733087.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297326.sc14.2733087.3 deleted file mode 100644 index 69bf7635fb0c2b5ad4acbd0d84d107f2249bf7a7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxFsat)T+xe!W6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWXmn{{i;xNU+ppb%*i0R25hdtBPK0zE>SMN z5Rmcu>8W|~Ir+uKq6`j~>{M&srKmwQ75hJC;PgY%B*7)d#ShVxl3AP#)um8s!`F7f F9snWhKmPy# diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297329.sc14.2733087.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297329.sc14.2733087.4 deleted file mode 100644 index 6f8366a1400201d267618ae169b5ef2bd3ae4b83..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzwSypB7=?R?WwiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-GHdPERq{}C1HD}&IL(|Az~%~UVAK-i;^*QB VOU%*F$uBMz<&aOXO_Opv1_1VVE5iT) diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297334.sc14.2733087.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297334.sc14.2733087.5 deleted file mode 100644 index 15cf8b288185b1e42b253140a60b2b9b7b4f7f6e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzw;-)uh1?R?WwiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-B7F0aw<^@!S9{DpbKXg41)D4Im`O{VOO%T* z1Z2E^dTL&LPJVH*D8qwGcFwUAgVdm!iv1rmaJDBRG)ZuYaq&YmrDPT-LvvRd G-vR&})IcQw diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297340.sc14.2733087.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297340.sc14.2733087.7 deleted file mode 100644 index 2271279208cecb823dfbdc7157f04d34de5060dc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzv<7gjsZ?R?WwiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-VkA{`Ode`(ptp+z=YwcQow30|1M)D^36a diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297345.sc14.2733087.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297345.sc14.2733087.8 deleted file mode 100644 index 78874e99b8d43b833e2dc5e21b532ef680224af2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxl6Lq`5?R?WwiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vgyR-?W$07U+ppb%qddq4mMZdDU+5smnaus z2*`N-^whlgoc!WqQ3i%9cE(zIwrWsK#r}^OI9ri4NpOjA@k2DFWELkwbuGxXxt|f} F3;?VNKX?ED diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297348.sc14.2733087.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297348.sc14.2733087.9 deleted file mode 100644 index 6beee5fff1bc4b36346f4be39a5195e98f3e16d9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvTi11(JcE0H-#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZkvB4BRE3)RYLD4x&W}j~U~>hYF=>f&iE{CU zfQ;8qPtA+Z$uBMzWl*?ar+kvlT@9+K*#9vDXJ#@&lLVI-7e7Q(N@j5~RM&%Ko5Lxq G{r~_ooJWp8%oW(gs3pk7&&3gz Un4_PQUtBCYJEy@`uVCMD00RRqQUCw| diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297355.sc14.2733087.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297355.sc14.2733087.11 deleted file mode 100644 index b40649ebd339f1260db1558661e69329a9c6125c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv*&cAz!+xe!W6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?v#6D1rC%?E@l)>S;U0FPfsTx#MvHxQR&VD3K5?o?j{18nknZ?O4UG+9nV#@CU Dv|&FF diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297358.sc14.2733087.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297358.sc14.2733087.12 deleted file mode 100644 index b653a4299abf518298130aaee349f8c116613a91..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwqcinuM+xe!W6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWY^(Cj;c^|U+ppb%z0+(7O=SjFPOB%xkS16 zLO{mrr>Ew{=j0a`i!wA^v@?0vE~Ez4RP6tlfzx;!LX!lS7#BZ8Q%YuWGE7&aO@{NU FL;%tOLg4@a diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297361.sc14.2733087.13 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297361.sc14.2733087.13 deleted file mode 100644 index c4962654df85cf1c34771e932016f87db5c10ea8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxx-b}v2?R?WwiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!u_ezNgir$ptp+zXJkSu*j$0lj9P+R{9GJi Vi8=Z?`NhSep`o$1K4;hT0|3VDEFu5^ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297366.sc14.2733087.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297366.sc14.2733087.14 deleted file mode 100644 index 9fc9746d7222bd1be22d644e768e99aca98ea737..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxpKW@6p?R?WwiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!tu)Qk1EvMS9{DpbG{912AeDJl1WRPOO%T* z1Z2E^dTL&LPJVH*CNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZF{(~eRE3)RYLD4xPWJ_VU~>gtF=>f&iE{CU zfQ;8qPtA+Z$uBMzWjJubPJhCEH8rTFV*kesoU4&ENpOjA@k2DFWELmGbXC}F?BAXT E06SMecmMzZ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297372.sc14.2733087.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297372.sc14.2733087.16 deleted file mode 100644 index 930287729966809c8aa455beae2f280099429107..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxB$S_~$cE0H-#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z;X2(GCl56@(A!0V^XOEFxdK}lwFJ5Nxj4cS UbM$lai;G1C>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vgvo2Nm@3rVS9{DpbNZNRg3T3p&7>vHCCbGY z0y17dJvA>rC%?E@lwrXIyQ?+FeAJ+tiv1rma6U)UB*7)d#ShVxl3AP#)zwgDlNFPv F3ILL_KWP8} diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297380.sc14.2733087.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297380.sc14.2733087.18 deleted file mode 100644 index 75e8b879cb72ce29983d603071b1a3296b460a2b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzu+y%)O4?R?WwiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-Qm1-GMipxAt376)Ip1W9fXx+n!=xq7CCbGY z0y17dJvA>rC%?E@ltJOLolf~O9W|(?V*kesoZdMIO%hyUT>KDCDVfE|P+br5Y;4lI G{{sM$lRwq~ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297383.sc14.2733087.19 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695297383.sc14.2733087.19 deleted file mode 100644 index dcd2258e5dbf35f063eaa0129e8cc1a5364ddcfa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzw;UORb{+xe!W6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vgm+C3k37`eKyMcb&ht}Ffz1`z%BUsC#m~hN VmYAcTlV4mcdP=R?c3-s-I{@eSEPDU| diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298153.sc9.1458633.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298153.sc9.1458633.0 deleted file mode 100644 index af676e43a007b7ea365eb85b7c327c156ce0e39a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzwqvJ0HW<9yRmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-()!nUvpm$?KyMcbPT8vva|O0CY6)`jb8&Nd>galYv&#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZVHOowt_n5x)gH6YoX@5}%oTXcq$SQJ%EcD~ zGG0GDH7`CVzqnYGf#JHHB5%btHK?Xy|HllRK2s5zB)G)5_#v87GK-U;x*UpaE-ttF F4FJlWKD__{ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298161.sc9.1458633.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298161.sc9.1458633.2 deleted file mode 100644 index 7a051472f85276a64acf108cb4443b01615d8ebe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxNvHQ>AalYv&#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZS)??5l`7QSS9{Dpa~=;10GliDj!8?LOO%T* z1Z2E^dTL&LPJVH*C_}>)JLgcx`D##2#r}^OI1R%Qnk2ZyxcDKOQZkE^VY;$xe)MYG F0suZfKZ*bV diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298164.sc9.1458633.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298164.sc9.1458633.3 deleted file mode 100644 index cc3d84a7c11d2269525c25d01dcc4c90b8e76ec6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzv*onV;D<9yRmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vgM=US$U|rf!;0>oQJ{~!R88VW7HDl;^*QB VOU%*F$uBMzot+b7`|c^rY5O!esQrVgTe(nzmF}~)S#M*{U0-MirqnIlHd~K;)iHT$t+HW>UxlAlXBs| FAOOiwK{@~c diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298172.sc9.1458633.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298172.sc9.1458633.5 deleted file mode 100644 index 38c7b38e08d3140df774fdd671e7c6b317a561e1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxVkz}9G<9yRmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^7i28X{u0jU+ppb%&FVX0XA3Q1Cy3Gmnaus z2*`N-^whlgoc!WqQHBE-?G#F`&r^eHD)xWOz&Q~~lLVI-7e7Q(N@j5~RM&$Pn~Ucz GYy<${c0x%2 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298174.sc9.1458633.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298174.sc9.1458633.6 deleted file mode 100644 index 2d8f5956ebb0d9f9b1f745e2b2f5c35d6b8d4d9f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxFR^L6J$N8qC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vMDS23w>;F`KyMcbPS318U~>hwGinKP@pEy6 VCFbbodVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWM1``C8|(!U+ppb%qh6^A=q4jk4#$PT%ufj zAt2-R(^K=}bMlLeMHvoUvvU#2=~shlD)xWOz}btWNrFp^iyxvXC9^mgs>>nW=8b;F F9RRNdLYM#m diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298183.sc9.1458633.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298183.sc9.1458633.8 deleted file mode 100644 index 463eb640fd0919b12ea13020f5a66c4180cbd89a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwG3msa><9yRmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-BA*jqqY5?m)gH6YoHjEc<_dgb(h}zq<>Ctg z8Lyw7nirpwUtBE8&~Vw#v0?rzHK?Xy|HllRlaMq?aEWp8Lo}si7AHe>IV9RdzHr+Q E0GWM2(EtDd diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298185.sc9.1458633.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298185.sc9.1458633.9 deleted file mode 100644 index 349db945d421866636f972602e11c1c20e69cd16..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzv*-6>hb<9yRmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-QZ0V7S{`a{ptp+zr+Zx)*j#}fj9P+R{9GJi Ui8=Z?`NhSehuA}H>utE10JWPcoB#j- diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298190.sc9.1458633.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298190.sc9.1458633.10 deleted file mode 100644 index 7ee8630c53f2b7164e5c71df5535f1fafaf4126a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwq)t|eV$N8qC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?va)U+ppb%*niN9@t!g&rDk4T%ufj zAt2-R(^K=}bMlLeMHvoUu(Me5!&D8bso4K91LtNWO%hyUT>KDCDVfE|P+baXHjk54 GegFVs>q1ii diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298193.sc9.1458633.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298193.sc9.1458633.11 deleted file mode 100644 index b40790fc1be8e6429132e1c5a468f764afba5ad1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv9e_g(W$N8qC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vMDarCaaE|fulAUI<~-_D2{u>Y3zL>Omnaus z2*`N-^whlgoc!WqQHBGT?V9bqs@0&Hiv1rmaPm1LG)ZuYaq&YmrDPT-LvNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZF>&5{NqlD)xWOz40Bto13 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298205.sc9.1458633.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298205.sc9.1458633.14 deleted file mode 100644 index ae366902375e96d957f108a5c7388c5de4f466c7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxhXuB-ualYv&#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIhs*4OBHJFt376)IXTz5g3T59#-t_ACCbGY z0y17dJvA>rC%?E@l)>SW-HmC&)77Awiv1rma7H3&lHd~K;)iHT$t+HW>RQlh^JBY+ FDgZk=Kd}G+ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298207.sc9.1458633.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298207.sc9.1458633.15 deleted file mode 100644 index 050ac2ff6fa26464048cc3044e8934ad6790f511..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxxmi=AM<9yRmiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-B3)O-E)O*~(A!0V^U}nBU~>g_F=`2N@pEy6 VCFbbofaOO%T* z1Z2E^dTL&LPJVH*CR6 F6#$@aKx6;_ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298218.sc9.1458633.18 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298218.sc9.1458633.18 deleted file mode 100644 index c9516e3f2ff78c38e7ce6970b7f0ec7658976ef4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxho!`ES$N8qC6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vWa+urqViC41HD}&IE{mMg3T4!&8Q{F#m~hN VmYAcTlV4mc8e39rn`+1$0s!kQE4=^! diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298994.sc14.2774085.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298994.sc14.2774085.0 deleted file mode 100644 index cb6dd8356a47f032bae8dab2d31fc9fd63927fe0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxxl+MiIb-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zv7Y$#mnziUS9{DpbKY4x3v8~yPbMvKE>SMN z5Rmcu>8W|~Ir+uKq6`U_?H)0Gw^oB{D)xWOz?r%Xp-F;EjEf(lDJ8Qw8Kx`O=I9lv Fs{k+NLMs3O diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298996.sc14.2774085.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298996.sc14.2774085.1 deleted file mode 100644 index d733b1643666a8b914a5bbc8a75827c9d56a76bd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzu+ZcxnSb-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZS@F*Ck1EvMS9{DpbG~v_0-G!Fi%CnIOO%T* z1Z2E^dTL&LPJVH*Co<@6@20iv1rmaK^hKG)ZuYaq&YmrDPT-Lv=ai*!;R= Gegy!v*Fe<( diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298999.sc14.2774085.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695298999.sc14.2774085.2 deleted file mode 100644 index 73a7b86729f63e33e85bcb526b8a13e16b36c745..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzwyXNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZF|B1gA`dk;(A!0Vb5{2?u(<+z7_|hs__;X3 V5_9x(@{5Z_SFg&mz1MVA5&)8tER_HN diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299003.sc14.2774085.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299003.sc14.2774085.3 deleted file mode 100644 index 298059ecf2c8e29ee1a97e8f70e4dad070e1d9b7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxVwMEuiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-!t?&p3RS4NulAUI=471;F<0O>la@G_C>LJ{ z$awwq)V%nd{NiF!28By@dwMEuiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-^6cofTve#KulAUI=3F0t4Q#H!A0{nvE>SMN z5Rmcu>8W|~Ir+uKq6`g}?KHNDO;m$wD)xWOz^R^q&?Lbn#>EfOl#*GT4ApfY(MBZE G@CX2q9zkXR diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299010.sc14.2774085.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299010.sc14.2774085.5 deleted file mode 100644 index 8d2b7fcc8a505c9eceb3d564cc4b4b2b57d96b0b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzwqeb`;V>wMEuiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-a{g{}xIEO{KyMcb&VH9(u(<+z8MOqt__;X3 V5_9x(@{5Z_7k78r&gZ}03;-9tEsX#G diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299014.sc14.2774085.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299014.sc14.2774085.6 deleted file mode 100644 index ba5920d36f09ec92f53ff8c66e4c56216e9682c4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwKjb2yC>wMEuiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-q8_AvQx$6Nt376)IXCTJ4>nieFO!xymnaus z2*`N-^whlgoc!WqQHF%8cF&~O+o(Y`75hJC;0!u|&?Lbn#>EfOl#*GT4Ar$D)`oM{ Gp>F^mdqJ=O diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299018.sc14.2774085.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299018.sc14.2774085.7 deleted file mode 100644 index 59aded474cc54941cd69eb72e7e72e7ec5af2aea..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwK`P5#->wMEuiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vSa6JepRTsulAUI<}6F^1e+`Hk4a0MOO%T* z1Z2E^dTL&LPJVH*C_}?#JCT0hy=qWR#r}^OI4>Y+lHd~K;)iHT$t+HW>N*f+6EszG F9sqK7K~?|& diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299020.sc14.2774085.8 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299020.sc14.2774085.8 deleted file mode 100644 index 2b992c93b3cc6bed5734981f91d5de58e66b4539..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzwKa<(nzb-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZIl?12Q66e;ptp+zr?Idd*j#~qj9P+R{9GJi Vi8=Z?`NhSe`n!s3PtKV=1psJ~D+~Yt diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299025.sc14.2774085.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299025.sc14.2774085.9 deleted file mode 100644 index cb5f669235406ebd729ae074775e29e2bbbc7d71..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv*Oo}Vvb-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zxw2jNfGX77S9{DpbAI?94>nieKa-X?mnaus z2*`N-^whlgoc!WqQ3i)gb~$2K&ZEfOl#*GT4AsRDX>+?V G%Mbts#z6M~ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299029.sc14.2774085.10 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299029.sc14.2774085.10 deleted file mode 100644 index c11c6f2520b287c7c78835a07e84decf5a053082..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzv<>+~q)b-w8+#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZiMw89qzX0n)gH6YoJNdZU~>f;n6<>YM7j7v zK*sB*r{=}yNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Zk@uQ#UKMKYt376)IYrC0z~%}xGHZ!*iE{CU zfQ;8qPtA+Z$uBMzWms^@&fG5kzZz6ivHxQR&K4w15?o?j{18nknZ?OaT@UHz~%}xF>8r)iE{CU zfQ;8qPtA+Z$uBMzWk|SUH!J63kQ!7|vHxQR&hYyPO%hyUT>KDCDVfE|P+bS&Y*w2m GrUL*2F+Rco diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299042.sc14.2774085.14 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299042.sc14.2774085.14 deleted file mode 100644 index 12a60b5b2f39d5fe95c774ee22b1a03247631950..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzv<+_dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vfyFlq^M@pEy6 VCFbbowMEuiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-;<#kRXH}@VulAUI=G^=IA=q4jW@asME>SMN z5Rmcu>8W|~Ir+uKq6`Nv*~#6g7g2+1D)xWOz?tv@p-F;EjEf(lDJ8Qw8LI0+rp=sa Gt$qO2Cqjz= diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299050.sc14.2774085.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299050.sc14.2774085.16 deleted file mode 100644 index 3f0da80d262a19cef98dec5d1433f1c4dc0e923c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwqwMEuiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-@@D7ddR3^oulAUI<`hfW4>nhzg;`6SOO%T* z1Z2E^dTL&LPJVH*D8qrvc1!Kq6V#xZiv1rmaIQkqB*7)d#ShVxl3AP#)uoVaWA^y# FbpY*5K-&NS diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299053.sc14.2774085.17 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299053.sc14.2774085.17 deleted file mode 100644 index 1a34f0490fec5be60db0624d5cad8ffa0cfe455b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzxR`W9F7I^T4Z;!P?_%*@ksElbTSu`&wMNT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZFwMEuiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-(lu+2t}4{rS9{DpbN;q20-Gz)#;hgICCbGY z0y17dJvA>rC%?E@l;Oc;J8Q<&UusZI#r}^OI6G_*nk2ZyxcDKOQZkE^p}HKBY^)O% F@&VtCKGOgI diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299832.sc11.2623573.0 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299832.sc11.2623573.0 deleted file mode 100644 index 2ff3c5eea602cde1122787a97f722ffc9e8ea79e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvvT4Q6!=X}#qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-azL$fo+{MbS9{DpbMDi%1)D3-#;hgICCbGY z0y17dJvA>rC%?E@l;Oc;yIBj|y49eXiv1rmaE9n1G)ZuYaq&YmrDPT-!*pfXG+cNU F2mm)kKTH4s diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299835.sc11.2623573.1 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299835.sc11.2623573.1 deleted file mode 100644 index 6cceb1d2777ef7747335f13bac949a22547c0bcc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzwqnDW|=&-tdK6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq%_u(TOMj|ptp+zrv~pEu(<+<7_|hs__;X3 V5_9x(@{5Z_IpjNR8<$L00syTBDrf)z diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299840.sc11.2623573.2 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299840.sc11.2623573.2 deleted file mode 100644 index baece8e5db66f8cccf42f5a881de5e7070a3299e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwST8TLDIp1`Y;!P?_%*@ksElbTSu`&wMrC%?E@lwrYjyC(+^pI3uwD)xWOz$v!^p-F;EjEf(lDJ8Qw8LCSm&8B0* GzES`Y9X}iZ diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299843.sc11.2623573.3 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299843.sc11.2623573.3 deleted file mode 100644 index 9f92a2f210ac4d03030985746358649963f0f2d4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwK6Z_`C=X}#qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-BFXtUSruyTt376)Ia{;8gUuD_VAc}n66N9x z0U58Ko|+e*lV4mc%Ajz;&iLu9N;RmaV*kesocEA4NpOjA@k2DFWELkwbsb2znIPA5 F7XYY;KtTWi diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299846.sc11.2623573.4 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299846.sc11.2623573.4 deleted file mode 100644 index 150931dae4084ecb21e6eb888f1fbacefb255ff4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzwqv|Q=P=X}#qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-V#xEZN*-!%ptp+zr+&{Wu(<+<8MOqt__;X3 V5_9x(@{5Z_1?+Nc-8B*x0RV%mDmVZD diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299851.sc11.2623573.5 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299851.sc11.2623573.5 deleted file mode 100644 index 22e8fe612d785d5439f3991fff699a691adcf4c7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzx#o^rv7&-tdK6mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBrM*}QKepaQD#YMkzOiDReV}zPHH?vq^C^XPZetJt376)Ih{9M1e+_+$*d*LCCbGY z0y17dJvA>rC%?E@ltJOL-R+%WY-&(V#r}^OIHw_LlHd~K;)iHT$t+HW>SD;UQDB*{ F5dhH4KE?n5 diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299854.sc11.2623573.6 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299854.sc11.2623573.6 deleted file mode 100644 index d757e9115afc66f465e8241d6b2ed6a61285ce4b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwK_=X}#qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-V%EodK^1E5t376)ISrCFfXx-?V%8Gp66N9x z0U58Ko|+e*lV4mc%CO*q-3qN6lhvS_iv1rmaBf1Bp F1^`r0K0N>c diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299857.sc11.2623573.7 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299857.sc11.2623573.7 deleted file mode 100644 index 3b4503a91326bbfaa35943cb740a0c04a8d592d6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzvTv}U^SIp1`Y;!P?_%*@ksElbTSu`&wMROOyGksRu FX8=FyKjQ!Z diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299865.sc11.2623573.9 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299865.sc11.2623573.9 deleted file mode 100644 index 71b176da9e0851eb3b8f26c7e64f9f6cb19dd763..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzxps^+=zIp1`Y;!P?_%*@ksElbTSu`&wMg}n6<>YM7j7v zK*sB*r{=}y)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z5uA41Ngir$ptp+zXXHyIu(<+98MOqt__;X3 V5_9x(@{5Z_d-TF=rz_s$2LNBfDs=z= diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299873.sc11.2623573.11 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299873.sc11.2623573.11 deleted file mode 100644 index 2ff8173b90bdf456c878758da7d4d50097efc854..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzx#yyWV^=X}#qiZ`h!F*8rkwJbHS#L6g0k4vW{HLp0oC@DX&C`GTh zG&eV~s8X-ID6=HBNG}znDn2bUCp8`-vRJ*`RuyXQt376)IRzr!!R89|GHZ!*iE{CU zfQ;8qPtA+Z$uBMzWoWo;_rCcMyBbtevHxQR&UPeC5?o?j{18nknZ?O4U6D2`rW>9E E07(Eog8%>k diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299876.sc11.2623573.12 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299876.sc11.2623573.12 deleted file mode 100644 index 60cfa1eccc104277a96b156bb39016d9e20d7aac..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzvrW-5B}Ip1`Y;!P?_%*@ksElbTSu`&wMs6x$swa4r;r}2Fyu(<+#%v$1HqFj6- zAmjDZQ}g0;@{5Z_85Uf!_h|KRy5e diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299887.sc11.2623573.15 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299887.sc11.2623573.15 deleted file mode 100644 index f67eb392cbd7da285e75104c2d0fab40b6870506..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 195 zcmeZZfPjCKJmzwyv%dA_bH3>)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{Z;g;8MP=%WNYLD4x&iwRuU~>f~Fl&i(iE{CU zfQ;8qPtA+Z$uBMzWnj2ucPCcSTMeqI*#9vDCrbuGlLVI-7e7Q(N@j5~RM&$@n?&LJ FeE^YKJt6=A diff --git a/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299890.sc11.2623573.16 b/monai_brats_mri_2d/exp_1855 (autoencoder)/tb/events.out.tfevents.1695299890.sc11.2623573.16 deleted file mode 100644 index 0ab62d033fa12415963fbf29363581c3a9a79c17..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 135 zcmeZZfPjCKJmzvbnlJI;bH3>)#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$ znwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZnOFAWf;`mRKyMcb&Us6hg3T2;&Zs5G#m~hN VmYAcTlV4mcT5>Vj)?xM~O8}Y5EYJV| diff --git a/monai_brats_mri_2d/loops.py b/monai_brats_mri_2d/loops.py index d60d5cb7..7d65f910 100644 --- a/monai_brats_mri_2d/loops.py +++ b/monai_brats_mri_2d/loops.py @@ -7,6 +7,7 @@ from generative.losses.adversarial_loss import PatchAdversarialLoss from torch.utils.tensorboard import SummaryWriter +from torchvision.utils import make_grid ## -- AUTO-ENCODER - ## @@ -190,12 +191,20 @@ def evaluate_generator( val_step = val_sampler.progress // val_loader.batch_size if val_step == total_steps: + val_loss = metrics["val"].agg["val_loss"] / metrics["val"].agg["val_images_seen"] if utils.is_main_process(): + writer = SummaryWriter(log_dir=args.tboard_path) writer.add_scalar("Val/loss", val_loss, epoch) + + image_pair = [images[0], reconstruction[0]] + grid = make_grid(image_pair) + writer.add_image('Val/images', grid, epoch) + # writer.add_graph(model, images) writer.flush() writer.close() + print(f"Epoch {epoch} val loss: {val_loss:.4f}") metrics["val"].end_epoch() diff --git a/monai_brats_mri_2d/train_cycling_gen.py b/monai_brats_mri_2d/train_cycling_gen.py index 7d3683bb..e2938eb5 100644 --- a/monai_brats_mri_2d/train_cycling_gen.py +++ b/monai_brats_mri_2d/train_cycling_gen.py @@ -174,7 +174,7 @@ def main(args, timer): ## -- TRAINING THE AUTO-ENCODER - ## - n_gen_epochs = 200 + n_gen_epochs = 100_000 gen_val_interval = 1 for epoch in range(args.start_epoch, n_gen_epochs): diff --git a/tv-detection/coco_eval.py b/tv-detection/coco_eval.py index 91b4269c..2a852b39 100644 --- a/tv-detection/coco_eval.py +++ b/tv-detection/coco_eval.py @@ -28,7 +28,9 @@ def __init__(self, coco_gt, iou_types): def update(self, predictions): ''' predictions = { - image_id: {"boxes": tensor(n x 4), "labels": tensor(n,), "scores": tensor(n,), "masks": tensor(n x 1 x H x W)}, # for n < N detections + image_id: { + "boxes": tensor(n x 4), "labels": tensor(n,), "scores": tensor(n,), "masks": tensor(n x 1 x H x W) + }, # for n < N detections ... for image_id in batch } ''' diff --git a/tv-detection/engine.py b/tv-detection/engine.py index 54b76712..4b64de73 100644 --- a/tv-detection/engine.py +++ b/tv-detection/engine.py @@ -139,15 +139,13 @@ def evaluate( timer.report('starting evaluation routine') n_threads = torch.get_num_threads() - # FIXME remove this and make paste_masks_in_image run on the GPU torch.set_num_threads(1) cpu_device = torch.device("cpu") model.eval() - timer.report(f'evaluation preliminaries') + timer.report('evaluation preliminaries') test_step = test_sampler.progress // data_loader_test.batch_size - total_steps = len(test_sampler) // data_loader_test.batch_size print(f'\nEvaluating / resuming epoch {epoch} from eval step {test_step}\n') for images, targets in data_loader_test: @@ -221,6 +219,6 @@ def evaluate( iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) - timer.report(f'evaluator accumulation, summarization, and reset') + timer.report('evaluator accumulation, summarization, and reset') return coco_evaluator, timer, metrics diff --git a/tv-detection/prep.py b/tv-detection/prep.py index 4a58ea61..d64ec4db 100644 --- a/tv-detection/prep.py +++ b/tv-detection/prep.py @@ -1,10 +1,3 @@ -import torchvision -from torchvision.models import resnet50, ResNet50_Weights - -# weights = torchvision.models.get_weight('ResNet50_Weights.IMAGENET1K_V1') -_ = resnet50(weights=ResNet50_Weights.IMAGENET1K_V1) -# weights = torchvision.models.get_weight(...) - -from torchvision.models.detection import MaskRCNN +# Downloading ResNet101 backbone weights ahead of training from torchvision.models.detection.backbone_utils import resnet_fpn_backbone _ = resnet_fpn_backbone(backbone_name="resnet101", weights="ResNet101_Weights.IMAGENET1K_V1") \ No newline at end of file diff --git a/tv-detection/pyproject.toml b/tv-detection/pyproject.toml new file mode 100644 index 00000000..f17061df --- /dev/null +++ b/tv-detection/pyproject.toml @@ -0,0 +1,3 @@ +[tool.ruff] +# Decrease the maximum line length to 79 characters. +line-length = 120 \ No newline at end of file diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index fa698f56..5dbe0fb8 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -1,22 +1,3 @@ -r"""PyTorch Detection Training. - -To run in a multi-gpu environment, use the distributed launcher:: - - python -m torch.distributed.launch --nproc_per_node=$NGPU --use_env \ - train.py ... --world-size $NGPU - -The default hyperparameters are tuned for training on 8 gpus and 2 images per gpu. - --lr 0.02 --batch-size 2 --world-size 8 -If you use different number of gpus, the learning rate should be changed to 0.02/8*$NGPU. - -On top of that, for training Faster/Mask R-CNN, the default hyperparameters are - --epochs 26 --lr-steps 16 22 --aspect-ratio-group-factor 3 - -Also, if you train Keypoint R-CNN, the default hyperparameters are - --epochs 46 --lr-steps 36 43 --aspect-ratio-group-factor 3 -Because the number of images is smaller in the person keypoint subset of COCO, -the number of epochs should be adapted so that we have the same number of iterations. -""" from cycling_utils import TimestampedTimer timer = TimestampedTimer() @@ -24,12 +5,15 @@ import os from pathlib import Path +import argparse import presets import torch import torch.utils.data import torchvision import utils from coco_utils import get_coco +from coco_eval import CocoEvaluator +from coco_utils import get_coco_api_from_dataset import torchvision.models.detection import torchvision.models.detection.mask_rcnn @@ -108,29 +92,11 @@ def main(args, timer): timer.report('main preliminaries') - # Data loading code dataset_train, num_classes = get_dataset(is_train=True, args=args) dataset_test, _ = get_dataset(is_train=False, args=args) - # ## SUBSET FOR TESTING - # dataset_train = torch.utils.data.Subset(dataset_train, torch.arange(500)) - # dataset_test = torch.utils.data.Subset(dataset_test, torch.arange(200)) - timer.report('loading data') - # if args.distributed: - # train_sampler = torch.utils.data.distributed.DistributedSampler(dataset) - # test_sampler = torch.utils.data.distributed.DistributedSampler(dataset_test, shuffle=False) - # else: - # train_sampler = torch.utils.data.RandomSampler(dataset) - # test_sampler = torch.utils.data.SequentialSampler(dataset_test) - - # if args.aspect_ratio_group_factor >= 0: # default == 3 - # group_ids = create_aspect_ratio_groups(dataset_train, k=args.aspect_ratio_group_factor) - # train_batch_sampler = GroupedBatchSampler(train_sampler, group_ids, args.batch_size) - # else: - # train_batch_sampler = torch.utils.data.BatchSampler(train_sampler, args.batch_size, drop_last=True) - group_ids = create_aspect_ratio_groups(dataset_train, k=args.aspect_ratio_group_factor) train_sampler = InterruptableDistributedGroupedBatchSampler(dataset_train, group_ids, args.batch_size) test_sampler = InterruptableDistributedSampler(dataset_test) @@ -169,9 +135,6 @@ def main(args, timer): elif args.model == "retinanet_resnet101_fpn": backbone = resnet_fpn_backbone(backbone_name="resnet101", weights="ResNet101_Weights.IMAGENET1K_V1") model = RetinaNet(backbone=backbone, num_classes=num_classes) - # model = torchvision.models.get_model( - # args.model, weights=args.weights, weights_backbone=args.weights_backbone, num_classes=num_classes, **kwargs - # ) model.to(device) timer.report('creating model and .to(device)') @@ -231,8 +194,6 @@ def main(args, timer): timer.report('learning rate schedulers') - from coco_eval import CocoEvaluator - from coco_utils import get_coco_api_from_dataset coco = get_coco_api_from_dataset(data_loader_test.dataset) iou_types = _get_iou_types(model) coco_evaluator = CocoEvaluator(coco, iou_types) @@ -260,19 +221,23 @@ def main(args, timer): test_sampler.load_state_dict(checkpoint["test_sampler"]) if args.amp: scaler.load_state_dict(checkpoint["scaler"]) - # Evaluator state variables + # Evaluator and metrics coco_evaluator.img_ids = checkpoint["img_ids"] coco_evaluator.eval_imgs = checkpoint["eval_imgs"] metrics = checkpoint["metrics"] timer.report('retrieving checkpoint') - # if args.test_only: - # # We disable the cudnn benchmarking because it can noticeably affect the accuracy - # torch.backends.cudnn.benchmark = False - # torch.backends.cudnn.deterministic = True - # coco_evaluator, timer = evaluate(model, data_loader_test, device, timer) - # return + if args.test_only: + # We disable the cudnn benchmarking because it can noticeably affect the accuracy + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + epoch = 0 + coco_evaluator, timer, metrics = evaluate( + model, data_loader_test, epoch, test_sampler, args, coco_evaluator, optimizer, + lr_scheduler, warmup_lr_scheduler, train_sampler, device, scaler, timer, metrics + ) + return for epoch in range(args.start_epoch, args.epochs): @@ -281,28 +246,26 @@ def main(args, timer): print('\n') with train_sampler.in_epoch(epoch): - timer = TimestampedTimer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch + timer = TimestampedTimer() # obtain time trial for each epoch model, timer, metrics = train_one_epoch( - model, optimizer, data_loader_train, train_sampler, test_sampler, lr_scheduler, warmup_lr_scheduler, - args, device, coco_evaluator, epoch, scaler, timer, metrics + model, optimizer, data_loader_train, train_sampler, test_sampler, + lr_scheduler, warmup_lr_scheduler, args, device, coco_evaluator, + epoch, scaler, timer, metrics ) - # NEST THE TEST SAMPLER IN TRAIN SAMPLER CONTEXT TO AVOID EPOCH RESTART? with test_sampler.in_epoch(epoch): - timer = TimestampedTimer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch + timer = TimestampedTimer() # obtain time trial for each epoch coco_evaluator, timer, metrics = evaluate( - model, data_loader_test, epoch, test_sampler, args, coco_evaluator, optimizer, lr_scheduler, warmup_lr_scheduler, - train_sampler, device, scaler, timer, metrics + model, data_loader_test, epoch, test_sampler, args, coco_evaluator, + optimizer, lr_scheduler, warmup_lr_scheduler, train_sampler, device, + scaler, timer, metrics ) def get_args_parser(add_help=True): - import argparse - parser = argparse.ArgumentParser(description="PyTorch Detection Training", add_help=add_help) - - parser.add_argument("--data-path", default="/datasets01/COCO/022719/", type=str, help="dataset path") - parser.add_argument("--dataset",default="coco",type=str,help="dataset name. Use coco for object detection and instance segmentation and coco_kp for Keypoint detection",) + parser.add_argument("--data-path", default=None, type=str, help="dataset path") + parser.add_argument("--dataset",default="coco",type=str, help="dataset name. Use coco for object detection and instance segmentation and coco_kp for Keypoint detection",) parser.add_argument("--model", default="maskrcnn_resnet50_fpn", type=str, help="model name") parser.add_argument("--device", default="cuda", type=str, help="device (Use cuda or cpu Default: cuda)") parser.add_argument("-b", "--batch-size", default=2, type=int, help="images per gpu, the total batch size is $NGPU x batch_size") @@ -320,12 +283,11 @@ def get_args_parser(add_help=True): parser.add_argument("--print-freq", default=1, type=int, help="print frequency") parser.add_argument("--output-dir", default=".", type=str, help="path to save outputs") + parser.add_argument("--start_epoch", default=0, type=int, help="start epoch") parser.add_argument("--resume", default="", type=str, help="path of checkpoint") parser.add_argument("--prev-resume", default=None, help="path of previous job checkpoint for strong fail resume", dest="prev_resume") # for checkpointing parser.add_argument("--tboard-path", default=None, help="path for saving tensorboard logs", dest="tboard_path") # for checkpointing - - - parser.add_argument("--start_epoch", default=0, type=int, help="start epoch") + parser.add_argument("--aspect-ratio-group-factor", default=3, type=int) parser.add_argument("--rpn-score-thresh", default=None, type=float, help="rpn score threshold for faster-rcnn") parser.add_argument("--trainable-backbone-layers", default=None, type=int, help="number of trainable layers of backbone") @@ -345,7 +307,6 @@ def get_args_parser(add_help=True): # Use CopyPaste augmentation training parameter parser.add_argument("--use-copypaste", action="store_true",help="Use CopyPaste data augmentation. Works only with data-augmentation='lsj'.",) - parser.add_argument("--backend", default="PIL", type=str.lower, help="PIL or tensor - case insensitive") parser.add_argument("--use-v2", action="store_true", help="Use V2 transforms") diff --git a/tv-segmentation/pyproject.toml b/tv-segmentation/pyproject.toml new file mode 100644 index 00000000..f17061df --- /dev/null +++ b/tv-segmentation/pyproject.toml @@ -0,0 +1,3 @@ +[tool.ruff] +# Decrease the maximum line length to 79 characters. +line-length = 120 \ No newline at end of file diff --git a/tv-segmentation/train_cycling.py b/tv-segmentation/train_cycling.py index c1ff52f9..07136a87 100644 --- a/tv-segmentation/train_cycling.py +++ b/tv-segmentation/train_cycling.py @@ -5,6 +5,7 @@ import os from pathlib import Path +import argparse import presets import torch import torch.utils.data @@ -14,7 +15,7 @@ from torch import nn from torch.optim.lr_scheduler import PolynomialLR from torchvision.transforms import functional as F, InterpolationMode -from cycling_utils import InterruptableDistributedSampler, atomic_torch_save, Timer, MetricsTracker +from cycling_utils import InterruptableDistributedSampler, atomic_torch_save, MetricsTracker from torch.utils.tensorboard import SummaryWriter @@ -138,7 +139,6 @@ def evaluate( model.eval() test_step = test_sampler.progress // data_loader_test.batch_size - total_steps = len(test_sampler) // data_loader_test.batch_size print(f'\nEvaluating / resuming epoch {epoch} from eval step {test_step}\n') timer.report('launch evaluation routine') @@ -156,9 +156,6 @@ def evaluate( confmat.update(target.flatten().detach().cpu(), output.argmax(1).flatten().detach().cpu()) confmat.reduce_from_all_processes() - # FIXME need to take into account that the datasets - # could have been padded in distributed setup - # num_processed_samples += images.shape[0] timer.report(f'Epoch {epoch} batch: {test_step} confmat update') print(f"Saving checkpoint at epoch {epoch} eval batch {test_step}") @@ -199,21 +196,6 @@ def evaluate( writer.flush() writer.close() - # num_processed_samples = utils.reduce_across_processes(num_processed_samples) - - # if ( - # hasattr(data_loader_test.dataset, "__len__") - # and len(data_loader_test.dataset) != num_processed_samples - # and torch.distributed.get_rank() == 0 - # ): - # # See FIXME above - # warnings.warn( - # f"It looks like the dataset has {len(data_loader_test.dataset)} samples, but {num_processed_samples} " - # "samples were used for the validation, which might bias the results. " - # "Try adjusting the batch size and / or the world size. " - # "Setting the world size to 1 is always a safe bet." - # ) - return confmat, timer, metrics timer.report('defined other functions') @@ -239,34 +221,27 @@ def main(args, timer): dataset_train, num_classes = get_dataset(args.data_path, args.dataset, "train", get_transform(True, args)) dataset_test, _ = get_dataset(args.data_path, args.dataset, "val", get_transform(False, args)) - # ## SUBSET FOR TESTING - # dataset_train = torch.utils.data.Subset(dataset_train, torch.arange(500)) - # dataset_test = torch.utils.data.Subset(dataset_test, torch.arange(200)) - timer.report('loading data') - # if args.distributed: - # test_sampler = torch.utils.data.distributed.DistributedSampler(dataset_test, shuffle=False) - # else: - # train_sampler = torch.utils.data.RandomSampler(dataset) - # test_sampler = torch.utils.data.SequentialSampler(dataset_test) - train_sampler = InterruptableDistributedSampler(dataset_train) test_sampler = InterruptableDistributedSampler(dataset_test) timer.report('creating data samplers') data_loader_train = torch.utils.data.DataLoader( - dataset_train, batch_size=args.batch_size, sampler=train_sampler, num_workers=args.workers, collate_fn=utils.collate_fn, drop_last=True, + dataset_train, batch_size=args.batch_size, sampler=train_sampler, num_workers=args.workers, + collate_fn=utils.collate_fn, drop_last=True, ) data_loader_test = torch.utils.data.DataLoader( - dataset_test, batch_size=1, sampler=test_sampler, num_workers=args.workers, collate_fn=utils.collate_fn + dataset_test, batch_size=1, sampler=test_sampler, num_workers=args.workers, + collate_fn=utils.collate_fn ) timer.report('creating data loaders') model = torchvision.models.get_model( - args.model, weights=args.weights, weights_backbone=args.weights_backbone, num_classes=num_classes, aux_loss=args.aux_loss, + args.model, weights=args.weights, weights_backbone=args.weights_backbone, num_classes=num_classes, + aux_loss=args.aux_loss, ) model.to(device) @@ -357,13 +332,18 @@ def main(args, timer): timer.report('retrieving checkpoint') - # if args.test_only: - # # We disable the cudnn benchmarking because it can noticeably affect the accuracy - # torch.backends.cudnn.benchmark = False - # torch.backends.cudnn.deterministic = True - # confmat, timer = evaluate(model, data_loader_test, num_classes, confmat, test_sampler, device, 0, timer) - # print(confmat) - # return + if args.test_only: + # We disable the cudnn benchmarking because it can noticeably affect the accuracy + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + epoch = 0 + confmat, timer, metrics = evaluate( + args, model, data_loader_test, num_classes, confmat, + optimizer, lr_scheduler, train_sampler, test_sampler, + device, epoch, scaler, timer, metrics, + ) + print(confmat) + return for epoch in range(args.start_epoch, args.epochs): @@ -372,7 +352,7 @@ def main(args, timer): print('\n') with train_sampler.in_epoch(epoch): - timer = TimestampedTimer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch + timer = TimestampedTimer() # obtain time trial for each epoch model, timer, metrics = train_one_epoch( args, model, criterion, optimizer, data_loader_train, train_sampler, test_sampler, confmat, lr_scheduler, @@ -380,9 +360,8 @@ def main(args, timer): ) timer.report(f'training for epoch {epoch}') - # NEST TEST SAMPLER IN TRAIN SAMPLER CONTEXT TO AVOID EPOCH RESTART? with test_sampler.in_epoch(epoch): - timer = TimestampedTimer() # Restarting timer, timed the preliminaries, now obtain time trial for each epoch + timer = TimestampedTimer() # obtain time trial for each epoch confmat, timer, metrics = evaluate( args, model, data_loader_test, num_classes, confmat, optimizer, lr_scheduler, train_sampler, test_sampler, @@ -392,10 +371,7 @@ def main(args, timer): def get_args_parser(add_help=True): - import argparse - parser = argparse.ArgumentParser(description="PyTorch Segmentation Training", add_help=add_help) - parser.add_argument("--data-path", default="/datasets01/COCO/022719/", type=str, help="dataset path") parser.add_argument("--dataset", default="coco", type=str, help="dataset name") parser.add_argument("--model", default="fcn_resnet101", type=str, help="model name") diff --git a/tv-segmentation/utils.py b/tv-segmentation/utils.py index 899a9bad..8f524d1e 100644 --- a/tv-segmentation/utils.py +++ b/tv-segmentation/utils.py @@ -118,7 +118,7 @@ def update(self, **kwargs): v = v.item() if not isinstance(v, (float, int)): raise TypeError( - f"This method expects the value of the input arguments to be of type float or int, instead got {type(v)}" + f"Method expects value of the input arguments to be of type float or int, instead got {type(v)}" ) self.meters[k].update(v) From 4ef575ec72e385caddce35aad5b2d14fa1e28bbb Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Fri, 22 Sep 2023 13:07:40 +1000 Subject: [PATCH 39/44] removed local pyproject.toml files --- tv-detection/pyproject.toml | 3 --- tv-segmentation/pyproject.toml | 3 --- 2 files changed, 6 deletions(-) delete mode 100644 tv-detection/pyproject.toml delete mode 100644 tv-segmentation/pyproject.toml diff --git a/tv-detection/pyproject.toml b/tv-detection/pyproject.toml deleted file mode 100644 index f17061df..00000000 --- a/tv-detection/pyproject.toml +++ /dev/null @@ -1,3 +0,0 @@ -[tool.ruff] -# Decrease the maximum line length to 79 characters. -line-length = 120 \ No newline at end of file diff --git a/tv-segmentation/pyproject.toml b/tv-segmentation/pyproject.toml deleted file mode 100644 index f17061df..00000000 --- a/tv-segmentation/pyproject.toml +++ /dev/null @@ -1,3 +0,0 @@ -[tool.ruff] -# Decrease the maximum line length to 79 characters. -line-length = 120 \ No newline at end of file From d09abfb3b841583d16c932b210aec885faa4d7b1 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Fri, 22 Sep 2023 13:14:25 +1000 Subject: [PATCH 40/44] removed monai --- monai_brats_mri_2d/brats_mri_2d_diff.isc | 6 - monai_brats_mri_2d/brats_mri_2d_gen.isc | 6 - monai_brats_mri_2d/loops.py | 395 ----------------------- monai_brats_mri_2d/prep.py | 20 -- monai_brats_mri_2d/train_cycling_diff.py | 215 ------------ monai_brats_mri_2d/train_cycling_gen.py | 206 ------------ monai_brats_mri_2d/utils.py | 72 ----- 7 files changed, 920 deletions(-) delete mode 100644 monai_brats_mri_2d/brats_mri_2d_diff.isc delete mode 100644 monai_brats_mri_2d/brats_mri_2d_gen.isc delete mode 100644 monai_brats_mri_2d/loops.py delete mode 100644 monai_brats_mri_2d/prep.py delete mode 100644 monai_brats_mri_2d/train_cycling_diff.py delete mode 100644 monai_brats_mri_2d/train_cycling_gen.py delete mode 100644 monai_brats_mri_2d/utils.py diff --git a/monai_brats_mri_2d/brats_mri_2d_diff.isc b/monai_brats_mri_2d/brats_mri_2d_diff.isc deleted file mode 100644 index 5f100352..00000000 --- a/monai_brats_mri_2d/brats_mri_2d_diff.isc +++ /dev/null @@ -1,6 +0,0 @@ -experiment_name="brats_mri_2d_diff" -gpu_type="24GB VRAM GPU" -nnodes = 11 -venv_path = "~/.venv/bin/activate" -output_path = "~/output_brats_mri_2d_diff" -command="train_cycling_diff.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc, --gen-load-path ~/output_brats_mri_2d_gen/exp_1855/checkpoint.isc --tboard-path $OUTPUT_PATH/tb" \ No newline at end of file diff --git a/monai_brats_mri_2d/brats_mri_2d_gen.isc b/monai_brats_mri_2d/brats_mri_2d_gen.isc deleted file mode 100644 index 802596ee..00000000 --- a/monai_brats_mri_2d/brats_mri_2d_gen.isc +++ /dev/null @@ -1,6 +0,0 @@ -experiment_name="brats_mri_2d_gen" -gpu_type="24GB VRAM GPU" -nnodes = 11 -venv_path = "~/.venv/bin/activate" -output_path = "~/output_brats_mri_2d_gen" -command="train_cycling_gen.py --data-path=/mnt/.node1/Open-Datsets/MONAI --resume $OUTPUT_PATH/checkpoint.isc --tboard-path $OUTPUT_PATH/tb" \ No newline at end of file diff --git a/monai_brats_mri_2d/loops.py b/monai_brats_mri_2d/loops.py deleted file mode 100644 index 7d65f910..00000000 --- a/monai_brats_mri_2d/loops.py +++ /dev/null @@ -1,395 +0,0 @@ -import torch -from torch.cuda.amp import autocast -import torch.nn.functional as F -import utils -from cycling_utils import atomic_torch_save - -from generative.losses.adversarial_loss import PatchAdversarialLoss - -from torch.utils.tensorboard import SummaryWriter -from torchvision.utils import make_grid - -## -- AUTO-ENCODER - ## - -def compute_kl_loss(z_mu, z_sigma): - kl_loss = 0.5 * torch.sum( - z_mu.pow(2) + z_sigma.pow(2) - torch.log(z_sigma.pow(2)) - 1, - dim=list(range(1, len(z_sigma.shape))) - ) - return torch.sum(kl_loss) / kl_loss.shape[0] - -intensity_loss = torch.nn.L1Loss() -adv_loss = PatchAdversarialLoss(criterion="least_squares") - -def generator_loss(gen_images, real_images, z_mu, z_sigma, disc_net, perceptual_loss, kl_weight, perceptual_weight, adv_weight): - # Image intrinsic qualities - recons_loss = intensity_loss(gen_images, real_images) - kl_loss = compute_kl_loss(z_mu, z_sigma) - p_loss = perceptual_loss(gen_images.float(), real_images.float()) - loss_g = recons_loss + (kl_weight * kl_loss) + (perceptual_weight * p_loss) - # Discrimnator-based loss - logits_fake = disc_net(gen_images)[-1] - generator_loss = adv_loss(logits_fake, target_is_real=True, for_discriminator=False) - loss_g = loss_g + (adv_weight * generator_loss) - return loss_g - -def discriminator_loss(gen_images, real_images, disc_net, adv_weight): - logits_fake = disc_net(gen_images.contiguous().detach())[-1] - loss_d_fake = adv_loss(logits_fake, target_is_real=False, for_discriminator=True) - logits_real = disc_net(real_images.contiguous().detach())[-1] - loss_d_real = adv_loss(logits_real, target_is_real=True, for_discriminator=True) - discriminator_loss = (loss_d_fake + loss_d_real) * 0.5 - loss_d = adv_weight * discriminator_loss - return loss_d - - -def train_generator_one_epoch( - args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, - scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, - metrics - ): - - # Obtained from scripts.losses.generator_loss - kl_weight = 1e-6 - perceptual_weight = 1.0 - adv_weight = 0.5 - # From tutorial ? - # generator_warm_up_n_epochs = 10 - - generator.train() - discriminator.train() - - train_step = train_sampler.progress // train_loader.batch_size - total_steps = int(len(train_sampler) / train_loader.batch_size) - print(f'\nTraining / resuming epoch {epoch} from training step {train_step}\n') - - for step, batch in enumerate(train_loader): - - images = batch["image"].to(device) - timer.report(f'train batch {train_step} to device') - - # TRAIN GENERATOR - - optimizer_g.zero_grad(set_to_none=True) - - with autocast(enabled=True): - - reconstruction, z_mu, z_sigma = generator(images) - timer.report(f'train batch {train_step} generator forward') - - loss_g = generator_loss( - reconstruction, images, z_mu, z_sigma, discriminator, perceptual_loss, - kl_weight, perceptual_weight, adv_weight - ) - timer.report(f'train batch {train_step} generator loss: {loss_g.item():.3f}') - - scaler_g.scale(loss_g).backward() - scaler_g.step(optimizer_g) - scaler_g.update() - timer.report(f'train batch {train_step} generator backward') - - # TRAIN DISCRIMINATOR - - optimizer_d.zero_grad(set_to_none=True) - - with autocast(enabled=True): - - loss_d = discriminator_loss( - reconstruction, images, discriminator, adv_weight - ) - timer.report(f'train batch {train_step} discriminator loss {loss_d.item():.3f}') - - scaler_d.scale(loss_d).backward() - scaler_d.step(optimizer_d) - scaler_d.update() - timer.report(f'train batch {train_step} discriminator backward') - - # Reduce metrics accross nodes - metrics["train"].update({"train_images_seen":len(images), "loss_g":loss_g.item(), "loss_d": loss_d.item()}) - metrics["train"].reduce() - - gen_loss = metrics["train"].local["loss_g"] / metrics["train"].local["train_images_seen"] - disc_loss = metrics["train"].local["loss_d"] / metrics["train"].local["train_images_seen"] - print("Epoch [{}] Step [{}/{}], gen_loss: {:.3f}, disc_loss: {:.3f}".format(epoch, train_step, total_steps, gen_loss, disc_loss)) - - metrics["train"].reset_local() - - timer.report(f'train batch {train_step} metrics update') - - ## Checkpointing - print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") - train_sampler.advance(len(images)) - train_step = train_sampler.progress // train_loader.batch_size - - if train_step == total_steps: - metrics["train"].end_epoch() - - if utils.is_main_process() and train_step % 1 == 0: # Checkpointing every batch - writer = SummaryWriter(log_dir=args.tboard_path) - writer.add_scalar("Train/gen_loss", gen_loss, train_step + epoch * total_steps) - writer.add_scalar("Train/disc_loss", disc_loss, train_step + epoch * total_steps) - writer.flush() - writer.close() - checkpoint = { - # Universals - "args": args, - "epoch": epoch, - # State variables - "generator": generator.module.state_dict(), - "discriminator": discriminator.module.state_dict(), - "optimizer_g": optimizer_g.state_dict(), - "optimizer_d": optimizer_d.state_dict(), - "scaler_g": scaler_g.state_dict(), - "scaler_d": scaler_d.state_dict(), - "train_sampler": train_sampler.state_dict(), - "val_sampler": val_sampler.state_dict(), - # Metrics - "metrics": metrics, - } - timer = atomic_torch_save(checkpoint, args.resume, timer) - - gen_loss = metrics["train"].epoch_reports[-1]["loss_g"] / metrics["train"].epoch_reports[-1]["train_images_seen"] - disc_loss = metrics["train"].epoch_reports[-1]["loss_d"] / metrics["train"].epoch_reports[-1]["train_images_seen"] - print("Epoch [{}] :: gen_loss: {:,.3f}, disc_loss: {:,.3f}".format(epoch, gen_loss, disc_loss)) - return generator, timer, metrics - - -def evaluate_generator( - args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, - scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, - metrics - ): - - generator.eval() - - val_step = val_sampler.progress // val_loader.batch_size - total_steps = int(len(val_sampler) / val_loader.batch_size) - print(f'\nEvaluating / resuming epoch {epoch} from eval step {val_step}\n') - - with torch.no_grad(): - for batch in val_loader: - - images = batch["image"].to(device) - timer.report(f'eval batch {val_step} to device') - - with autocast(enabled=True): - - reconstruction, _, _ = generator(images) - timer.report(f'eval batch {val_step} forward') - recons_loss = F.l1_loss(images.float(), reconstruction.float()) - timer.report(f'eval batch {val_step} recons_loss') - - metrics["val"].update({"val_images_seen": len(images), "val_loss": recons_loss.item()}) - metrics["val"].reduce() - metrics["val"].reset_local() - - timer.report(f'eval batch {val_step} metrics update') - - ## Checkpointing - print(f"Saving checkpoint at epoch {epoch} val batch {val_step}") - val_sampler.advance(len(images)) - val_step = val_sampler.progress // val_loader.batch_size - - if val_step == total_steps: - - val_loss = metrics["val"].agg["val_loss"] / metrics["val"].agg["val_images_seen"] - if utils.is_main_process(): - - writer = SummaryWriter(log_dir=args.tboard_path) - writer.add_scalar("Val/loss", val_loss, epoch) - - image_pair = [images[0], reconstruction[0]] - grid = make_grid(image_pair) - writer.add_image('Val/images', grid, epoch) - # writer.add_graph(model, images) - writer.flush() - writer.close() - - print(f"Epoch {epoch} val loss: {val_loss:.4f}") - metrics["val"].end_epoch() - - if utils.is_main_process() and val_step % 1 == 0: # Checkpointing every batch - checkpoint = { - # Universals - "args": args, - "epoch": epoch, - # State variables - "generator": generator.module.state_dict(), - "discriminator": discriminator.module.state_dict(), - "optimizer_g": optimizer_g.state_dict(), - "optimizer_d": optimizer_d.state_dict(), - "scaler_g": scaler_g.state_dict(), - "scaler_d": scaler_d.state_dict(), - "train_sampler": train_sampler.state_dict(), - "val_sampler": val_sampler.state_dict(), - # Metrics - "metrics": metrics, - } - timer = atomic_torch_save(checkpoint, args.resume, timer) - - return timer, metrics - - -## -- DIFFUSION MODEL - ## - -def train_diffusion_one_epoch( - args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, - train_sampler, val_sampler, lr_scheduler, device, timer, metrics - ): - - unet.train() - generator.eval() - - train_step = train_sampler.progress // train_loader.batch_size - total_steps = len(train_sampler) // train_loader.batch_size - print(f'\nTraining / resuming epoch {epoch} from training step {train_step}\n') - - for step, batch in enumerate(train_loader): - - images = batch["image"].to(device) - timer.report(f'train batch {train_step} to device') - - optimizer_u.zero_grad(set_to_none=True) - - with autocast(enabled=True): - - z_mu, z_sigma = generator.encode(images) - timer.report(f'train batch {train_step} generator encoded') - z = generator.sampling(z_mu, z_sigma) - timer.report(f'train batch {train_step} generator sampling') - noise = torch.randn_like(z).to(device) - timer.report(f'train batch {train_step} noise') - timesteps = torch.randint(0, inferer.scheduler.num_train_timesteps, (z.shape[0],), device=z.device).long() - timer.report(f'train batch {train_step} timesteps') - noise_pred = inferer(inputs=images, diffusion_model=unet, noise=noise, timesteps=timesteps, autoencoder_model=generator) - timer.report(f'train batch {train_step} noise_pred') - loss = F.mse_loss(noise_pred.float(), noise.float()) - timer.report(f'train batch {train_step} loss') - - scaler_u.scale(loss).backward() - scaler_u.step(optimizer_u) - scaler_u.update() - lr_scheduler.step() - timer.report(f'train batch {train_step} unet backward') - - # Reduce metrics accross nodes - metrics["train"].update({"images_seen":len(images), "loss":loss.item()}) - metrics["train"].reduce() - - recons_loss = metrics["train"].local["loss"] / metrics["train"].local["images_seen"] - print("Epoch [{}] Step [{}/{}] :: loss: {:,.3f}".format(epoch, train_step, total_steps, recons_loss)) - - metrics["train"].reset_local() - - timer.report(f'train batch {train_step} metrics update') - - ## Checkpointing - print(f"Saving checkpoint at epoch {epoch} train batch {train_step}") - train_sampler.advance(len(images)) - train_step = train_sampler.progress // train_loader.batch_size - - if train_step == total_steps: - metrics["train"].end_epoch() - - if utils.is_main_process() and train_step % 1 == 0: # Checkpointing every batch - - writer = SummaryWriter(log_dir=args.tboard_path) - writer.add_scalar("Train/loss", recons_loss, train_step + epoch * total_steps) - writer.flush() - writer.close() - - checkpoint = { - # Universals - "args": args, - "epoch": epoch, - # State variables - "unet": unet.module.state_dict(), - "optimizer_u": optimizer_u.state_dict(), - "scaler_u": scaler_u.state_dict(), - "train_sampler": train_sampler.state_dict(), - "val_sampler": val_sampler.state_dict(), - "lr_scheduler": lr_scheduler.state_dict(), - # Metrics - "metrics": metrics, - } - timer = atomic_torch_save(checkpoint, args.resume, timer) - - train_loss = metrics["train"].epoch_reports[-1]["loss"] / metrics["train"].epoch_reports[-1]["images_seen"] - print("Epoch [{}] :: epoch_loss: {:,.3f}".format(epoch, train_loss)) - return unet, timer, metrics - - -def evaluate_diffusion( - args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, - train_sampler, val_sampler, lr_scheduler, device, timer, metrics - ): - - unet.eval() - - val_step = val_sampler.progress // val_loader.batch_size - total_steps = int(len(val_sampler) / val_loader.batch_size) - print(f'\nEvaluating / resuming epoch {epoch} from eval step {val_step}\n') - - with torch.no_grad(): - for step, batch in enumerate(val_loader): - - images = batch["image"].to(device) - timer.report(f'eval batch {val_step} to device') - - with autocast(enabled=True): - - z_mu, z_sigma = generator.encode(images) - timer.report(f'eval batch {val_step} generator encoded') - z = generator.sampling(z_mu, z_sigma) - timer.report(f'eval batch {val_step} generator sampling') - noise = torch.randn_like(z).to(device) - timer.report(f'eval batch {val_step} noise') - timesteps = torch.randint(0, inferer.scheduler.num_train_timesteps, (z.shape[0],), device=z.device).long() - timer.report(f'eval batch {val_step} timesteps') - noise_pred = inferer(inputs=images,diffusion_model=unet,noise=noise,timesteps=timesteps,autoencoder_model=generator) - timer.report(f'eval batch {val_step} noise_pred') - loss = F.mse_loss(noise_pred.float(), noise.float()) - timer.report(f'eval batch {val_step} loss') - - metrics["val"].update({"images_seen": len(images), "loss": loss.item()}) - metrics["val"].reduce() - metrics["val"].reset_local() - - timer.report(f'eval batch {val_step} metrics update') - - ## Checkpointing - print(f"Saving checkpoint at epoch {epoch} val batch {val_step}") - val_sampler.advance(len(images)) - val_step = val_sampler.progress // val_loader.batch_size - - if val_step == total_steps: - metrics["val"].end_epoch() - - if utils.is_main_process() and val_step % 1 == 0: # Checkpointing every batch - print(f"Saving checkpoint at epoch {epoch} train batch {val_step}") - checkpoint = { - # Universals - "args": args, - "epoch": epoch, - # State variables - "unet": unet.module.state_dict(), - "optimizer_u": optimizer_u.state_dict(), - "scaler_u": scaler_u.state_dict(), - "train_sampler": train_sampler.state_dict(), - "val_sampler": val_sampler.state_dict(), - "lr_scheduler": lr_scheduler.state_dict(), - # Metrics - "metrics": metrics, - } - timer = atomic_torch_save(checkpoint, args.resume, timer) - - # val_loss = metrics["val"].agg[metrics["val"].map["val_loss"]] / metrics["val"].agg[metrics["val"].map["val_images_seen"]] - val_loss = metrics["val"].epoch_reports[-1]["loss"] / metrics["val"].epoch_reports[-1]["images_seen"] - if utils.is_main_process(): - writer = SummaryWriter(log_dir=args.tboard_path) - writer.add_scalar("Val/loss", val_loss, epoch) - writer.flush() - writer.close() - print(f"Epoch [{epoch}] :: diff val loss: {val_loss:.4f}") - - return timer, metrics diff --git a/monai_brats_mri_2d/prep.py b/monai_brats_mri_2d/prep.py deleted file mode 100644 index 80c5c480..00000000 --- a/monai_brats_mri_2d/prep.py +++ /dev/null @@ -1,20 +0,0 @@ -# Download the toy dataset from MONAI -print("Downloadning BraTS2016/17") -from monai.apps import DecathlonDataset -from generative.losses.perceptual import PerceptualLoss - -# _ = DecathlonDataset(root_dir="/mnt/Datasets/Open-Datasets/MONAI", task="Task01_BrainTumour", section="training", download=True) -_ = DecathlonDataset(root_dir="/mnt/.node1/Open-Datsets/MONAI", task="Task01_BrainTumour", section="training", download=True) - -perceptual_loss = PerceptualLoss( - spatial_dims=2, network_type="resnet50", pretrained=True, #ImageNet pretrained weights used -) - -# # Download the bigger dataset from Synapse -# print("Downloadning BraTS2023") -# import synapseclient -# syn = synapseclient.Synapse() -# syn.login('adam_peaston','AXXXXXXXXX2') -# syn51514132 = syn.get(entity='syn51514132', downloadFile=True, downloadLocation="/mnt/Datasets/strongcompute_adam/MONAI", ifcollision="overwrite.local") -# filepath = syn51514132.path -# print(f"BraTS2023-GLI downloaded to {filepath}") \ No newline at end of file diff --git a/monai_brats_mri_2d/train_cycling_diff.py b/monai_brats_mri_2d/train_cycling_diff.py deleted file mode 100644 index 13768e1a..00000000 --- a/monai_brats_mri_2d/train_cycling_diff.py +++ /dev/null @@ -1,215 +0,0 @@ -from cycling_utils import TimestampedTimer - -timer = TimestampedTimer() -timer.report('importing Timer') - -import os -import torch -# import torch.nn.functional as F -from monai import transforms -from monai.apps import DecathlonDataset -# from monai.config import print_config -from monai.data import DataLoader #, Dataset -from monai.utils import first, set_determinism -from torch.cuda.amp import GradScaler, autocast -from pathlib import Path -# from tqdm import tqdm - -from generative.inferers import LatentDiffusionInferer -# from generative.losses.adversarial_loss import PatchAdversarialLoss -# from generative.losses.perceptual import PerceptualLoss -from generative.networks.nets import AutoencoderKL, DiffusionModelUNet # , PatchDiscriminator -from generative.networks.schedulers import DDPMScheduler - -from cycling_utils import InterruptableDistributedSampler, Timer, MetricsTracker -# from loops import train_generator_one_epoch, evaluate_generator -from loops import train_diffusion_one_epoch, evaluate_diffusion -import utils - -def get_args_parser(add_help=True): - import argparse - parser = argparse.ArgumentParser(description="Latent Diffusion Model Training", add_help=add_help) - parser.add_argument("--resume", type=str, help="path of checkpoint", required=True) # for checkpointing - parser.add_argument("--gen-load-path", type=str, help="path of checkpoint", dest="gen_load_path") # for checkpointing - parser.add_argument("--prev-resume", default=None, help="path of previous job checkpoint for strong fail resume", dest="prev_resume") # for checkpointing - parser.add_argument("--tboard-path", default=None, help="path for saving tensorboard logs", dest="tboard_path") # for checkpointing - parser.add_argument("--data-path", default="/mnt/Datasets/Open-Datasets/MONAI", type=str, help="dataset path", dest="data_path") - parser.add_argument("--device", default="cuda", type=str, help="device (Use cuda or cpu Default: cuda)") - parser.add_argument("--start-epoch", default=0, type=int, metavar="N", help="start epoch") - parser.add_argument("--dist-url", default="env://", type=str, help="url used to set up distributed training") - parser.add_argument("-j", "--workers", default=16, type=int, metavar="N", help="number of data loading workers (default: 16)") - return parser - -def compute_scale_factor(autoencoder, train_loader, device): - with torch.no_grad(): - check_data = first(train_loader) - z = autoencoder.encode_stage_2_inputs(check_data["image"].to(device)) - scale_factor = 1 / torch.std(z) - return scale_factor.item() - -timer.report('importing everything else') - -def main(args, timer): - - utils.init_distributed_mode(args) # Sets args.distributed among other things - assert args.distributed # don't support cycling when not distributed for simplicity - - device = torch.device(args.device) - - timer.report('preliminaries') - - # Maybe this will work? - set_determinism(42) - - channel = 0 # 0 = "Flair" channel - assert channel in [0, 1, 2, 3], "Choose a valid channel" - preprocessing_transform = transforms.Compose([ - transforms.LoadImaged(keys="image", image_only=False), # image_only current default will change soon, so including explicitly - transforms.EnsureChannelFirstd(keys="image"), - transforms.Lambdad(keys="image", func=lambda x: x[channel, :, :, :]), - transforms.AddChanneld(keys="image"), - transforms.EnsureTyped(keys="image"), - transforms.Orientationd(keys="image", axcodes="RAS"), - transforms.CenterSpatialCropd(keys="image", roi_size=(240, 240, 100)), - transforms.ScaleIntensityRangePercentilesd(keys="image", lower=0, upper=100, b_min=0, b_max=1), - ]) - - crop_transform = transforms.Compose([ - transforms.DivisiblePadd(keys="image", k=[32,32,1]), - transforms.RandSpatialCropd(keys="image", roi_size=(256, 256, 1), random_size=False), # Each of the 100 slices will be randomly sampled. - transforms.SqueezeDimd(keys="image", dim=3), - # transforms.RandFlipd(keys="image", prob=0.5, spatial_axis=0), - # transforms.RandFlipd(keys="image", prob=0.5, spatial_axis=1), - ]) - - preprocessing = transforms.Compose([preprocessing_transform, crop_transform]) - - train_ds = DecathlonDataset( - root_dir=args.data_path, task="Task01_BrainTumour", section="training", cache_rate=0.0, - num_workers=8, download=False, seed=0, transform=preprocessing, - ) - val_ds = DecathlonDataset( - root_dir=args.data_path, task="Task01_BrainTumour", section="validation", cache_rate=0.0, - num_workers=8, download=False, seed=0, transform=preprocessing, - ) - - timer.report('build datasets') - - train_sampler = InterruptableDistributedSampler(train_ds) - val_sampler = InterruptableDistributedSampler(val_ds) - - timer.report('build samplers') - - # Original trainer had batch size = 2 * 50. Using 11 nodes x 6 GPUs x batch size 2 => eff batch size = 132 - train_loader = DataLoader(train_ds, batch_size=2, sampler=train_sampler, num_workers=1) - val_loader = DataLoader(val_ds, batch_size=1, sampler=val_sampler, num_workers=1) - # check_data = first(train_loader) # Used later - - timer.report('build dataloaders') - - # Auto-encoder definition - generator = AutoencoderKL( - spatial_dims=2, in_channels=1, out_channels=1, num_channels=(64, 128, 256), - latent_channels=1, num_res_blocks=2, norm_num_groups=32, norm_eps=1e-06, - attention_levels=(False, False, False), with_encoder_nonlocal_attn=True, - with_decoder_nonlocal_attn=True, - ) - # saved_generator_checkpoint = torch.load("/output_brats_mri_2d_gen/exp_1645/checkpoint.isc", map_location="cpu") - saved_generator_checkpoint = torch.load(args.gen_load_path, map_location="cpu") - generator.load_state_dict(saved_generator_checkpoint["generator"]) - generator = generator.to(device) - - timer.report('generator to device') - - # Diffusion model (unet) - unet = DiffusionModelUNet( - spatial_dims=2, in_channels=1, out_channels=1, num_res_blocks=2, - num_channels=(32, 64, 128, 256), attention_levels=(False, True, True, True), - num_head_channels=(0, 32, 32, 32), - ) - unet = unet.to(device) - - timer.report('unet to device') - - # Prepare for distributed training - unet = torch.nn.SyncBatchNorm.convert_sync_batchnorm(unet) - - unet_without_ddp = unet - if args.distributed: - unet = torch.nn.parallel.DistributedDataParallel(unet, device_ids=[args.gpu], find_unused_parameters=True) - unet_without_ddp = unet.module - - timer.report('unet prepped for distribution') - - # Optimizers - optimizer_u = torch.optim.Adam(unet_without_ddp.parameters(), lr=5e-5) - lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer_u, milestones=[1000], gamma=0.1) - - # For mixed precision training - scaler_u = GradScaler() - - timer.report('optimizer, lr_scheduler and grad scaler') - - # Init metric tracker - metrics = {'train': MetricsTracker(), 'val': MetricsTracker()} - - # Prepare LatentDiffusionInferer - - scale_factor = compute_scale_factor(generator, train_loader, device) - scheduler = DDPMScheduler(num_train_timesteps=1000, schedule="scaled_linear_beta", beta_start=0.0015, beta_end=0.0195) - inferer = LatentDiffusionInferer(scheduler, scale_factor=scale_factor) - - timer.report('building inferer') - - # RETRIEVE CHECKPOINT - Path(args.resume).parent.mkdir(parents=True, exist_ok=True) - checkpoint = None - if args.resume and os.path.isfile(args.resume): # If we're resuming... - checkpoint = torch.load(args.resume, map_location="cpu") - elif args.prev_resume and os.path.isfile(args.prev_resume): - checkpoint = torch.load(args.prev_resume, map_location="cpu") - if checkpoint is not None: - args.start_epoch = checkpoint["epoch"] - unet_without_ddp.load_state_dict(checkpoint["unet"]) - optimizer_u.load_state_dict(checkpoint["optimizer_u"]) - scaler_u.load_state_dict(checkpoint["scaler_u"]) - train_sampler.load_state_dict(checkpoint["train_sampler"]) - val_sampler.load_state_dict(checkpoint["val_sampler"]) - lr_scheduler.load_state_dict(checkpoint["lr_scheduler"]) - # Metrics - metrics = checkpoint["metrics"] - - timer.report('checkpoint retrieval') - - ## -- TRAINING THE DIFFUSION MODEL - ## - - n_diff_epochs = 200 - diff_val_interval = 1 - - for epoch in range(args.start_epoch, n_diff_epochs): - - print('\n') - print(f"EPOCH :: {epoch}") - print('\n') - - with train_sampler.in_epoch(epoch): - timer = TimestampedTimer("Start training") - unet, timer, metrics = train_diffusion_one_epoch( - args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, - train_sampler, val_sampler, lr_scheduler, device, timer, metrics - ) - timer.report(f'training unet for epoch {epoch}') - - if epoch % diff_val_interval == 0: - with val_sampler.in_epoch(epoch): - timer = TimestampedTimer("Start evaluation") - timer, metrics = evaluate_diffusion( - args, epoch, unet, generator, optimizer_u, scaler_u, inferer, train_loader, val_loader, - train_sampler, val_sampler, lr_scheduler, device, timer, metrics - ) - timer.report(f'evaluating unet for epoch {epoch}') - - -if __name__ == "__main__": - args = get_args_parser().parse_args() - main(args, timer) diff --git a/monai_brats_mri_2d/train_cycling_gen.py b/monai_brats_mri_2d/train_cycling_gen.py deleted file mode 100644 index e2938eb5..00000000 --- a/monai_brats_mri_2d/train_cycling_gen.py +++ /dev/null @@ -1,206 +0,0 @@ -from cycling_utils import TimestampedTimer - -timer = TimestampedTimer() -timer.report('importing Timer') - -import os -import torch -from monai import transforms -from monai.apps import DecathlonDataset -from monai.data import DataLoader -from monai.utils import set_determinism -from torch.cuda.amp import GradScaler -from pathlib import Path - -from generative.losses.adversarial_loss import PatchAdversarialLoss -from generative.losses.perceptual import PerceptualLoss -from generative.networks.nets import AutoencoderKL, PatchDiscriminator - -from cycling_utils import InterruptableDistributedSampler, Timer, MetricsTracker -from loops import train_generator_one_epoch, evaluate_generator -import utils - -def get_args_parser(add_help=True): - import argparse - parser = argparse.ArgumentParser(description="Latent Diffusion Model Training", add_help=add_help) - parser.add_argument("--resume", type=str, help="path of checkpoint", required=True) # for checkpointing - parser.add_argument("--prev-resume", default=None, help="path of previous job checkpoint for strong fail resume", dest="prev_resume") # for checkpointing - parser.add_argument("--tboard-path", default=None, help="path for saving tensorboard logs", dest="tboard_path") # for checkpointing - parser.add_argument("--data-path", default="/mnt/Datasets/Open-Datasets/MONAI", type=str, help="dataset path", dest="data_path") - parser.add_argument("--device", default="cuda", type=str, help="device (Use cuda or cpu Default: cuda)") - parser.add_argument("--start-epoch", default=0, type=int, metavar="N", help="start epoch") - parser.add_argument("--dist-url", default="env://", type=str, help="url used to set up distributed training") - parser.add_argument("-j", "--workers", default=16, type=int, metavar="N", help="number of data loading workers (default: 16)") - return parser - -timer.report('importing everything else') - -def main(args, timer): - - utils.init_distributed_mode(args) # Sets args.distributed among other things - assert args.distributed # don't support cycling when not distributed for simplicity - - device = torch.device(args.device) - - # Maybe this will work? - set_determinism(42) - - timer.report('preliminaries') - - channel = 0 # 0 = "Flair" channel - assert channel in [0, 1, 2, 3], "Choose a valid channel" - preprocessing_transform = transforms.Compose([ - transforms.LoadImaged(keys="image", image_only=False), # image_only current default will change soon, so including explicitly - transforms.EnsureChannelFirstd(keys="image"), - transforms.Lambdad(keys="image", func=lambda x: x[channel, :, :, :]), - transforms.AddChanneld(keys="image"), - transforms.EnsureTyped(keys="image"), - transforms.Orientationd(keys="image", axcodes="RAS"), - transforms.CenterSpatialCropd(keys="image", roi_size=(240, 240, 100)), - transforms.ScaleIntensityRangePercentilesd(keys="image", lower=0, upper=100, b_min=0, b_max=1), - ]) - - crop_transform = transforms.Compose([ - transforms.DivisiblePadd(keys="image", k=[4,4,1]), - transforms.RandSpatialCropd(keys="image", roi_size=(240, 240, 1), random_size=False), # Each of the 100 slices will be randomly sampled. - transforms.SqueezeDimd(keys="image", dim=3), - transforms.RandFlipd(keys="image", prob=0.5, spatial_axis=0), - transforms.RandFlipd(keys="image", prob=0.5, spatial_axis=1), - ]) - - preprocessing = transforms.Compose([preprocessing_transform, crop_transform]) - - train_ds = DecathlonDataset( - root_dir=args.data_path, task="Task01_BrainTumour", section="training", cache_rate=0.0, - num_workers=8, download=False, seed=0, transform=preprocessing, - ) - val_ds = DecathlonDataset( - root_dir=args.data_path, task="Task01_BrainTumour", section="validation", cache_rate=0.0, - num_workers=8, download=False, seed=0, transform=preprocessing, - ) - - timer.report('build datasets') - - train_sampler = InterruptableDistributedSampler(train_ds) - val_sampler = InterruptableDistributedSampler(val_ds) - - timer.report('build samplers') - - # Original trainer had batch size = 26. Using 9 nodes x batch size 3 = eff batch size = 27 - train_loader = DataLoader(train_ds, batch_size=3, sampler=train_sampler, num_workers=1) - val_loader = DataLoader(val_ds, batch_size=1, sampler=val_sampler, num_workers=1) - - timer.report('build dataloaders') - - # Auto-encoder definition - generator = AutoencoderKL( - spatial_dims=2, in_channels=1, out_channels=1, num_channels=(64, 128, 256), - latent_channels=1, num_res_blocks=2, norm_num_groups=32, norm_eps=1e-06, - attention_levels=(False, False, False), with_encoder_nonlocal_attn=True, - with_decoder_nonlocal_attn=True, - ) - generator = generator.to(device) - - timer.report('generator to device') - - # Discriminator definition - discriminator = PatchDiscriminator( - spatial_dims=2, num_layers_d=3, num_channels=32, - in_channels=1, out_channels=1, norm="INSTANCE" - ) - discriminator = discriminator.to(device) - - timer.report('discriminator to device') - - # Autoencoder loss functions - adv_loss = PatchAdversarialLoss(criterion="least_squares") - perceptual_loss = PerceptualLoss( - spatial_dims=2, network_type="resnet50", pretrained=True, #ImageNet pretrained weights used - ) - perceptual_loss.to(device) - - timer.report('loss functions') - - # Prepare for distributed training - generator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) - discriminator = torch.nn.SyncBatchNorm.convert_sync_batchnorm(generator) - - generator_without_ddp = generator - discriminator_without_ddp = discriminator - if args.distributed: - generator = torch.nn.parallel.DistributedDataParallel(generator, device_ids=[args.gpu], find_unused_parameters=True) # find_unused_parameters necessary for monai training - discriminator = torch.nn.parallel.DistributedDataParallel(discriminator, device_ids=[args.gpu], find_unused_parameters=True) # find_unused_parameters necessary for monai training - generator_without_ddp = generator.module - discriminator_without_ddp = discriminator.module - - timer.report('models prepped for distribution') - - # Optimizers - optimizer_g = torch.optim.Adam(generator_without_ddp.parameters(), lr=5e-5) - optimizer_d = torch.optim.Adam(discriminator_without_ddp.parameters(), lr=5e-5) - - timer.report('optimizers') - - # For mixed precision training - scaler_g = GradScaler() - scaler_d = GradScaler() - - timer.report('grad scalers') - - # Init metric tracker - metrics = {'train': MetricsTracker(), 'val': MetricsTracker()} - - # RETRIEVE CHECKPOINT - Path(args.resume).parent.mkdir(parents=True, exist_ok=True) - checkpoint = None - if args.resume and os.path.isfile(args.resume): # If we're resuming... - checkpoint = torch.load(args.resume, map_location="cpu") - elif args.prev_resume and os.path.isfile(args.prev_resume): - checkpoint = torch.load(args.prev_resume, map_location="cpu") - if checkpoint is not None: - args.start_epoch = checkpoint["epoch"] - generator_without_ddp.load_state_dict(checkpoint["generator"]) - discriminator_without_ddp.load_state_dict(checkpoint["discriminator"]) - optimizer_g.load_state_dict(checkpoint["optimizer_g"]) - optimizer_d.load_state_dict(checkpoint["optimizer_d"]) - scaler_g.load_state_dict(checkpoint["scaler_g"]) - scaler_d.load_state_dict(checkpoint["scaler_d"]) - train_sampler.load_state_dict(checkpoint["train_sampler"]) - val_sampler.load_state_dict(checkpoint["val_sampler"]) - # Metrics - metrics = checkpoint["metrics"] - - timer.report('checkpoint retrieval') - - ## -- TRAINING THE AUTO-ENCODER - ## - - n_gen_epochs = 100_000 - gen_val_interval = 1 - - for epoch in range(args.start_epoch, n_gen_epochs): - - print('\n') - print(f"EPOCH :: {epoch}") - print('\n') - - with train_sampler.in_epoch(epoch): - timer = TimestampedTimer("Start training") - generator, timer, metrics = train_generator_one_epoch( - args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, - scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, metrics - ) - timer.report(f'training generator for epoch {epoch}') - - if epoch % gen_val_interval == 0: # Eval every epoch - with val_sampler.in_epoch(epoch): - timer = TimestampedTimer("Start evaluation") - timer, metrics = evaluate_generator( - args, epoch, generator, discriminator, optimizer_g, optimizer_d, train_sampler, val_sampler, - scaler_g, scaler_d, train_loader, val_loader, perceptual_loss, adv_loss, device, timer, metrics - ) - timer.report(f'evaluating generator for epoch {epoch}') - - -if __name__ == "__main__": - args = get_args_parser().parse_args() - main(args, timer) diff --git a/monai_brats_mri_2d/utils.py b/monai_brats_mri_2d/utils.py deleted file mode 100644 index 1b268b82..00000000 --- a/monai_brats_mri_2d/utils.py +++ /dev/null @@ -1,72 +0,0 @@ -import torch, os, errno -import torch.distributed as dist - -def mkdir(path): - try: - os.makedirs(path) - except OSError as e: - if e.errno != errno.EEXIST: - raise - -def setup_for_distributed(is_master): - """ - This function disables printing when not in master process - """ - import builtins as __builtin__ - - builtin_print = __builtin__.print - - def print(*args, **kwargs): - force = kwargs.pop("force", False) - if is_master or force: - builtin_print(*args, **kwargs) - - __builtin__.print = print - -def init_distributed_mode(args): - if "RANK" in os.environ and "WORLD_SIZE" in os.environ: - args.rank = int(os.environ["RANK"]) - args.world_size = int(os.environ["WORLD_SIZE"]) - args.gpu = int(os.environ["LOCAL_RANK"]) - elif "SLURM_PROCID" in os.environ: - args.rank = int(os.environ["SLURM_PROCID"]) - args.gpu = args.rank % torch.cuda.device_count() - else: - print("Not using distributed mode") - args.distributed = False - return - - args.distributed = True - - torch.cuda.set_device(args.gpu) - args.dist_backend = "nccl" - print(f"| distributed init (rank {args.rank}): {args.dist_url}", flush=True) - torch.distributed.init_process_group( - backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank - ) - torch.distributed.barrier() - setup_for_distributed(args.rank == 0) - - -def is_dist_avail_and_initialized(): - if not dist.is_available(): - return False - if not dist.is_initialized(): - return False - return True - - -def get_world_size(): - if not is_dist_avail_and_initialized(): - return 1 - return dist.get_world_size() - - -def get_rank(): - if not is_dist_avail_and_initialized(): - return 0 - return dist.get_rank() - - -def is_main_process(): - return get_rank() == 0 \ No newline at end of file From addc304f26e501121088d812c82948c3521cf5ca Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Fri, 22 Sep 2023 13:27:26 +1000 Subject: [PATCH 41/44] updated readmes --- tv-detection/README.md | 71 +++------------------------------------ tv-segmentation/README.md | 36 +++----------------- 2 files changed, 9 insertions(+), 98 deletions(-) diff --git a/tv-detection/README.md b/tv-detection/README.md index d9af2652..fcd2e838 100644 --- a/tv-detection/README.md +++ b/tv-detection/README.md @@ -12,77 +12,14 @@ pycocotools matplotlib ``` -You must modify the following flags: - -`--data-path=/path/to/coco/dataset` - -`--nproc_per_node=` - -Except otherwise noted, all models have been trained on 8x V100 GPUs. - -### Faster R-CNN ResNet-50 FPN -``` -torchrun --nproc_per_node=8 train.py\ - --dataset coco --model fasterrcnn_resnet50_fpn --epochs 26\ - --lr-steps 16 22 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 -``` - -### Faster R-CNN MobileNetV3-Large FPN -``` -torchrun --nproc_per_node=8 train.py\ - --dataset coco --model fasterrcnn_mobilenet_v3_large_fpn --epochs 26\ - --lr-steps 16 22 --aspect-ratio-group-factor 3 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 -``` - -### Faster R-CNN MobileNetV3-Large 320 FPN -``` -torchrun --nproc_per_node=8 train.py\ - --dataset coco --model fasterrcnn_mobilenet_v3_large_320_fpn --epochs 26\ - --lr-steps 16 22 --aspect-ratio-group-factor 3 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 -``` - -### FCOS ResNet-50 FPN -``` -torchrun --nproc_per_node=8 train.py\ - --dataset coco --model fcos_resnet50_fpn --epochs 26\ - --lr-steps 16 22 --aspect-ratio-group-factor 3 --lr 0.01 --amp --weights-backbone ResNet50_Weights.IMAGENET1K_V1 -``` +You can then run the training routines for the following models using cli. ### RetinaNet ``` -torchrun --nproc_per_node=8 train.py\ - --dataset coco --model retinanet_resnet50_fpn --epochs 26\ - --lr-steps 16 22 --aspect-ratio-group-factor 3 --lr 0.01 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 -``` - -### SSD300 VGG16 -``` -torchrun --nproc_per_node=8 train.py\ - --dataset coco --model ssd300_vgg16 --epochs 120\ - --lr-steps 80 110 --aspect-ratio-group-factor 3 --lr 0.002 --batch-size 4\ - --weight-decay 0.0005 --data-augmentation ssd --weights-backbone VGG16_Weights.IMAGENET1K_FEATURES -``` - -### SSDlite320 MobileNetV3-Large -``` -torchrun --nproc_per_node=8 train.py\ - --dataset coco --model ssdlite320_mobilenet_v3_large --epochs 660\ - --aspect-ratio-group-factor 3 --lr-scheduler cosineannealinglr --lr 0.15 --batch-size 24\ - --weight-decay 0.00004 --data-augmentation ssdlite +isc train ./retinanet_resnet101_fpn.isc ``` - ### Mask R-CNN ``` -torchrun --nproc_per_node=8 train.py\ - --dataset coco --model maskrcnn_resnet50_fpn --epochs 26\ - --lr-steps 16 22 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 -``` - - -### Keypoint R-CNN -``` -torchrun --nproc_per_node=8 train.py\ - --dataset coco_kp --model keypointrcnn_resnet50_fpn --epochs 46\ - --lr-steps 36 43 --aspect-ratio-group-factor 3 --weights-backbone ResNet50_Weights.IMAGENET1K_V1 -``` +isc train ./maskrcnn_resnet101_fpn.isc +``` \ No newline at end of file diff --git a/tv-segmentation/README.md b/tv-segmentation/README.md index 2c7391c8..c507f90c 100644 --- a/tv-segmentation/README.md +++ b/tv-segmentation/README.md @@ -4,40 +4,14 @@ This folder contains reference training scripts for semantic segmentation. They serve as a log of how to train specific models, as provide baseline training and evaluation scripts to quickly bootstrap research. -All models have been trained on 8x V100 GPUs. +You can run the training routines for the following models using cli. -You must modify the following flags: - -`--data-path=/path/to/dataset` - -`--nproc_per_node=` - -## fcn_resnet50 -``` -torchrun --nproc_per_node=8 train.py --lr 0.02 --dataset coco -b 4 --model fcn_resnet50 --aux-loss --weights-backbone ResNet50_Weights.IMAGENET1K_V1 -``` - -## fcn_resnet101 -``` -torchrun --nproc_per_node=8 train.py --lr 0.02 --dataset coco -b 4 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 -``` - -## deeplabv3_resnet50 -``` -torchrun --nproc_per_node=8 train.py --lr 0.02 --dataset coco -b 4 --model deeplabv3_resnet50 --aux-loss --weights-backbone ResNet50_Weights.IMAGENET1K_V1 -``` - -## deeplabv3_resnet101 -``` -torchrun --nproc_per_node=8 train.py --lr 0.02 --dataset coco -b 4 --model deeplabv3_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 -``` - -## deeplabv3_mobilenet_v3_large +### RetinaNet ``` -torchrun --nproc_per_node=8 train.py --dataset coco -b 4 --model deeplabv3_mobilenet_v3_large --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 +isc train ./retinanet_resnet101_fpn.isc ``` -## lraspp_mobilenet_v3_large +### Mask R-CNN ``` -torchrun --nproc_per_node=8 train.py --dataset coco -b 4 --model lraspp_mobilenet_v3_large --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 +isc train ./maskrcnn_resnet101_fpn.isc ``` From 73531f4e95ba7a24defa1992b9fd19ea67d307fe Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Fri, 22 Sep 2023 13:28:47 +1000 Subject: [PATCH 42/44] updated readme with ref to requirements --- tv-segmentation/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tv-segmentation/README.md b/tv-segmentation/README.md index c507f90c..0386e747 100644 --- a/tv-segmentation/README.md +++ b/tv-segmentation/README.md @@ -4,6 +4,8 @@ This folder contains reference training scripts for semantic segmentation. They serve as a log of how to train specific models, as provide baseline training and evaluation scripts to quickly bootstrap research. +You must ensure all dependencies in "requirements.txt" are installed. + You can run the training routines for the following models using cli. ### RetinaNet From 0d099047f3719197c78718cd8bf1106e00646c5b Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Fri, 22 Sep 2023 14:03:04 +1000 Subject: [PATCH 43/44] removed resuming dir arg from fcn_resnet101.isc --- tv-detection/README.md | 3 +++ tv-segmentation/README.md | 4 +++- tv-segmentation/fcn_resnet101.isc | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tv-detection/README.md b/tv-detection/README.md index fcd2e838..bb211ba1 100644 --- a/tv-detection/README.md +++ b/tv-detection/README.md @@ -12,6 +12,9 @@ pycocotools matplotlib ``` +You must also run "prep.py" to download pretrained model weights before +launching your training job. + You can then run the training routines for the following models using cli. ### RetinaNet diff --git a/tv-segmentation/README.md b/tv-segmentation/README.md index 0386e747..7226f371 100644 --- a/tv-segmentation/README.md +++ b/tv-segmentation/README.md @@ -4,7 +4,9 @@ This folder contains reference training scripts for semantic segmentation. They serve as a log of how to train specific models, as provide baseline training and evaluation scripts to quickly bootstrap research. -You must ensure all dependencies in "requirements.txt" are installed. +You must ensure all dependencies in "requirements.txt" are installed, and +run "prep.py" to download pretrained model weights before launching your +training job. You can run the training routines for the following models using cli. diff --git a/tv-segmentation/fcn_resnet101.isc b/tv-segmentation/fcn_resnet101.isc index aef54fbb..c714d4a6 100644 --- a/tv-segmentation/fcn_resnet101.isc +++ b/tv-segmentation/fcn_resnet101.isc @@ -4,4 +4,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_fcn_resnet101" -command="train_cycling.py --lr 0.02 --dataset coco -b 4 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path $OUTPUT_PATH/tb --prev-resume /mnt/Client/StrongUniversity/USYD-04/usyd04_adam/output_fcn_resnet101/exp_1847/checkpoint.isc" \ No newline at end of file +command="train_cycling.py --lr 0.02 --dataset coco -b 4 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path $OUTPUT_PATH/tb" \ No newline at end of file From e043663d891716c33d3d52271c915e7637117a05 Mon Sep 17 00:00:00 2001 From: usyd04_adam Date: Mon, 25 Sep 2023 15:47:52 +1000 Subject: [PATCH 44/44] tackling instabilities --- tv-detection/engine.py | 45 ++++++++++++++++--- tv-detection/retinanet_resnet101_fpn.isc | 2 +- tv-detection/train_cycling.py | 5 +-- .../deeplabv3_mobilenet_v3_large.isc | 2 +- tv-segmentation/fcn_resnet101.isc | 2 +- 5 files changed, 43 insertions(+), 13 deletions(-) diff --git a/tv-detection/engine.py b/tv-detection/engine.py index 4b64de73..9a43586c 100644 --- a/tv-detection/engine.py +++ b/tv-detection/engine.py @@ -34,6 +34,7 @@ def train_one_epoch( optimizer.zero_grad() with torch.cuda.amp.autocast(enabled=scaler is not None): + assert len(targets) > 0, "Targets iterable of length 0, will return infinite loss." loss_dict = model(images, targets) # CHECK IF NUMERIC ERROR HAS OCCURRED AND IF SO, SKIP THIS BATCH @@ -42,13 +43,38 @@ def train_one_epoch( check_tensor = torch.tensor([check_0, check_1], requires_grad=False, device=device) dist.all_reduce(check_tensor, op=dist.ReduceOp.SUM) if check_tensor.sum() > 0: - print(f"CONTINUE CONDITION: {[e for e in check_tensor]}") - train_sampler.advance() # Advance sampler to try next batch + print(f"CONTINUE CONDITION - NaN: {check_tensor[0].item()}, Infinite: {check_tensor[1].item()}") + + # reset optimizer to prevent momentum carrying model into same issue + del optimizer + if args.norm_weight_decay is None: + parameters = [p for p in model.parameters() if p.requires_grad] + else: + param_groups = torchvision.ops._utils.split_normalization_params(model) + wd_groups = [args.norm_weight_decay, args.weight_decay] + parameters = [{"params": p, "weight_decay": w} for p, w in zip(param_groups, wd_groups) if p] + + opt_name = args.opt.lower() + if opt_name.startswith("sgd"): + optimizer = torch.optim.SGD( + parameters, lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay, + nesterov="nesterov" in opt_name, + ) + elif opt_name == "adamw": + optimizer = torch.optim.AdamW(parameters, lr=args.lr, weight_decay=args.weight_decay) + + # Advance sampler to try next batch + train_sampler.advance() continue losses = sum(loss for loss in loss_dict.values()) timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: forward pass') + # trying gradient clipping to prevent gradient issues with retinanet... + if args.model == 'retinanet_resnet101_fpn': + torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5) + timer.report(f'Epoch: {epoch} batch {train_sampler.progress}: graient clipping') + if scaler is not None: scaler.scale(losses).backward() scaler.step(optimizer) @@ -197,10 +223,17 @@ def evaluate( coco_evaluator.accumulate() results = coco_evaluator.summarize() - metric_A = ["bbox-", "segm-"] - metric_B = ["AP", "AR"] - metric_C = ["", "50", "75", "-S", "-M", "-L"] - metric_names = ["".join(t) for t in product(metric_A, metric_B, metric_C)] + # metric_A = ["bbox-", "segm-"] + # metric_B = ["AP", "AR"] + # metric_C = ["", "50", "75", "-S", "-M", "-L"] + # metric_names = ["".join(t) for t in product(metric_A, metric_B, metric_C)] + metric_names = [ + "bbox/AP", "bbox/AP-50", "bbox/AP-75", "bbox/AP-S", "bbox/AP-M", "bbox/AP-L", + "bbox/AR-MD1", "bbox/AR-MD10", "bbox/AR-MD100", "bbox/AR-S", "bbox/AR-M", "bbox/AR-L" + ] + [ + "segm/AP", "segm/AP-50", "segm/AP-75", "segm/AP-S", "segm/AP-M", "segm/AP-L", + "segm/AR-MD1", "segm/AR-MD10", "segm/AR-MD100", "segm/AR-S", "segm/AR-M", "segm/AR-L" + ] metrics["val"].update({name: val for name,val in zip(metric_names, results)}) metrics["val"].reduce() metrics["val"].end_epoch() diff --git a/tv-detection/retinanet_resnet101_fpn.isc b/tv-detection/retinanet_resnet101_fpn.isc index e6606711..5a930462 100644 --- a/tv-detection/retinanet_resnet101_fpn.isc +++ b/tv-detection/retinanet_resnet101_fpn.isc @@ -3,5 +3,5 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_retinanet_resnet101_fpn" -command="train_cycling.py --dataset coco --model retinanet_resnet101_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --lr 0.01 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path $OUTPUT_PATH/tb" +command="train_cycling.py --dataset coco --model retinanet_resnet101_fpn --epochs 26 --lr-steps 16 22 -b 2 --aspect-ratio-group-factor 3 --lr 0.0001 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path $OUTPUT_PATH/tb" diff --git a/tv-detection/train_cycling.py b/tv-detection/train_cycling.py index 5dbe0fb8..e2997da1 100644 --- a/tv-detection/train_cycling.py +++ b/tv-detection/train_cycling.py @@ -159,10 +159,7 @@ def main(args, timer): opt_name = args.opt.lower() if opt_name.startswith("sgd"): optimizer = torch.optim.SGD( - parameters, - lr=args.lr, - momentum=args.momentum, - weight_decay=args.weight_decay, + parameters, lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay, nesterov="nesterov" in opt_name, ) elif opt_name == "adamw": diff --git a/tv-segmentation/deeplabv3_mobilenet_v3_large.isc b/tv-segmentation/deeplabv3_mobilenet_v3_large.isc index 2a7c8e34..c9999efe 100644 --- a/tv-segmentation/deeplabv3_mobilenet_v3_large.isc +++ b/tv-segmentation/deeplabv3_mobilenet_v3_large.isc @@ -4,4 +4,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_deeplabv3_mobilenet_v3_large" -command="train_cycling.py --dataset coco -b 2 --model deeplabv3_mobilenet_v3_large --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path $OUTPUT_PATH/tb" +command="train_cycling.py --dataset coco -b 2 --model deeplabv3_mobilenet_v3_large --lr 0.001 --aux-loss --wd 0.000001 --weights-backbone MobileNet_V3_Large_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path $OUTPUT_PATH/tb" diff --git a/tv-segmentation/fcn_resnet101.isc b/tv-segmentation/fcn_resnet101.isc index c714d4a6..a6dbb0bf 100644 --- a/tv-segmentation/fcn_resnet101.isc +++ b/tv-segmentation/fcn_resnet101.isc @@ -4,4 +4,4 @@ gpu_type="24GB VRAM GPU" nnodes = 11 venv_path = "~/.venv/bin/activate" output_path = "~/output_fcn_resnet101" -command="train_cycling.py --lr 0.02 --dataset coco -b 4 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path $OUTPUT_PATH/tb" \ No newline at end of file +command="train_cycling.py --lr 0.001 --dataset coco -b 4 --model fcn_resnet101 --aux-loss --weights-backbone ResNet101_Weights.IMAGENET1K_V1 --data-path=/mnt/.node1/Open-Datsets/coco --resume $OUTPUT_PATH/checkpoint.isc --tboard-path $OUTPUT_PATH/tb" \ No newline at end of file