diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..bbceb85 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +runs +weights diff --git a/.env b/.env new file mode 100644 index 0000000..cdbc703 --- /dev/null +++ b/.env @@ -0,0 +1 @@ +TAG=0.1.0 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..5f5be47 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,55 @@ +#FROM nvcr.io/nvidia/pytorch:20.06-py3 +#FROM continuumio/miniconda3 +#FROM gpuci/miniconda-cuda:11.0-devel-ubuntu20.04 +FROM nvidia/cuda:11.1-cudnn8-devel-ubuntu20.04 + +#COPY environment.yml . +#RUN conda update --force-reinstall conda +#RUN conda env update --name base --file environment.yml --prune + +ENV MY_ROOT=/workspace \ + PKG_PATH=/yolo_src \ + NUMPROC=4 \ + PYTHON_VER=3.8 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=. \ + DEBIAN_FRONTEND=noninteractive + +WORKDIR $PKG_PATH + +RUN apt-get update && apt-get install -y apt-utils && apt-get -y upgrade && \ + apt-get install -y git libsnappy-dev libopencv-dev libhdf5-serial-dev libboost-all-dev libatlas-base-dev \ + libgflags-dev libgoogle-glog-dev liblmdb-dev curl unzip\ + python${PYTHON_VER}-dev && \ + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python${PYTHON_VER} get-pip.py && \ + rm get-pip.py && \ + # Clean UP + apt upgrade -y && \ + apt clean && \ + apt autoremove -y && \ + rm -rf /var/lib/apt/lists/* # cleanup to reduce image size + +RUN ln -s /usr/bin/python${PYTHON_VER} /usr/bin/python + +RUN apt install unzip +RUN pip install torch==1.7.1+cu110 torchvision==0.8.2+cu110 -f https://download.pytorch.org/whl/torch_stable.html + +WORKDIR $MY_ROOT +# We have to install mish-cuda from source due to an issue with one of the header files +ADD https://github.com/thomasbrandon/mish-cuda/archive/master.zip $MY_ROOT/mish-cuda.zip +RUN unzip mish-cuda.zip +WORKDIR $MY_ROOT/mish-cuda-master +RUN cp external/CUDAApplyUtils.cuh csrc/ +RUN python setup.py build install +WORKDIR $PKG_PATH +# ADD https://drive.google.com/file/d/1NQwz47cW0NUgy7L3_xOKaNEfLoQuq3EL/view?usp=sharing /weights/yolov4-csp.weights +ADD requirements.txt $PKG_PATH/requirements.txt +RUN pip install -r $PKG_PATH/requirements.txt +ADD yolo $PKG_PATH/yolo +ADD train.py $PKG_PATH/train.py +ADD test.py $PKG_PATH/test.py +ADD setup.py $PKG_PATH/setup.py +ADD data $PKG_PATH/data +RUN pip install . + diff --git a/data/speedco.yaml b/data/speedco.yaml index 8d25b63..84b59b4 100644 --- a/data/speedco.yaml +++ b/data/speedco.yaml @@ -1,7 +1,7 @@ # train and val datasets (image directory or *.txt file with image paths) -train: data/speedco_train_images.txt -val: data/speedco_val_images.txt -test: data/speedco_val_images.txt +train: data/speedco_dataset_train_images.txt +val: data/speedco_dataset_val_images.txt +test: data/speedco_dataset_val_images.txt # number of classes nc: 1 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..146aa23 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,24 @@ +--- +version: '2.3' + +services: + train: + image: gcr.io/kinsol-generic/yolov4-csp:${TAG:-dev} + build: + context: . + dockerfile: ./Dockerfile + runtime: nvidia + volumes: + - .:/home/dev + - /mnt/NAS/Production/TruckBay/:/mnt/NAS/Production/TruckBay + - /mnt/NAS/Public/parque_research/datasets/coco_yolo/coco:/mnt/coco + - /home/kweston/darknet_utils:/home/kweston/darknet_utils + - /home/kweston/speedco/baywatchr-inference/speedco_dataset:/mnt/speedco_dataset + - /home/kweston/speedco/baywatchr-inference/data/lists:/mnt/speedco_datalists + - /data/kweston/sandbox/mlannotation/results:/results + environment: + - GOOGLE_APPLICATION_CREDENTIALS=/app/baywatchr-api-key.json + command: + - bash + shm_size: 64g + working_dir: /home/dev diff --git a/run_train.sh b/run_train.sh index a4caeda..0aae77f 100755 --- a/run_train.sh +++ b/run_train.sh @@ -1,3 +1 @@ -#!/bin/bash -# Used for training with 3 GTX 1070 GPUs -python -m torch.distributed.launch --nproc_per_node 3 train.py --device 0,1,2 --batch-size 21 --data speedco.yaml --weights '' --cfg yolov4-csp.cfg --name yolov4-csp-speedco --sync-bn --rect --single-cl \ No newline at end of file +python -m torch.distributed.launch --nproc_per_node 3 train.py --device 0,1,2 --batch-size 21 --data speedco.yaml --weights --cfg yolov4-csp-single-class.cfg --name yolov4-csp-speedco --sync-bn --rect --single-cls diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..7310b97 --- /dev/null +++ b/setup.py @@ -0,0 +1,20 @@ +import os +from setuptools import setup, find_namespace_packages + + +def readlines(fname): + with open(os.path.join(os.path.dirname(__file__), fname)) as f: + return f.readlines() + + +install_requires = readlines('requirements.txt') + + +setup( + name='yolov4-csp', + version='1.0.0', + install_requires=install_requires, + packages=find_namespace_packages(include=['yolo', 'yolo.*']), + include_package_data=True, + python_requires='>=3.7' +) diff --git a/test.py b/test.py index bd99b59..8e0df4e 100644 --- a/test.py +++ b/test.py @@ -2,6 +2,7 @@ import glob import json import os +import shutil from pathlib import Path import numpy as np @@ -9,16 +10,16 @@ import yaml from tqdm import tqdm -from utils.google_utils import attempt_load -from utils.datasets import create_dataloader -from utils.general import coco80_to_coco91_class, check_dataset, check_file, check_img_size, box_iou, \ +from yolo.utils.google_utils import attempt_load +from yolo.utils.datasets import create_dataloader +from yolo.utils.general import coco80_to_coco91_class, check_dataset, check_file, check_img_size, box_iou, \ non_max_suppression, scale_coords, xyxy2xywh, xywh2xyxy, clip_coords, set_logging, increment_path -from utils.loss import compute_loss -from utils.metrics import ap_per_class -from utils.plots import plot_images, output_to_target -from utils.torch_utils import select_device, time_synchronized +from yolo.utils.loss import compute_loss +from yolo.utils.metrics import ap_per_class +from yolo.utils.plots import plot_images, output_to_target +from yolo.utils.torch_utils import select_device, time_synchronized -from models.models import * +from yolo.models.models import * def load_classes(path): # Loads *.names file at 'path' @@ -27,6 +28,7 @@ def load_classes(path): return list(filter(None, names)) # filter removes empty strings (such as last line) + def test(data, weights=None, batch_size=16, diff --git a/train.py b/train.py index d7cbf1c..03bc917 100644 --- a/train.py +++ b/train.py @@ -22,16 +22,16 @@ import test # import test.py to get mAP after each epoch #from models.yolo import Model -from models.models import * -from utils.autoanchor import check_anchors -from utils.datasets import create_dataloader -from utils.general import labels_to_class_weights, increment_path, labels_to_image_weights, init_seeds, \ +from yolo.models.models import * +from yolo.utils.autoanchor import check_anchors +from yolo.utils.datasets import create_dataloader +from yolo.utils.general import labels_to_class_weights, increment_path, labels_to_image_weights, init_seeds, \ fitness, fitness_p, fitness_r, fitness_ap50, fitness_ap, fitness_f, strip_optimizer, get_latest_run,\ check_dataset, check_file, check_git_status, check_img_size, print_mutation, set_logging -from utils.google_utils import attempt_download -from utils.loss import compute_loss -from utils.plots import plot_images, plot_labels, plot_results, plot_evolution -from utils.torch_utils import ModelEMA, select_device, intersect_dicts, torch_distributed_zero_first +from yolo.utils.google_utils import attempt_download +from yolo.utils.loss import compute_loss +from yolo.utils.plots import plot_images, plot_labels, plot_results, plot_evolution +from yolo.utils.torch_utils import ModelEMA, select_device, intersect_dicts, torch_distributed_zero_first logger = logging.getLogger(__name__) @@ -69,7 +69,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None): check_dataset(data_dict) # check train_path = data_dict['train'] test_path = data_dict['val'] - nc, names = (1, ['item']) if opt.single_cls else (int(data_dict['nc']), data_dict['names']) # number classes, names + nc, names = (1, data_dict['names']) if opt.single_cls else (int(data_dict['nc']), data_dict['names']) # number classes, names assert len(names) == nc, '%g names found for nc=%g dataset in %s' % (len(names), nc, opt.data) # check # Model diff --git a/models/__init__.py b/yolo/models/__init__.py similarity index 100% rename from models/__init__.py rename to yolo/models/__init__.py diff --git a/models/export.py b/yolo/models/export.py similarity index 98% rename from models/export.py rename to yolo/models/export.py index 947a7a8..c6cc8cd 100644 --- a/models/export.py +++ b/yolo/models/export.py @@ -2,7 +2,7 @@ import torch -from utils.google_utils import attempt_download +from yolo.utils.google_utils import attempt_download if __name__ == '__main__': parser = argparse.ArgumentParser() diff --git a/models/models.py b/yolo/models/models.py similarity index 99% rename from models/models.py rename to yolo/models/models.py index 8b3bd2c..a93d0b8 100644 --- a/models/models.py +++ b/yolo/models/models.py @@ -1,7 +1,7 @@ -from utils.google_utils import * -from utils.layers import * -from utils.parse_config import * -from utils import torch_utils +from yolo.utils.google_utils import * +from yolo.utils.layers import * +from yolo.utils.parse_config import * +from yolo.utils import torch_utils ONNX_EXPORT = False diff --git a/models/yolov3-spp.cfg b/yolo/models/yolov3-spp.cfg similarity index 100% rename from models/yolov3-spp.cfg rename to yolo/models/yolov3-spp.cfg diff --git a/models/yolov4-csp-single-class.cfg b/yolo/models/yolov4-csp-single-class.cfg similarity index 100% rename from models/yolov4-csp-single-class.cfg rename to yolo/models/yolov4-csp-single-class.cfg diff --git a/models/yolov4-csp.cfg b/yolo/models/yolov4-csp.cfg similarity index 100% rename from models/yolov4-csp.cfg rename to yolo/models/yolov4-csp.cfg diff --git a/models/yolov4.cfg b/yolo/models/yolov4.cfg similarity index 100% rename from models/yolov4.cfg rename to yolo/models/yolov4.cfg diff --git a/utils/__init__.py b/yolo/utils/__init__.py similarity index 100% rename from utils/__init__.py rename to yolo/utils/__init__.py diff --git a/utils/activations.py b/yolo/utils/activations.py similarity index 100% rename from utils/activations.py rename to yolo/utils/activations.py diff --git a/utils/autoanchor.py b/yolo/utils/autoanchor.py similarity index 100% rename from utils/autoanchor.py rename to yolo/utils/autoanchor.py diff --git a/utils/datasets.py b/yolo/utils/datasets.py similarity index 99% rename from utils/datasets.py rename to yolo/utils/datasets.py index aa31808..06fd089 100644 --- a/utils/datasets.py +++ b/yolo/utils/datasets.py @@ -23,8 +23,8 @@ from pycocotools import mask as maskUtils from torchvision.utils import save_image -from utils.general import xyxy2xywh, xywh2xyxy -from utils.torch_utils import torch_distributed_zero_first +from yolo.utils.general import xyxy2xywh, xywh2xyxy +from yolo.utils.torch_utils import torch_distributed_zero_first # Parameters help_url = 'https://github.com/ultralytics/yolov5/wiki/Train-Custom-Data' @@ -649,7 +649,7 @@ def __init__(self, path, img_size=640, batch_size=16, augment=False, hyp=None, r def img2label_paths(img_paths): # Define label paths as a function of image paths - sa, sb = os.sep + 'images' + os.sep, os.sep + 'labels' + os.sep # /images/, /labels/ substrings + sa, sb = os.sep + 'JPEGImages' + os.sep, os.sep + 'labels' + os.sep # /images/, /labels/ substrings return [x.replace(sa, sb, 1).replace(x.split('.')[-1], 'txt') for x in img_paths] try: diff --git a/utils/general.py b/yolo/utils/general.py similarity index 98% rename from utils/general.py rename to yolo/utils/general.py index 0585f28..0cefcff 100644 --- a/utils/general.py +++ b/yolo/utils/general.py @@ -17,9 +17,9 @@ import torch import yaml -from utils.google_utils import gsutil_getsize -from utils.metrics import fitness, fitness_p, fitness_r, fitness_ap50, fitness_ap, fitness_f -from utils.torch_utils import init_torch_seeds +from yolo.utils.google_utils import gsutil_getsize +from yolo.utils.metrics import fitness, fitness_p, fitness_r, fitness_ap50, fitness_ap, fitness_f +from yolo.utils.torch_utils import init_torch_seeds # Set printoptions torch.set_printoptions(linewidth=320, precision=5, profile='long') diff --git a/utils/google_utils.py b/yolo/utils/google_utils.py similarity index 100% rename from utils/google_utils.py rename to yolo/utils/google_utils.py diff --git a/utils/layers.py b/yolo/utils/layers.py similarity index 99% rename from utils/layers.py rename to yolo/utils/layers.py index 1d46a18..1b5cb8e 100644 --- a/utils/layers.py +++ b/yolo/utils/layers.py @@ -1,6 +1,6 @@ import torch.nn.functional as F -from utils.general import * +from yolo.utils.general import * import torch from torch import nn diff --git a/utils/loss.py b/yolo/utils/loss.py similarity index 98% rename from utils/loss.py rename to yolo/utils/loss.py index 288b388..482a213 100644 --- a/utils/loss.py +++ b/yolo/utils/loss.py @@ -3,8 +3,8 @@ import torch import torch.nn as nn -from utils.general import bbox_iou -from utils.torch_utils import is_parallel +from yolo.utils.general import bbox_iou +from yolo.utils.torch_utils import is_parallel def smooth_BCE(eps=0.1): # https://github.com/ultralytics/yolov3/issues/238#issuecomment-598028441 diff --git a/utils/metrics.py b/yolo/utils/metrics.py similarity index 100% rename from utils/metrics.py rename to yolo/utils/metrics.py diff --git a/utils/parse_config.py b/yolo/utils/parse_config.py similarity index 100% rename from utils/parse_config.py rename to yolo/utils/parse_config.py diff --git a/utils/plots.py b/yolo/utils/plots.py similarity index 99% rename from utils/plots.py rename to yolo/utils/plots.py index c90a96b..a4df1ad 100644 --- a/utils/plots.py +++ b/yolo/utils/plots.py @@ -16,8 +16,8 @@ from PIL import Image from scipy.signal import butter, filtfilt -from utils.general import xywh2xyxy, xyxy2xywh -from utils.metrics import fitness +from yolo.utils.general import xywh2xyxy, xyxy2xywh +from yolo.utils.metrics import fitness # Settings matplotlib.use('Agg') # for writing to files only diff --git a/utils/torch_utils.py b/yolo/utils/torch_utils.py similarity index 100% rename from utils/torch_utils.py rename to yolo/utils/torch_utils.py