- added docker-compose.yml file

- start pushing to docker images to gcr.io
- added a .dockerignore file

- moved models and utils directory into yolo directory for better package management

- fixed import paths to conform with new directory structure
- added setup.py
- install yolo as a package in docker image

- changed the default label cache location so it doesn't mess with DVC
- fixed a run-time error

- use correct label name for single class

- set working_dir in docker-compose.yml

- add version tag to pushed image

- push tagged docker image

.dockerignore

@@ -0,0 +1,2 @@
+runs
+weights

.env

@@ -0,0 +1 @@
+TAG=0.1.0

Dockerfile

@@ -0,0 +1,55 @@
+#FROM nvcr.io/nvidia/pytorch:20.06-py3
+#FROM continuumio/miniconda3
+#FROM gpuci/miniconda-cuda:11.0-devel-ubuntu20.04
+FROM nvidia/cuda:11.1-cudnn8-devel-ubuntu20.04
+
+#COPY environment.yml .
+#RUN conda update --force-reinstall conda
+#RUN conda env update --name base --file environment.yml --prune
+
+ENV MY_ROOT=/workspace \
+    PKG_PATH=/yolo_src \
+    NUMPROC=4 \
+    PYTHON_VER=3.8 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONPATH=. \
+    DEBIAN_FRONTEND=noninteractive 
+
+WORKDIR $PKG_PATH
+
+RUN apt-get update && apt-get install -y apt-utils && apt-get -y upgrade && \
+    apt-get install -y git libsnappy-dev libopencv-dev libhdf5-serial-dev libboost-all-dev libatlas-base-dev \
+        libgflags-dev libgoogle-glog-dev liblmdb-dev curl unzip\
+        python${PYTHON_VER}-dev && \
+    curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python${PYTHON_VER} get-pip.py && \
+    rm get-pip.py && \
+    # Clean UP
+    apt upgrade -y && \
+    apt clean && \
+    apt autoremove -y && \
+    rm -rf /var/lib/apt/lists/*  # cleanup to reduce image size
+
+RUN ln -s /usr/bin/python${PYTHON_VER} /usr/bin/python
+
+RUN apt install unzip
+RUN pip install torch==1.7.1+cu110 torchvision==0.8.2+cu110 -f https://download.pytorch.org/whl/torch_stable.html
+
+WORKDIR $MY_ROOT
+# We have to install mish-cuda from source due to an issue with one of the header files
+ADD https://github.com/thomasbrandon/mish-cuda/archive/master.zip $MY_ROOT/mish-cuda.zip
+RUN unzip mish-cuda.zip
+WORKDIR $MY_ROOT/mish-cuda-master
+RUN cp external/CUDAApplyUtils.cuh csrc/
+RUN python setup.py build install
+WORKDIR $PKG_PATH
+# ADD https://drive.google.com/file/d/1NQwz47cW0NUgy7L3_xOKaNEfLoQuq3EL/view?usp=sharing /weights/yolov4-csp.weights
+ADD requirements.txt $PKG_PATH/requirements.txt
+RUN pip install -r $PKG_PATH/requirements.txt
+ADD yolo $PKG_PATH/yolo
+ADD train.py $PKG_PATH/train.py
+ADD test.py $PKG_PATH/test.py
+ADD setup.py $PKG_PATH/setup.py
+ADD data $PKG_PATH/data
+RUN pip install .
+

data/speedco.yaml

@@ -1,7 +1,7 @@
 # train and val datasets (image directory or *.txt file with image paths)
-train: data/speedco_train_images.txt
-val: data/speedco_val_images.txt
-test: data/speedco_val_images.txt
+train: data/speedco_dataset_train_images.txt
+val: data/speedco_dataset_val_images.txt
+test: data/speedco_dataset_val_images.txt
 
 # number of classes
 nc: 1

docker-compose.yml

@@ -0,0 +1,24 @@
+---
+version: '2.3'
+
+services:
+  train:
+    image: gcr.io/kinsol-generic/yolov4-csp:${TAG:-dev}
+    build:
+      context: .
+      dockerfile: ./Dockerfile
+    runtime: nvidia
+    volumes:
+      - .:/home/dev
+      - /mnt/NAS/Production/TruckBay/:/mnt/NAS/Production/TruckBay
+      - /mnt/NAS/Public/parque_research/datasets/coco_yolo/coco:/mnt/coco
+      - /home/kweston/darknet_utils:/home/kweston/darknet_utils
+      - /home/kweston/speedco/baywatchr-inference/speedco_dataset:/mnt/speedco_dataset
+      - /home/kweston/speedco/baywatchr-inference/data/lists:/mnt/speedco_datalists
+      - /data/kweston/sandbox/mlannotation/results:/results
+    environment:
+      - GOOGLE_APPLICATION_CREDENTIALS=/app/baywatchr-api-key.json
+    command: 
+      - bash
+    shm_size: 64g
+    working_dir: /home/dev

run_train.sh

@@ -1,3 +1 @@
-#!/bin/bash
-# Used for training with 3 GTX 1070 GPUs
-python -m torch.distributed.launch --nproc_per_node 3 train.py --device 0,1,2 --batch-size 21 --data speedco.yaml --weights '' --cfg yolov4-csp.cfg --name yolov4-csp-speedco --sync-bn --rect --single-cl
+python -m torch.distributed.launch --nproc_per_node 3 train.py --device 0,1,2 --batch-size 21 --data speedco.yaml --weights  --cfg yolov4-csp-single-class.cfg --name yolov4-csp-speedco --sync-bn --rect --single-cls

setup.py

@@ -0,0 +1,20 @@
+import os
+from setuptools import setup, find_namespace_packages
+
+
+def readlines(fname):
+    with open(os.path.join(os.path.dirname(__file__), fname)) as f:
+        return f.readlines()
+
+
+install_requires = readlines('requirements.txt')
+
+
+setup(
+    name='yolov4-csp',
+    version='1.0.0',
+    install_requires=install_requires,
+    packages=find_namespace_packages(include=['yolo', 'yolo.*']),
+    include_package_data=True,
+    python_requires='>=3.7'
+)

test.py

@@ -2,23 +2,24 @@
 import glob
 import json
 import os
+import shutil
 from pathlib import Path
 
 import numpy as np
 import torch
 import yaml
 from tqdm import tqdm
 
-from utils.google_utils import attempt_load
-from utils.datasets import create_dataloader
-from utils.general import coco80_to_coco91_class, check_dataset, check_file, check_img_size, box_iou, \
+from yolo.utils.google_utils import attempt_load
+from yolo.utils.datasets import create_dataloader
+from yolo.utils.general import coco80_to_coco91_class, check_dataset, check_file, check_img_size, box_iou, \
     non_max_suppression, scale_coords, xyxy2xywh, xywh2xyxy, clip_coords, set_logging, increment_path
-from utils.loss import compute_loss
-from utils.metrics import ap_per_class
-from utils.plots import plot_images, output_to_target
-from utils.torch_utils import select_device, time_synchronized
+from yolo.utils.loss import compute_loss
+from yolo.utils.metrics import ap_per_class
+from yolo.utils.plots import plot_images, output_to_target
+from yolo.utils.torch_utils import select_device, time_synchronized
 
-from models.models import *
+from yolo.models.models import *
 
 def load_classes(path):
     # Loads *.names file at 'path'
@@ -27,6 +28,7 @@ def load_classes(path):
     return list(filter(None, names))  # filter removes empty strings (such as last line)
 
 
+
 def test(data,
          weights=None,
          batch_size=16,

train.py

@@ -22,16 +22,16 @@
 
 import test  # import test.py to get mAP after each epoch
 #from models.yolo import Model
-from models.models import *
-from utils.autoanchor import check_anchors
-from utils.datasets import create_dataloader
-from utils.general import labels_to_class_weights, increment_path, labels_to_image_weights, init_seeds, \
+from yolo.models.models import *
+from yolo.utils.autoanchor import check_anchors
+from yolo.utils.datasets import create_dataloader
+from yolo.utils.general import labels_to_class_weights, increment_path, labels_to_image_weights, init_seeds, \
     fitness, fitness_p, fitness_r, fitness_ap50, fitness_ap, fitness_f, strip_optimizer, get_latest_run,\
     check_dataset, check_file, check_git_status, check_img_size, print_mutation, set_logging
-from utils.google_utils import attempt_download
-from utils.loss import compute_loss
-from utils.plots import plot_images, plot_labels, plot_results, plot_evolution
-from utils.torch_utils import ModelEMA, select_device, intersect_dicts, torch_distributed_zero_first
+from yolo.utils.google_utils import attempt_download
+from yolo.utils.loss import compute_loss
+from yolo.utils.plots import plot_images, plot_labels, plot_results, plot_evolution
+from yolo.utils.torch_utils import ModelEMA, select_device, intersect_dicts, torch_distributed_zero_first
 
 logger = logging.getLogger(__name__)
 
@@ -69,7 +69,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None):
         check_dataset(data_dict)  # check
     train_path = data_dict['train']
     test_path = data_dict['val']
-    nc, names = (1, ['item']) if opt.single_cls else (int(data_dict['nc']), data_dict['names'])  # number classes, names
+    nc, names = (1, data_dict['names']) if opt.single_cls else (int(data_dict['nc']), data_dict['names'])  # number classes, names
     assert len(names) == nc, '%g names found for nc=%g dataset in %s' % (len(names), nc, opt.data)  # check
 
     # Model

models/export.py → yolo/models/export.py

@@ -2,7 +2,7 @@
 
 import torch
 
-from utils.google_utils import attempt_download
+from yolo.utils.google_utils import attempt_download
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()

models/models.py → yolo/models/models.py

@@ -1,7 +1,7 @@
-from utils.google_utils import *
-from utils.layers import *
-from utils.parse_config import *
-from utils import torch_utils
+from yolo.utils.google_utils import *
+from yolo.utils.layers import *
+from yolo.utils.parse_config import *
+from yolo.utils import torch_utils
 
 ONNX_EXPORT = False
 

utils/datasets.py → yolo/utils/datasets.py

@@ -23,8 +23,8 @@
 from pycocotools import mask as maskUtils
 from torchvision.utils import save_image
 
-from utils.general import xyxy2xywh, xywh2xyxy
-from utils.torch_utils import torch_distributed_zero_first
+from yolo.utils.general import xyxy2xywh, xywh2xyxy
+from yolo.utils.torch_utils import torch_distributed_zero_first
 
 # Parameters
 help_url = 'https://github.com/ultralytics/yolov5/wiki/Train-Custom-Data'
@@ -649,7 +649,7 @@ def __init__(self, path, img_size=640, batch_size=16, augment=False, hyp=None, r
 
         def img2label_paths(img_paths):
             # Define label paths as a function of image paths
-            sa, sb = os.sep + 'images' + os.sep, os.sep + 'labels' + os.sep  # /images/, /labels/ substrings
+            sa, sb = os.sep + 'JPEGImages' + os.sep, os.sep + 'labels' + os.sep  # /images/, /labels/ substrings
             return [x.replace(sa, sb, 1).replace(x.split('.')[-1], 'txt') for x in img_paths]
 
         try:

utils/general.py → yolo/utils/general.py

@@ -17,9 +17,9 @@
 import torch
 import yaml
 
-from utils.google_utils import gsutil_getsize
-from utils.metrics import fitness, fitness_p, fitness_r, fitness_ap50, fitness_ap, fitness_f   
-from utils.torch_utils import init_torch_seeds
+from yolo.utils.google_utils import gsutil_getsize
+from yolo.utils.metrics import fitness, fitness_p, fitness_r, fitness_ap50, fitness_ap, fitness_f   
+from yolo.utils.torch_utils import init_torch_seeds
 
 # Set printoptions
 torch.set_printoptions(linewidth=320, precision=5, profile='long')

utils/layers.py → yolo/utils/layers.py

@@ -1,6 +1,6 @@
 import torch.nn.functional as F
 
-from utils.general import *
+from yolo.utils.general import *
 
 import torch
 from torch import nn

utils/loss.py → yolo/utils/loss.py

@@ -3,8 +3,8 @@
 import torch
 import torch.nn as nn
 
-from utils.general import bbox_iou
-from utils.torch_utils import is_parallel
+from yolo.utils.general import bbox_iou
+from yolo.utils.torch_utils import is_parallel
 
 
 def smooth_BCE(eps=0.1):  # https://github.com/ultralytics/yolov3/issues/238#issuecomment-598028441

utils/plots.py → yolo/utils/plots.py

@@ -16,8 +16,8 @@
 from PIL import Image
 from scipy.signal import butter, filtfilt
 
-from utils.general import xywh2xyxy, xyxy2xywh
-from utils.metrics import fitness
+from yolo.utils.general import xywh2xyxy, xyxy2xywh
+from yolo.utils.metrics import fitness
 
 # Settings
 matplotlib.use('Agg')  # for writing to files only