Rebase upstream

.dockerignore

@@ -0,0 +1,2 @@
+runs
+weights

.env

@@ -0,0 +1 @@
+TAG=0.1.0

Dockerfile

@@ -0,0 +1,55 @@
+#FROM nvcr.io/nvidia/pytorch:20.06-py3
+#FROM continuumio/miniconda3
+#FROM gpuci/miniconda-cuda:11.0-devel-ubuntu20.04
+FROM nvidia/cuda:11.1-cudnn8-devel-ubuntu20.04
+
+#COPY environment.yml .
+#RUN conda update --force-reinstall conda
+#RUN conda env update --name base --file environment.yml --prune
+
+ENV MY_ROOT=/workspace \
+    PKG_PATH=/yolo_src \
+    NUMPROC=4 \
+    PYTHON_VER=3.8 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONPATH=. \
+    DEBIAN_FRONTEND=noninteractive 
+
+WORKDIR $PKG_PATH
+
+RUN apt-get update && apt-get install -y apt-utils && apt-get -y upgrade && \
+    apt-get install -y git libsnappy-dev libopencv-dev libhdf5-serial-dev libboost-all-dev libatlas-base-dev \
+        libgflags-dev libgoogle-glog-dev liblmdb-dev curl unzip\
+        python${PYTHON_VER}-dev && \
+    curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python${PYTHON_VER} get-pip.py && \
+    rm get-pip.py && \
+    # Clean UP
+    apt upgrade -y && \
+    apt clean && \
+    apt autoremove -y && \
+    rm -rf /var/lib/apt/lists/*  # cleanup to reduce image size
+
+RUN ln -s /usr/bin/python${PYTHON_VER} /usr/bin/python
+
+RUN apt install unzip
+RUN pip install torch==1.7.1+cu110 torchvision==0.8.2+cu110 -f https://download.pytorch.org/whl/torch_stable.html
+
+WORKDIR $MY_ROOT
+# We have to install mish-cuda from source due to an issue with one of the header files
+ADD https://github.com/thomasbrandon/mish-cuda/archive/master.zip $MY_ROOT/mish-cuda.zip
+RUN unzip mish-cuda.zip
+WORKDIR $MY_ROOT/mish-cuda-master
+RUN cp external/CUDAApplyUtils.cuh csrc/
+RUN python setup.py build install
+WORKDIR $PKG_PATH
+# ADD https://drive.google.com/file/d/1NQwz47cW0NUgy7L3_xOKaNEfLoQuq3EL/view?usp=sharing /weights/yolov4-csp.weights
+ADD requirements.txt $PKG_PATH/requirements.txt
+RUN pip install -r $PKG_PATH/requirements.txt
+ADD yolo $PKG_PATH/yolo
+ADD train.py $PKG_PATH/train.py
+ADD test.py $PKG_PATH/test.py
+ADD setup.py $PKG_PATH/setup.py
+ADD data $PKG_PATH/data
+RUN pip install .
+

data/coco.yaml

@@ -1,7 +1,8 @@
 # train and val datasets (image directory or *.txt file with image paths)
-train: ../coco/train2017.txt  # 118k images
-val: ../coco/val2017.txt  # 5k images
-test: ../coco/testdev2017.txt  # 20k images for submission to https://competitions.codalab.org/competitions/20794
+train: /mnt/coco/5k.txt  # 118k images
+#val: /mnt/coco/5k.txt  # 5k images
+val: data/coco_5k_500_subset.txt
+test: data/coco_5k_500_subset.txt  # 20k images for submission to https://competitions.codalab.org/competitions/20794
 
 # number of classes
 nc: 80

data/speedco.yaml

@@ -0,0 +1,10 @@
+# train and val datasets (image directory or *.txt file with image paths)
+train: data/speedco_dataset_train_images.txt
+val: data/speedco_dataset_val_images.txt
+test: data/speedco_dataset_val_images.txt
+
+# number of classes
+nc: 1
+
+# class names
+names: ['truck']

docker-compose.yml

@@ -0,0 +1,24 @@
+---
+version: '2.3'
+
+services:
+  train:
+    image: gcr.io/kinsol-generic/yolov4-csp:${TAG:-dev}
+    build:
+      context: .
+      dockerfile: ./Dockerfile
+    runtime: nvidia
+    volumes:
+      - .:/home/dev
+      - /mnt/NAS/Production/TruckBay/:/mnt/NAS/Production/TruckBay
+      - /mnt/NAS/Public/parque_research/datasets/coco_yolo/coco:/mnt/coco
+      - /home/kweston/darknet_utils:/home/kweston/darknet_utils
+      - /home/kweston/speedco/baywatchr-inference/speedco_dataset:/mnt/speedco_dataset
+      - /home/kweston/speedco/baywatchr-inference/data/lists:/mnt/speedco_datalists
+      - /data/kweston/sandbox/mlannotation/results:/results
+    environment:
+      - GOOGLE_APPLICATION_CREDENTIALS=/app/baywatchr-api-key.json
+    command: 
+      - bash
+    shm_size: 64g
+    working_dir: /home/dev

run_train.sh

@@ -0,0 +1 @@
+python -m torch.distributed.launch --nproc_per_node 3 train.py --device 0,1,2 --batch-size 21 --data speedco.yaml --weights  --cfg yolov4-csp-single-class.cfg --name yolov4-csp-speedco --sync-bn --rect --single-cls

setup.py

@@ -0,0 +1,20 @@
+import os
+from setuptools import setup, find_namespace_packages
+
+
+def readlines(fname):
+    with open(os.path.join(os.path.dirname(__file__), fname)) as f:
+        return f.readlines()
+
+
+install_requires = readlines('requirements.txt')
+
+
+setup(
+    name='yolov4-csp',
+    version='1.0.0',
+    install_requires=install_requires,
+    packages=find_namespace_packages(include=['yolo', 'yolo.*']),
+    include_package_data=True,
+    python_requires='>=3.7'
+)

test.py

@@ -2,23 +2,24 @@
 import glob
 import json
 import os
+import shutil
 from pathlib import Path
 
 import numpy as np
 import torch
 import yaml
 from tqdm import tqdm
 
-from utils.google_utils import attempt_load
-from utils.datasets import create_dataloader
-from utils.general import coco80_to_coco91_class, check_dataset, check_file, check_img_size, box_iou, \
+from yolo.utils.google_utils import attempt_load
+from yolo.utils.datasets import create_dataloader
+from yolo.utils.general import coco80_to_coco91_class, check_dataset, check_file, check_img_size, box_iou, \
     non_max_suppression, scale_coords, xyxy2xywh, xywh2xyxy, clip_coords, set_logging, increment_path
-from utils.loss import compute_loss
-from utils.metrics import ap_per_class
-from utils.plots import plot_images, output_to_target
-from utils.torch_utils import select_device, time_synchronized
+from yolo.utils.loss import compute_loss
+from yolo.utils.metrics import ap_per_class
+from yolo.utils.plots import plot_images, output_to_target
+from yolo.utils.torch_utils import select_device, time_synchronized
 
-from models.models import *
+from yolo.models.models import *
 
 def load_classes(path):
     # Loads *.names file at 'path'
@@ -27,6 +28,7 @@ def load_classes(path):
     return list(filter(None, names))  # filter removes empty strings (such as last line)
 
 
+
 def test(data,
          weights=None,
          batch_size=16,
@@ -310,6 +312,7 @@ def test(data,
     parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment')
     parser.add_argument('--cfg', type=str, default='models/yolov4-csp.cfg', help='*.cfg path')
     parser.add_argument('--names', type=str, default='data/coco.names', help='*.cfg path')
+    parser.add_argument('--save-errors', help='Save the error cases to file', action='store_true')
     opt = parser.parse_args()
     opt.save_json |= opt.data.endswith('coco.yaml')
     opt.data = check_file(opt.data)  # check file

train.py

@@ -22,16 +22,16 @@
 
 import test  # import test.py to get mAP after each epoch
 #from models.yolo import Model
-from models.models import *
-from utils.autoanchor import check_anchors
-from utils.datasets import create_dataloader
-from utils.general import labels_to_class_weights, increment_path, labels_to_image_weights, init_seeds, \
+from yolo.models.models import *
+from yolo.utils.autoanchor import check_anchors
+from yolo.utils.datasets import create_dataloader
+from yolo.utils.general import labels_to_class_weights, increment_path, labels_to_image_weights, init_seeds, \
     fitness, fitness_p, fitness_r, fitness_ap50, fitness_ap, fitness_f, strip_optimizer, get_latest_run,\
     check_dataset, check_file, check_git_status, check_img_size, print_mutation, set_logging
-from utils.google_utils import attempt_download
-from utils.loss import compute_loss
-from utils.plots import plot_images, plot_labels, plot_results, plot_evolution
-from utils.torch_utils import ModelEMA, select_device, intersect_dicts, torch_distributed_zero_first
+from yolo.utils.google_utils import attempt_download
+from yolo.utils.loss import compute_loss
+from yolo.utils.plots import plot_images, plot_labels, plot_results, plot_evolution
+from yolo.utils.torch_utils import ModelEMA, select_device, intersect_dicts, torch_distributed_zero_first
 
 logger = logging.getLogger(__name__)
 
@@ -69,7 +69,7 @@ def train(hyp, opt, device, tb_writer=None, wandb=None):
         check_dataset(data_dict)  # check
     train_path = data_dict['train']
     test_path = data_dict['val']
-    nc, names = (1, ['item']) if opt.single_cls else (int(data_dict['nc']), data_dict['names'])  # number classes, names
+    nc, names = (1, data_dict['names']) if opt.single_cls else (int(data_dict['nc']), data_dict['names'])  # number classes, names
     assert len(names) == nc, '%g names found for nc=%g dataset in %s' % (len(names), nc, opt.data)  # check
 
     # Model

models/export.py → yolo/models/export.py

@@ -2,7 +2,7 @@
 
 import torch
 
-from utils.google_utils import attempt_download
+from yolo.utils.google_utils import attempt_download
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()

models/models.py → yolo/models/models.py

@@ -1,7 +1,7 @@
-from utils.google_utils import *
-from utils.layers import *
-from utils.parse_config import *
-from utils import torch_utils
+from yolo.utils.google_utils import *
+from yolo.utils.layers import *
+from yolo.utils.parse_config import *
+from yolo.utils import torch_utils
 
 ONNX_EXPORT = False