Merge pull request #383 from voxel51/use-video-reader

Passing VideoReader to video models

eta/core/learning.py

@@ -293,11 +293,12 @@ class VideoModel(Model):
     does not fit any of the concrete classifier/detector interfaces.
     '''
 
-    def process(self, video_path):
+    def process(self, video_reader):
         '''Generates labels for the given video.
 
         Args:
-            video_path: the path to the video
+            video_reader: an `eta.core.video.VideoReader` that can be used to
+                read the video
 
         Returns:
             an `eta.core.video.VideoLabels` instance containing the labels
@@ -472,11 +473,12 @@ class VideoClassifier(Classifier):
     that featurizes the frames of the input video.
     '''
 
-    def predict(self, video_path):
+    def predict(self, video_reader):
         '''Peforms prediction on the given video.
 
         Args:
-            video_path: the path to the video
+            video_reader: an `eta.core.video.VideoReader` that can be used to
+                read the video
 
         Returns:
             an `eta.core.data.AttributeContainer` instance containing the
@@ -651,11 +653,12 @@ class VideoObjectDetector(Detector):
     that featurizes the frames of the input video.
     '''
 
-    def detect(self, video_path):
+    def detect(self, video_reader):
         '''Peforms detection on the given video.
 
         Args:
-            video_path: the path to the video
+            video_reader: an `eta.core.video.VideoReader` that can be used to
+                read the video
 
         Returns:
             an `eta.core.objects.DetectedObjectContainer` instance describing

eta/modules/apply_video_classifier.json

@@ -11,7 +11,13 @@
             "name": "video_path",
             "type": "eta.core.types.Video",
             "description": "the input video",
-            "required": true
+            "required": false
+        },
+        {
+            "name": "video_frames_dir",
+            "type": "eta.core.types.ImageSequenceDirectory",
+            "description": "a directory containing the frames of the video",
+            "required": false
         },
         {
             "name": "input_labels_path",
@@ -29,18 +35,18 @@
         }
     ],
     "parameters": [
+        {
+            "name": "classifier",
+            "type": "eta.core.types.VideoClassifier",
+            "description": "an eta.core.learning.VideoClassifierConfig describing the eta.core.learning.VideoClassifier to use",
+            "required": true
+        },
         {
             "name": "confidence_threshold",
             "type": "eta.core.types.Number",
             "description": "a confidence threshold to use when assigning labels",
             "required": false,
             "default": null
-        },
-        {
-            "name": "classifier",
-            "type": "eta.core.types.VideoClassifier",
-            "description": "an eta.core.learning.VideoClassifierConfig describing the eta.core.learning.VideoClassifier to use",
-            "required": true
         }
     ]
 }

eta/modules/apply_video_classifier.py

@@ -26,7 +26,7 @@
 import logging
 import sys
 
-from eta.core.config import Config
+from eta.core.config import Config, ConfigError
 import eta.core.learning as etal
 import eta.core.module as etam
 import eta.core.video as etav
@@ -53,7 +53,9 @@ class DataConfig(Config):
     '''Data configuration settings.
 
     Inputs:
-        video_path (eta.core.types.Video): the input video
+        video_path (eta.core.types.Video): [None] the input video
+        video_frames_dir (eta.core.types.ImageSequenceDirectory): [None] a
+            directory containing the frames of the video
         input_labels_path (eta.core.types.VideoLabels): [None] an optional
             input VideoLabels file to which to add the predictions
 
@@ -63,7 +65,9 @@ class DataConfig(Config):
     '''
 
     def __init__(self, d):
-        self.video_path = self.parse_string(d, "video_path")
+        self.video_path = self.parse_string(d, "video_path", default=None)
+        self.video_frames_dir = self.parse_string(
+            d, "video_frames_dir", default=None)
         self.input_labels_path = self.parse_string(
             d, "input_labels_path", default=None)
         self.output_labels_path = self.parse_string(d, "output_labels_path")
@@ -122,11 +126,22 @@ def _process_video(data, classifier, parameters):
     else:
         labels = etav.VideoLabels()
 
-    # Build filter
     attr_filter = _build_attribute_filter(parameters.confidence_threshold)
 
-    logger.info("Classifying video '%s'", data.video_path)
-    attrs = classifier.predict(data.video_path)
+    # Construct VideoReader
+    if data.video_path:
+        logger.info("Classifying video '%s'", data.video_path)
+        video_reader = etav.FFmpegVideoReader(data.video_path)
+    elif data.video_frames_dir:
+        logger.info("Classifying video frames in '%s'", data.video_frames_dir)
+        video_reader = etav.SampledFramesVideoReader(data.video_frames_dir)
+    else:
+        raise ConfigError(
+            "Either `video_path` or `video_frames_dir` must be provided")
+
+    with video_reader:
+        attrs = classifier.predict(video_reader)
+
     labels.add_video_attributes(attr_filter(attrs))
 
     logger.info("Writing labels to '%s'", data.output_labels_path)

eta/modules/apply_video_model.json

@@ -11,7 +11,13 @@
             "name": "video_path",
             "type": "eta.core.types.Video",
             "description": "the input video",
-            "required": true
+            "required": false
+        },
+        {
+            "name": "video_frames_dir",
+            "type": "eta.core.types.ImageSequenceDirectory",
+            "description": "a directory containing the frames of the video",
+            "required": false
         },
         {
             "name": "input_labels_path",

eta/modules/apply_video_model.py

@@ -26,7 +26,7 @@
 import logging
 import sys
 
-from eta.core.config import Config
+from eta.core.config import Config, ConfigError
 import eta.core.learning as etal
 import eta.core.module as etam
 import eta.core.video as etav
@@ -53,7 +53,9 @@ class DataConfig(Config):
     '''Data configuration settings.
 
     Inputs:
-        video_path (eta.core.types.Video): the input video
+        video_path (eta.core.types.Video): [None] the input video
+        video_frames_dir (eta.core.types.ImageSequenceDirectory): [None] a
+            directory containing the frames of the video
         input_labels_path (eta.core.types.VideoLabels): [None] an optional
             input VideoLabels file to which to add the predictions
 
@@ -63,7 +65,9 @@ class DataConfig(Config):
     '''
 
     def __init__(self, d):
-        self.video_path = self.parse_string(d, "video_path")
+        self.video_path = self.parse_string(d, "video_path", default=None)
+        self.video_frames_dir = self.parse_string(
+            d, "video_frames_dir", default=None)
         self.input_labels_path = self.parse_string(
             d, "input_labels_path", default=None)
         self.output_labels_path = self.parse_string(d, "output_labels_path")
@@ -102,8 +106,20 @@ def _process_video(data, model):
     else:
         labels = etav.VideoLabels()
 
-    logger.info("Applying model to video '%s'", data.video_path)
-    new_labels = model.process(data.video_path)
+    # Construct VideoReader
+    if data.video_path:
+        logger.info("Applying model to video '%s'", data.video_path)
+        video_reader = etav.FFmpegVideoReader(data.video_path)
+    elif data.video_frames_dir:
+        logger.info("Applying model to frames in '%s'", data.video_frames_dir)
+        video_reader = etav.SampledFramesVideoReader(data.video_frames_dir)
+    else:
+        raise ConfigError(
+            "Either `video_path` or `video_frames_dir` must be provided")
+
+    with video_reader:
+        new_labels = model.process(video_reader)
+
     labels.merge_video_labels(new_labels)
 
     logger.info("Writing labels to '%s'", data.output_labels_path)