diff --git a/evadb/constants.py b/evadb/constants.py index 80777c5ab4..06e3973ea3 100644 --- a/evadb/constants.py +++ b/evadb/constants.py @@ -21,3 +21,5 @@ IFRAMES = "IFRAMES" AUDIORATE = "AUDIORATE" DEFAULT_FUNCTION_EXPRESSION_COST = 100 +DBFUNCTIONS = __file__[0: -13] + "/functions/" +ENVFUNCTIONS = "./Lib/site-packages/evadb/functions/" \ No newline at end of file diff --git a/evadb/functions/abstract/pytorch_abstract_function.py b/evadb/functions/abstract/pytorch_abstract_function.py index 49e531655e..763f3658f7 100644 --- a/evadb/functions/abstract/pytorch_abstract_function.py +++ b/evadb/functions/abstract/pytorch_abstract_function.py @@ -17,6 +17,7 @@ import pandas as pd from numpy.typing import ArrayLike +from evadb.configuration.configuration_manager import ConfigurationManager from evadb.functions.abstract.abstract_function import ( AbstractClassifierFunction, AbstractTransformationFunction, @@ -73,8 +74,7 @@ def __call__(self, *args, **kwargs) -> pd.DataFrame: if isinstance(frames, pd.DataFrame): frames = frames.transpose().values.tolist()[0] - # hardcoding it for now, need to be fixed @xzdandy - gpu_batch_size = 1 + gpu_batch_size = ConfigurationManager().get_value("executor", "gpu_batch_size") import torch tens_batch = torch.cat([self.transform(x) for x in frames]).to( diff --git a/evadb/functions/chatgpt.py b/evadb/functions/chatgpt.py index bf0d338689..61253116fe 100644 --- a/evadb/functions/chatgpt.py +++ b/evadb/functions/chatgpt.py @@ -20,6 +20,7 @@ from retry import retry from evadb.catalog.catalog_type import NdArrayType +from evadb.configuration.configuration_manager import ConfigurationManager from evadb.functions.abstract.abstract_function import AbstractFunction from evadb.functions.decorators.decorators import forward, setup from evadb.functions.decorators.io_descriptors.data_types import PandasDataframe @@ -84,12 +85,10 @@ def setup( self, model="gpt-3.5-turbo", temperature: float = 0, - openai_api_key="", ) -> None: assert model in _VALID_CHAT_COMPLETION_MODEL, f"Unsupported ChatGPT {model}" self.model = model self.temperature = temperature - self.openai_api_key = openai_api_key @forward( input_signatures=[ @@ -115,20 +114,20 @@ def setup( ) def forward(self, text_df): try_to_import_openai() - from openai import OpenAI - - api_key = self.openai_api_key - if len(self.openai_api_key) == 0: - api_key = os.environ.get("OPENAI_API_KEY", "") - assert ( - len(api_key) != 0 - ), "Please set your OpenAI API key using SET OPENAI_API_KEY = 'sk-' or environment variable (OPENAI_API_KEY)" - - client = OpenAI(api_key=api_key) + import openai @retry(tries=6, delay=20) def completion_with_backoff(**kwargs): - return client.chat.completions.create(**kwargs) + return openai.ChatCompletion.create(**kwargs) + + # Register API key, try configuration manager first + openai.api_key = ConfigurationManager().get_value("third_party", "OPENAI_KEY") + # If not found, try OS Environment Variable + if len(openai.api_key) == 0: + openai.api_key = os.environ.get("OPENAI_KEY", "") + assert ( + len(openai.api_key) != 0 + ), "Please set your OpenAI API key in evadb.yml file (third_party, open_api_key) or environment variable (OPENAI_KEY)" queries = text_df[text_df.columns[0]] content = text_df[text_df.columns[0]] diff --git a/evadb/functions/dalle.py b/evadb/functions/dalle.py index 03c2e77f88..d373fda383 100644 --- a/evadb/functions/dalle.py +++ b/evadb/functions/dalle.py @@ -22,6 +22,7 @@ from PIL import Image from evadb.catalog.catalog_type import NdArrayType +from evadb.configuration.configuration_manager import ConfigurationManager from evadb.functions.abstract.abstract_function import AbstractFunction from evadb.functions.decorators.decorators import forward from evadb.functions.decorators.io_descriptors.data_types import PandasDataframe @@ -33,8 +34,8 @@ class DallEFunction(AbstractFunction): def name(self) -> str: return "DallE" - def setup(self, openai_api_key="") -> None: - self.openai_api_key = openai_api_key + def setup(self) -> None: + pass @forward( input_signatures=[ @@ -56,25 +57,25 @@ def setup(self, openai_api_key="") -> None: ) def forward(self, text_df): try_to_import_openai() - from openai import OpenAI + import openai - api_key = self.openai_api_key - if len(self.openai_api_key) == 0: - api_key = os.environ.get("OPENAI_API_KEY", "") + # Register API key, try configuration manager first + openai.api_key = ConfigurationManager().get_value("third_party", "OPENAI_KEY") + # If not found, try OS Environment Variable + if openai.api_key is None or len(openai.api_key) == 0: + openai.api_key = os.environ.get("OPENAI_KEY", "") assert ( - len(api_key) != 0 - ), "Please set your OpenAI API key using SET OPENAI_API_KEY = 'sk-' or environment variable (OPENAI_API_KEY)" - - client = OpenAI(api_key=api_key) + len(openai.api_key) != 0 + ), "Please set your OpenAI API key in evadb.yml file (third_party, open_api_key) or environment variable (OPENAI_KEY)" def generate_image(text_df: PandasDataframe): results = [] queries = text_df[text_df.columns[0]] for query in queries: - response = client.images.generate(prompt=query, n=1, size="1024x1024") + response = openai.Image.create(prompt=query, n=1, size="1024x1024") # Download the image from the link - image_response = requests.get(response.data[0].url) + image_response = requests.get(response["data"][0]["url"]) image = Image.open(BytesIO(image_response.content)) # Convert the image to an array format suitable for the DataFrame diff --git a/evadb/functions/forecast.py b/evadb/functions/forecast.py index 6041e6b499..1571f6c4fc 100644 --- a/evadb/functions/forecast.py +++ b/evadb/functions/forecast.py @@ -14,7 +14,6 @@ # limitations under the License. -import os import pickle import pandas as pd @@ -38,86 +37,29 @@ def setup( id_column_rename: str, horizon: int, library: str, - conf: int, ): - self.library = library - if "neuralforecast" in self.library: - from neuralforecast import NeuralForecast - - loaded_model = NeuralForecast.load(path=model_path) - self.model_name = model_name[4:] if "Auto" in model_name else model_name - else: - with open(model_path, "rb") as f: - loaded_model = pickle.load(f) - self.model_name = model_name + f = open(model_path, "rb") + loaded_model = pickle.load(f) + f.close() self.model = loaded_model + self.model_name = model_name self.predict_column_rename = predict_column_rename self.time_column_rename = time_column_rename self.id_column_rename = id_column_rename self.horizon = int(horizon) self.library = library - self.suggestion_dict = { - 1: "Predictions are flat. Consider using LIBRARY 'neuralforecast' for more accrate predictions.", - } - self.conf = conf - self.hypers = None - self.rmse = None - if os.path.isfile(model_path + "_rmse"): - with open(model_path + "_rmse", "r") as f: - self.rmse = float(f.readline()) - if "arima" in model_name.lower(): - self.hypers = "p,d,q: " + f.readline() def forward(self, data) -> pd.DataFrame: - log_str = "" if self.library == "statsforecast": - forecast_df = self.model.predict( - h=self.horizon, level=[self.conf] - ).reset_index() + forecast_df = self.model.predict(h=self.horizon) else: - forecast_df = self.model.predict().reset_index() - - # Feedback - if len(data) == 0 or list(list(data.iloc[0]))[0] is True: - # Suggestions - suggestion_list = [] - # 1: Flat predictions - if self.library == "statsforecast": - for type_here in forecast_df["unique_id"].unique(): - if ( - forecast_df.loc[forecast_df["unique_id"] == type_here][ - self.model_name - ].nunique() - == 1 - ): - suggestion_list.append(1) - - for suggestion in set(suggestion_list): - log_str += "\nSUGGESTION: " + self.suggestion_dict[suggestion] - - # Metrics - if self.rmse is not None: - log_str += "\nMean normalized RMSE: " + str(self.rmse) - if self.hypers is not None: - log_str += "\nHyperparameters: " + self.hypers - - print(log_str) - + forecast_df = self.model.predict() + forecast_df.reset_index(inplace=True) forecast_df = forecast_df.rename( columns={ "unique_id": self.id_column_rename, "ds": self.time_column_rename, - self.model_name - if self.library == "statsforecast" - else self.model_name + "-median": self.predict_column_rename, - self.model_name - + "-lo-" - + str(self.conf): self.predict_column_rename - + "-lo", - self.model_name - + "-hi-" - + str(self.conf): self.predict_column_rename - + "-hi", + self.model_name: self.predict_column_rename, } )[: self.horizon * forecast_df["unique_id"].nunique()] return forecast_df diff --git a/evadb/functions/function_bootstrap_queries.py b/evadb/functions/function_bootstrap_queries.py index f8186d4dd3..e20c72b1c5 100644 --- a/evadb/functions/function_bootstrap_queries.py +++ b/evadb/functions/function_bootstrap_queries.py @@ -17,6 +17,9 @@ from evadb.database import EvaDBDatabase from evadb.server.command_handler import execute_query_fetch_all +import sys +import subprocess + NDARRAY_DIR = "ndarray" TUTORIALS_DIR = "tutorials" @@ -214,30 +217,6 @@ EvaDB_INSTALLATION_DIR ) -Upper_function_query = """CREATE FUNCTION IF NOT EXISTS UPPER - INPUT (input ANYTYPE) - OUTPUT (output NDARRAY STR(ANYDIM)) - IMPL '{}/functions/helpers/upper.py'; - """.format( - EvaDB_INSTALLATION_DIR -) - -Lower_function_query = """CREATE FUNCTION IF NOT EXISTS LOWER - INPUT (input ANYTYPE) - OUTPUT (output NDARRAY STR(ANYDIM)) - IMPL '{}/functions/helpers/lower.py'; - """.format( - EvaDB_INSTALLATION_DIR -) - -Concat_function_query = """CREATE FUNCTION IF NOT EXISTS CONCAT - INPUT (input ANYTYPE) - OUTPUT (output NDARRAY STR(ANYDIM)) - IMPL '{}/functions/helpers/concat.py'; - """.format( - EvaDB_INSTALLATION_DIR -) - def init_builtin_functions(db: EvaDBDatabase, mode: str = "debug") -> None: """Load the built-in functions into the system during system bootstrapping. @@ -285,9 +264,6 @@ def init_builtin_functions(db: EvaDBDatabase, mode: str = "debug") -> None: Yolo_function_query, stablediffusion_function_query, dalle_function_query, - Upper_function_query, - Lower_function_query, - Concat_function_query, ] # if mode is 'debug', add debug functions @@ -306,6 +282,11 @@ def init_builtin_functions(db: EvaDBDatabase, mode: str = "debug") -> None: # ignore exceptions during the bootstrapping phase due to missing packages for query in queries: try: + #Uncomment to force pip installs onto local device + #subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'norfair']) + #subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'ultralytics']) + #subprocess.check_call([sys.executable, '-m', 'pip', 'install', 'facenet-pytorch']) + #cursor.query("DROP FUNCTION IF EXISTS NorFairTracker;").df() execute_query_fetch_all( db, query, do_not_print_exceptions=False, do_not_raise_exceptions=True ) diff --git a/evadb/functions/norfair/__init__.py b/evadb/functions/norfair/__init__.py new file mode 100644 index 0000000000..9e3db231ad --- /dev/null +++ b/evadb/functions/norfair/__init__.py @@ -0,0 +1,37 @@ +""" +A customizable lightweight Python library for real-time multi-object tracking. + +Examples +-------- +>>> from norfair import Detection, Tracker, Video, draw_tracked_objects +>>> detector = MyDetector() # Set up a detector +>>> video = Video(input_path="video.mp4") +>>> tracker = Tracker(distance_function="euclidean", distance_threshold=50) +>>> for frame in video: +>>> detections = detector(frame) +>>> norfair_detections = [Detection(points) for points in detections] +>>> tracked_objects = tracker.update(detections=norfair_detections) +>>> draw_tracked_objects(frame, tracked_objects) +>>> video.write(frame) +""" +import sys + +from .distances import * +from .drawing import * +from .filter import ( + FilterPyKalmanFilterFactory, + NoFilterFactory, + OptimizedKalmanFilterFactory, +) +from .tracker import Detection, Tracker +from .utils import get_cutout, print_objects_as_table +from .video import Video + +if sys.version_info >= (3, 8): + import importlib.metadata + + __version__ = importlib.metadata.version(__name__) +elif sys.version_info < (3, 8): + import importlib_metadata + + __version__ = importlib_metadata.version(__name__) diff --git a/evadb/functions/norfair/camera_motion.py b/evadb/functions/norfair/camera_motion.py new file mode 100644 index 0000000000..6e89bd6fd7 --- /dev/null +++ b/evadb/functions/norfair/camera_motion.py @@ -0,0 +1,406 @@ +"Camera motion stimation module." +from abc import ABC, abstractmethod +from typing import Optional, Tuple + +import numpy as np + +try: + import cv2 +except ImportError: + from .utils import DummyOpenCVImport + + cv2 = DummyOpenCVImport() + + +# +# Abstract interfaces +# +class CoordinatesTransformation(ABC): + """ + Abstract class representing a coordinate transformation. + + Detections' and tracked objects' coordinates can be interpreted in 2 reference: + + - _Relative_: their position on the current frame, (0, 0) is top left + - _Absolute_: their position on an fixed space, (0, 0) + is the top left of the first frame of the video. + + Therefore, coordinate transformation in this context is a class that can transform + coordinates in one reference to another. + """ + + @abstractmethod + def abs_to_rel(self, points: np.ndarray) -> np.ndarray: + pass + + @abstractmethod + def rel_to_abs(self, points: np.ndarray) -> np.ndarray: + pass + + +class TransformationGetter(ABC): + """ + Abstract class representing a method for finding CoordinatesTransformation between 2 sets of points + """ + + @abstractmethod + def __call__( + self, curr_pts: np.ndarray, prev_pts: np.ndarray + ) -> Tuple[bool, CoordinatesTransformation]: + pass + + +# +# Translation +# +class TranslationTransformation(CoordinatesTransformation): + """ + Coordinate transformation between points using a simple translation + + Parameters + ---------- + movement_vector : np.ndarray + The vector representing the translation. + """ + + def __init__(self, movement_vector): + self.movement_vector = movement_vector + + def abs_to_rel(self, points: np.ndarray): + return points + self.movement_vector + + def rel_to_abs(self, points: np.ndarray): + return points - self.movement_vector + + +class TranslationTransformationGetter(TransformationGetter): + """ + Calculates TranslationTransformation between points. + + The camera movement is calculated as the mode of optical flow between the previous reference frame + and the current. + + Comparing consecutive frames can make differences too small to correctly estimate the translation, + for this reason the reference frame is kept fixed as we progress through the video. + Eventually, if the transformation is no longer able to match enough points, the reference frame is updated. + + Parameters + ---------- + bin_size : float + Before calculatin the mode, optiocal flow is bucketized into bins of this size. + proportion_points_used_threshold: float + Proportion of points that must be matched, otherwise the reference frame must be updated. + """ + + def __init__( + self, bin_size: float = 0.2, proportion_points_used_threshold: float = 0.9 + ) -> None: + self.bin_size = bin_size + self.proportion_points_used_threshold = proportion_points_used_threshold + self.data = None + + def __call__( + self, curr_pts: np.ndarray, prev_pts: np.ndarray + ) -> Tuple[bool, TranslationTransformation]: + # get flow + flow = curr_pts - prev_pts + + # get mode + flow = np.around(flow / self.bin_size) * self.bin_size + unique_flows, counts = np.unique(flow, axis=0, return_counts=True) + + max_index = counts.argmax() + + proportion_points_used = counts[max_index] / len(prev_pts) + update_prvs = proportion_points_used < self.proportion_points_used_threshold + + flow_mode = unique_flows[max_index] + + try: + flow_mode += self.data + except TypeError: + pass + + if update_prvs: + self.data = flow_mode + + return update_prvs, TranslationTransformation(flow_mode) + + +# +# Homography +# +class HomographyTransformation(CoordinatesTransformation): + """ + Coordinate transformation beweent points using an homography + + Parameters + ---------- + homography_matrix : np.ndarray + The matrix representing the homography + """ + + def __init__(self, homography_matrix: np.ndarray): + self.homography_matrix = homography_matrix + self.inverse_homography_matrix = np.linalg.inv(homography_matrix) + + def abs_to_rel(self, points: np.ndarray): + ones = np.ones((len(points), 1)) + points_with_ones = np.hstack((points, ones)) + points_transformed = points_with_ones @ self.homography_matrix.T + points_transformed = points_transformed / points_transformed[:, -1].reshape( + -1, 1 + ) + return points_transformed[:, :2] + + def rel_to_abs(self, points: np.ndarray): + ones = np.ones((len(points), 1)) + points_with_ones = np.hstack((points, ones)) + points_transformed = points_with_ones @ self.inverse_homography_matrix.T + points_transformed = points_transformed / points_transformed[:, -1].reshape( + -1, 1 + ) + return points_transformed[:, :2] + + +class HomographyTransformationGetter(TransformationGetter): + """ + Calculates HomographyTransformation between points. + + The camera movement is represented as an homography that matches the optical flow between the previous reference frame + and the current. + + Comparing consecutive frames can make differences too small to correctly estimate the homography, often resulting in the identity. + For this reason the reference frame is kept fixed as we progress through the video. + Eventually, if the transformation is no longer able to match enough points, the reference frame is updated. + + Parameters + ---------- + method : Optional[int], optional + One of openCV's method for finding homographies. + Valid options are: `[0, cv.RANSAC, cv.LMEDS, cv.RHO]`, by default `cv.RANSAC` + ransac_reproj_threshold : int, optional + Maximum allowed reprojection error to treat a point pair as an inlier. More info in links below. + max_iters : int, optional + The maximum number of RANSAC iterations. More info in links below. + confidence : float, optional + Confidence level, must be between 0 and 1. More info in links below. + proportion_points_used_threshold : float, optional + Proportion of points that must be matched, otherwise the reference frame must be updated. + + See Also + -------- + [opencv.findHomography](https://docs.opencv.org/3.4/d9/d0c/group__calib3d.html#ga4abc2ece9fab9398f2e560d53c8c9780) + """ + + def __init__( + self, + method: Optional[int] = None, + ransac_reproj_threshold: int = 3, + max_iters: int = 2000, + confidence: float = 0.995, + proportion_points_used_threshold: float = 0.9, + ) -> None: + self.data = None + if method is None: + method = cv2.RANSAC + self.method = method + self.ransac_reproj_threshold = ransac_reproj_threshold + self.max_iters = max_iters + self.confidence = confidence + self.proportion_points_used_threshold = proportion_points_used_threshold + + def __call__( + self, curr_pts: np.ndarray, prev_pts: np.ndarray + ) -> Tuple[bool, HomographyTransformation]: + homography_matrix, points_used = cv2.findHomography( + prev_pts, + curr_pts, + method=self.method, + ransacReprojThreshold=self.ransac_reproj_threshold, + maxIters=self.max_iters, + confidence=self.confidence, + ) + + proportion_points_used = np.sum(points_used) / len(points_used) + + update_prvs = proportion_points_used < self.proportion_points_used_threshold + + try: + homography_matrix = homography_matrix @ self.data + except (TypeError, ValueError): + pass + + if update_prvs: + self.data = homography_matrix + + return update_prvs, HomographyTransformation(homography_matrix) + + +# +# Motion estimation +# +def _get_sparse_flow( + gray_next, + gray_prvs, + prev_pts=None, + max_points=300, + min_distance=15, + block_size=3, + mask=None, + quality_level=0.01, +): + if prev_pts is None: + # get points + prev_pts = cv2.goodFeaturesToTrack( + gray_prvs, + maxCorners=max_points, + qualityLevel=quality_level, + minDistance=min_distance, + blockSize=block_size, + mask=mask, + ) + + # compute optical flow + curr_pts, status, err = cv2.calcOpticalFlowPyrLK( + gray_prvs, gray_next, prev_pts, None + ) + # filter valid points + idx = np.where(status == 1)[0] + prev_pts = prev_pts[idx].reshape((-1, 2)) + curr_pts = curr_pts[idx].reshape((-1, 2)) + return curr_pts, prev_pts + + +class MotionEstimator: + """ + Estimator of the motion of the camera. + + Uses optical flow to estimate the motion of the camera from frame to frame. + The optical flow is calculated on a sample of strong points (corners). + + Parameters + ---------- + max_points : int, optional + Maximum amount of points sampled. + More points make the estimation process slower but more precise + min_distance : int, optional + Minimum distance between the sample points. + block_size : int, optional + Size of an average block when finding the corners. More info in links below. + transformations_getter : TransformationGetter, optional + An instance of TransformationGetter. By default [`HomographyTransformationGetter`][norfair.camera_motion.HomographyTransformationGetter] + draw_flow : bool, optional + Draws the optical flow on the frame for debugging. + flow_color : Optional[Tuple[int, int, int]], optional + Color of the drawing, by default blue. + quality_level : float, optional + Parameter characterizing the minimal accepted quality of image corners. + + Examples + -------- + >>> from norfair import Tracker, Video + >>> from norfair.camera_motion MotionEstimator + >>> video = Video("video.mp4") + >>> tracker = Tracker(...) + >>> motion_estimator = MotionEstimator() + >>> for frame in video: + >>> detections = get_detections(frame) # runs detector and returns Detections + >>> coord_transformation = motion_estimator.update(frame) + >>> tracked_objects = tracker.update(detections, coord_transformations=coord_transformation) + + See Also + -------- + For more infor on how the points are sampled: [OpenCV.goodFeaturesToTrack](https://docs.opencv.org/3.4/dd/d1a/group__imgproc__feature.html#ga1d6bb77486c8f92d79c8793ad995d541) + """ + + def __init__( + self, + max_points: int = 200, + min_distance: int = 15, + block_size: int = 3, + transformations_getter: TransformationGetter = None, + draw_flow: bool = False, + flow_color: Optional[Tuple[int, int, int]] = None, + quality_level: float = 0.01, + ): + + self.max_points = max_points + self.min_distance = min_distance + self.block_size = block_size + + self.draw_flow = draw_flow + if self.draw_flow and flow_color is None: + flow_color = [0, 0, 100] + self.flow_color = flow_color + + self.gray_prvs = None + self.prev_pts = None + if transformations_getter is None: + transformations_getter = HomographyTransformationGetter() + + self.transformations_getter = transformations_getter + self.prev_mask = None + self.gray_next = None + self.quality_level = quality_level + + def update( + self, frame: np.ndarray, mask: np.ndarray = None + ) -> CoordinatesTransformation: + """ + Estimate camera motion for each frame + + Parameters + ---------- + frame : np.ndarray + The frame. + mask : np.ndarray, optional + An optional mask to avoid areas of the frame when sampling the corner. + Must be an array of shape `(frame.shape[0], frame.shape[1])`, dtype same as frame, + and values in {0, 1}. + + In general, the estimation will work best when it samples many points from the background; + with that intention, this parameters is usefull for masking out the detections/tracked objects, + forcing the MotionEstimator ignore the moving objects. + Can be used to mask static areas of the image, such as score overlays in sport transmisions or + timestamps in security cameras. + + Returns + ------- + CoordinatesTransformation + The CoordinatesTransformation that can transform coordinates on this frame to absolute coordinates + or vice versa. + """ + self.gray_next = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) + if self.gray_prvs is None: + self.gray_prvs = self.gray_next + self.prev_mask = mask + + curr_pts, self.prev_pts = _get_sparse_flow( + self.gray_next, + self.gray_prvs, + self.prev_pts, + self.max_points, + self.min_distance, + self.block_size, + self.prev_mask, + quality_level=self.quality_level, + ) + if self.draw_flow: + for (curr, prev) in zip(curr_pts, self.prev_pts): + c = tuple(curr.astype(int).ravel()) + p = tuple(prev.astype(int).ravel()) + cv2.line(frame, c, p, self.flow_color, 2) + cv2.circle(frame, c, 3, self.flow_color, -1) + + update_prvs, coord_transformations = self.transformations_getter( + curr_pts, + self.prev_pts, + ) + + if update_prvs: + self.gray_prvs = self.gray_next + self.prev_pts = None + self.prev_mask = mask + + return coord_transformations diff --git a/evadb/functions/norfair/distances.py b/evadb/functions/norfair/distances.py new file mode 100644 index 0000000000..d7d7e4c947 --- /dev/null +++ b/evadb/functions/norfair/distances.py @@ -0,0 +1,550 @@ +"""Predefined distances""" +from abc import ABC, abstractmethod +from functools import partial +from logging import warning +from typing import TYPE_CHECKING, Callable, List, Optional, Sequence, Union + +import numpy as np +from scipy.spatial.distance import cdist + +if TYPE_CHECKING: + from .tracker import Detection, TrackedObject + + +class Distance(ABC): + """ + Abstract class representing a distance. + + Subclasses must implement the method `get_distances` + """ + + @abstractmethod + def get_distances( + self, + objects: Sequence["TrackedObject"], + candidates: Optional[Union[List["Detection"], List["TrackedObject"]]], + ) -> np.ndarray: + """ + Method that calculates the distances between new candidates and objects. + + Parameters + ---------- + objects : Sequence[TrackedObject] + Sequence of [TrackedObject][norfair.tracker.TrackedObject] to be compared with potential [Detection][norfair.tracker.Detection] or [TrackedObject][norfair.tracker.TrackedObject] + candidates. + candidates : Union[List[Detection], List[TrackedObject]], optional + List of candidates ([Detection][norfair.tracker.Detection] or [TrackedObject][norfair.tracker.TrackedObject]) to be compared to [TrackedObject][norfair.tracker.TrackedObject]. + + Returns + ------- + np.ndarray + A matrix containing the distances between objects and candidates. + """ + + +class ScalarDistance(Distance): + """ + ScalarDistance class represents a distance that is calculated pointwise. + + Parameters + ---------- + distance_function : Union[Callable[["Detection", "TrackedObject"], float], Callable[["TrackedObject", "TrackedObject"], float]] + Distance function used to determine the pointwise distance between new candidates and objects. + This function should take 2 input arguments, the first being a `Union[Detection, TrackedObject]`, + and the second [TrackedObject][norfair.tracker.TrackedObject]. It has to return a `float` with the distance it calculates. + """ + + def __init__( + self, + distance_function: Union[ + Callable[["Detection", "TrackedObject"], float], + Callable[["TrackedObject", "TrackedObject"], float], + ], + ): + self.distance_function = distance_function + + def get_distances( + self, + objects: Sequence["TrackedObject"], + candidates: Optional[Union[List["Detection"], List["TrackedObject"]]], + ) -> np.ndarray: + """ + Method that calculates the distances between new candidates and objects. + + Parameters + ---------- + objects : Sequence[TrackedObject] + Sequence of [TrackedObject][norfair.tracker.TrackedObject] to be compared with potential [Detection][norfair.tracker.Detection] or [TrackedObject][norfair.tracker.TrackedObject] + candidates. + candidates : Union[List[Detection], List[TrackedObject]], optional + List of candidates ([Detection][norfair.tracker.Detection] or [TrackedObject][norfair.tracker.TrackedObject]) to be compared to [TrackedObject][norfair.tracker.TrackedObject]. + + Returns + ------- + np.ndarray + A matrix containing the distances between objects and candidates. + """ + distance_matrix = np.full( + (len(candidates), len(objects)), + fill_value=np.inf, + dtype=np.float32, + ) + if not objects or not candidates: + return distance_matrix + for c, candidate in enumerate(candidates): + for o, obj in enumerate(objects): + if candidate.label != obj.label: + if (candidate.label is None) or (obj.label is None): + print("\nThere are detections with and without label!") + continue + distance = self.distance_function(candidate, obj) + distance_matrix[c, o] = distance + return distance_matrix + + +class VectorizedDistance(Distance): + """ + VectorizedDistance class represents a distance that is calculated in a vectorized way. This means + that instead of going through every pair and explicitly calculating its distance, VectorizedDistance + uses the entire vectors to compare to each other in a single operation. + + Parameters + ---------- + distance_function : Callable[[np.ndarray, np.ndarray], np.ndarray] + Distance function used to determine the distances between new candidates and objects. + This function should take 2 input arguments, the first being a `np.ndarray` and the second + `np.ndarray`. It has to return a `np.ndarray` with the distance matrix it calculates. + """ + + def __init__( + self, + distance_function: Callable[[np.ndarray, np.ndarray], np.ndarray], + ): + self.distance_function = distance_function + + def get_distances( + self, + objects: Sequence["TrackedObject"], + candidates: Optional[Union[List["Detection"], List["TrackedObject"]]], + ) -> np.ndarray: + """ + Method that calculates the distances between new candidates and objects. + + Parameters + ---------- + objects : Sequence[TrackedObject] + Sequence of [TrackedObject][norfair.tracker.TrackedObject] to be compared with potential [Detection][norfair.tracker.Detection] or [TrackedObject][norfair.tracker.TrackedObject] + candidates. + candidates : Union[List[Detection], List[TrackedObject]], optional + List of candidates ([Detection][norfair.tracker.Detection] or [TrackedObject][norfair.tracker.TrackedObject]) to be compared to [TrackedObject][norfair.tracker.TrackedObject]. + + Returns + ------- + np.ndarray + A matrix containing the distances between objects and candidates. + """ + distance_matrix = np.full( + (len(candidates), len(objects)), + fill_value=np.inf, + dtype=np.float32, + ) + if not objects or not candidates: + return distance_matrix + + object_labels = np.array([o.label for o in objects]).astype(str) + candidate_labels = np.array([c.label for c in candidates]).astype(str) + + # iterate over labels that are present both in objects and detections + for label in np.intersect1d( + np.unique(object_labels), np.unique(candidate_labels) + ): + # generate masks of the subset of object and detections for this label + obj_mask = object_labels == label + cand_mask = candidate_labels == label + + stacked_objects = [] + for o in objects: + if str(o.label) == label: + stacked_objects.append(o.estimate.ravel()) + stacked_objects = np.stack(stacked_objects) + + stacked_candidates = [] + for c in candidates: + if str(c.label) == label: + if "Detection" in str(type(c)): + stacked_candidates.append(c.points.ravel()) + else: + stacked_candidates.append(c.estimate.ravel()) + stacked_candidates = np.stack(stacked_candidates) + + # calculate the pairwise distances between objects and candidates with this label + # and assign the result to the correct positions inside distance_matrix + distance_matrix[np.ix_(cand_mask, obj_mask)] = self._compute_distance( + stacked_candidates, stacked_objects + ) + + return distance_matrix + + def _compute_distance( + self, stacked_candidates: np.ndarray, stacked_objects: np.ndarray + ) -> np.ndarray: + """ + Method that computes the pairwise distances between new candidates and objects. + It is intended to use the entire vectors to compare to each other in a single operation. + + Parameters + ---------- + stacked_candidates : np.ndarray + np.ndarray containing a stack of candidates to be compared with the stacked_objects. + stacked_objects : np.ndarray + np.ndarray containing a stack of objects to be compared with the stacked_objects. + + Returns + ------- + np.ndarray + A matrix containing the distances between objects and candidates. + """ + return self.distance_function(stacked_candidates, stacked_objects) + + +class ScipyDistance(VectorizedDistance): + """ + ScipyDistance class extends VectorizedDistance for the use of Scipy's vectorized distances. + + This class uses `scipy.spatial.distance.cdist` to calculate distances between two `np.ndarray`. + + Parameters + ---------- + metric : str, optional + Defines the specific Scipy metric to use to calculate the pairwise distances between + new candidates and objects. + + Other kwargs are passed through to cdist + + See Also + -------- + [`scipy.spatial.distance.cdist`](https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.cdist.html) + """ + + def __init__(self, metric: str = "euclidean", **kwargs): + self.metric = metric + super().__init__(distance_function=partial(cdist, metric=self.metric, **kwargs)) + + +def frobenius(detection: "Detection", tracked_object: "TrackedObject") -> float: + """ + Frobernius norm on the difference of the points in detection and the estimates in tracked_object. + + The Frobenius distance and norm are given by: + + $$ + d_f(a, b) = ||a - b||_F + $$ + + $$ + ||A||_F = [\\sum_{i,j} abs(a_{i,j})^2]^{1/2} + $$ + + Parameters + ---------- + detection : Detection + A detection. + tracked_object : TrackedObject + A tracked object. + + Returns + ------- + float + The distance. + + See Also + -------- + [`np.linalg.norm`](https://numpy.org/doc/stable/reference/generated/numpy.linalg.norm.html) + """ + return np.linalg.norm(detection.points - tracked_object.estimate) + + +def mean_euclidean(detection: "Detection", tracked_object: "TrackedObject") -> float: + """ + Average euclidean distance between the points in detection and estimates in tracked_object. + + $$ + d(a, b) = \\frac{\\sum_{i=0}^N ||a_i - b_i||_2}{N} + $$ + + Parameters + ---------- + detection : Detection + A detection. + tracked_object : TrackedObject + A tracked object + + Returns + ------- + float + The distance. + + See Also + -------- + [`np.linalg.norm`](https://numpy.org/doc/stable/reference/generated/numpy.linalg.norm.html) + """ + return np.linalg.norm(detection.points - tracked_object.estimate, axis=1).mean() + + +def mean_manhattan(detection: "Detection", tracked_object: "TrackedObject") -> float: + """ + Average manhattan distance between the points in detection and the estimates in tracked_object + + Given by: + + $$ + d(a, b) = \\frac{\\sum_{i=0}^N ||a_i - b_i||_1}{N} + $$ + + Where $||a||_1$ is the manhattan norm. + + Parameters + ---------- + detection : Detection + A detection. + tracked_object : TrackedObject + a tracked object. + + Returns + ------- + float + The distance. + + See Also + -------- + [`np.linalg.norm`](https://numpy.org/doc/stable/reference/generated/numpy.linalg.norm.html) + """ + return np.linalg.norm( + detection.points - tracked_object.estimate, ord=1, axis=1 + ).mean() + + +def _boxes_area(boxes: np.ndarray) -> np.ndarray: + """ + Calculate the area of bounding boxes. + """ + return (boxes[2] - boxes[0]) * (boxes[3] - boxes[1]) + + +def _validate_bboxes(bboxes: np.ndarray): + """ + Validate that bounding boxes are well formed. + """ + assert ( + isinstance(bboxes, np.ndarray) + and len(bboxes.shape) == 2 + and bboxes.shape[1] == 4 + ), f"Bounding boxes must be defined as np.array with (N, 4) shape, {bboxes} given" + + if not (all(bboxes[:, 0] < bboxes[:, 2]) and all(bboxes[:, 1] < bboxes[:, 3])): + warning("Incorrect bounding boxes. Check that x_min < x_max and y_min < y_max.") + + +def iou(candidates: np.ndarray, objects: np.ndarray) -> np.ndarray: + """ + Calculate IoU between two sets of bounding boxes. Both sets of boxes are expected + to be in `[x_min, y_min, x_max, y_max]` format. + + Normal IoU is 1 when the boxes are the same and 0 when they don't overlap, + to transform that into a distance that makes sense we return `1 - iou`. + + Parameters + ---------- + candidates : numpy.ndarray + (N, 4) numpy.ndarray containing candidates bounding boxes. + objects : numpy.ndarray + (K, 4) numpy.ndarray containing objects bounding boxes. + + Returns + ------- + numpy.ndarray + (N, K) numpy.ndarray of `1 - iou` between candidates and objects. + """ + _validate_bboxes(candidates) + + area_candidates = _boxes_area(candidates.T) + area_objects = _boxes_area(objects.T) + + top_left = np.maximum(candidates[:, None, :2], objects[:, :2]) + bottom_right = np.minimum(candidates[:, None, 2:], objects[:, 2:]) + + area_intersection = np.prod( + np.clip(bottom_right - top_left, a_min=0, a_max=None), 2 + ) + return 1 - area_intersection / ( + area_candidates[:, None] + area_objects - area_intersection + ) + + +iou_opt = iou # deprecated + + +_SCALAR_DISTANCE_FUNCTIONS = { + "frobenius": frobenius, + "mean_manhattan": mean_manhattan, + "mean_euclidean": mean_euclidean, +} +_VECTORIZED_DISTANCE_FUNCTIONS = { + "iou": iou, + "iou_opt": iou, # deprecated +} +_SCIPY_DISTANCE_FUNCTIONS = [ + "braycurtis", + "canberra", + "chebyshev", + "cityblock", + "correlation", + "cosine", + "dice", + "euclidean", + "hamming", + "jaccard", + "jensenshannon", + "kulczynski1", + "mahalanobis", + "matching", + "minkowski", + "rogerstanimoto", + "russellrao", + "seuclidean", + "sokalmichener", + "sokalsneath", + "sqeuclidean", + "yule", +] +AVAILABLE_VECTORIZED_DISTANCES = ( + list(_VECTORIZED_DISTANCE_FUNCTIONS.keys()) + _SCIPY_DISTANCE_FUNCTIONS +) + + +def get_distance_by_name(name: str) -> Distance: + """ + Select a distance by name. + + Parameters + ---------- + name : str + A string defining the metric to get. + + Returns + ------- + Distance + The distance object. + """ + + if name in _SCALAR_DISTANCE_FUNCTIONS: + warning( + "You are using a scalar distance function. If you want to speed up the" + " tracking process please consider using a vectorized distance function" + f" such as {AVAILABLE_VECTORIZED_DISTANCES}." + ) + distance = _SCALAR_DISTANCE_FUNCTIONS[name] + distance_function = ScalarDistance(distance) + elif name in _SCIPY_DISTANCE_FUNCTIONS: + distance_function = ScipyDistance(name) + elif name in _VECTORIZED_DISTANCE_FUNCTIONS: + if name == "iou_opt": + warning("iou_opt is deprecated, use iou instead") + distance = _VECTORIZED_DISTANCE_FUNCTIONS[name] + distance_function = VectorizedDistance(distance) + else: + raise ValueError( + f"Invalid distance '{name}', expecting one of" + f" {list(_SCALAR_DISTANCE_FUNCTIONS.keys()) + AVAILABLE_VECTORIZED_DISTANCES}" + ) + + return distance_function + + +def create_keypoints_voting_distance( + keypoint_distance_threshold: float, detection_threshold: float +) -> Callable[["Detection", "TrackedObject"], float]: + """ + Construct a keypoint voting distance function configured with the thresholds. + + Count how many points in a detection match the with a tracked_object. + A match is considered when distance between the points is < `keypoint_distance_threshold` + and the score of the last_detection of the tracked_object is > `detection_threshold`. + Notice the if multiple points are tracked, the ith point in detection can only match the ith + point in the tracked object. + + Distance is 1 if no point matches and approximates 0 as more points are matched. + + Parameters + ---------- + keypoint_distance_threshold: float + Points closer than this threshold are considered a match. + detection_threshold: float + Detections and objects with score lower than this threshold are ignored. + + Returns + ------- + Callable + The distance funtion that must be passed to the Tracker. + """ + + def keypoints_voting_distance( + detection: "Detection", tracked_object: "TrackedObject" + ) -> float: + distances = np.linalg.norm(detection.points - tracked_object.estimate, axis=1) + match_num = np.count_nonzero( + (distances < keypoint_distance_threshold) + * (detection.scores > detection_threshold) + * (tracked_object.last_detection.scores > detection_threshold) + ) + return 1 / (1 + match_num) + + return keypoints_voting_distance + + +def create_normalized_mean_euclidean_distance( + height: int, width: int +) -> Callable[["Detection", "TrackedObject"], float]: + """ + Construct a normalized mean euclidean distance function configured with the max height and width. + + The result distance is bound to [0, 1] where 1 indicates oposite corners of the image. + + Parameters + ---------- + height: int + Height of the image. + width: int + Width of the image. + + Returns + ------- + Callable + The distance funtion that must be passed to the Tracker. + """ + + def normalized__mean_euclidean_distance( + detection: "Detection", tracked_object: "TrackedObject" + ) -> float: + """Normalized mean euclidean distance""" + # calculate distances and normalized it by width and height + difference = (detection.points - tracked_object.estimate).astype(float) + difference[:, 0] /= width + difference[:, 1] /= height + + # calculate eucledean distance and average + return np.linalg.norm(difference, axis=1).mean() + + return normalized__mean_euclidean_distance + + +__all__ = [ + "frobenius", + "mean_manhattan", + "mean_euclidean", + "iou", + "iou_opt", + "get_distance_by_name", + "create_keypoints_voting_distance", + "create_normalized_mean_euclidean_distance", +] diff --git a/evadb/functions/norfair/drawing/__init__.py b/evadb/functions/norfair/drawing/__init__.py new file mode 100644 index 0000000000..328ba2c29a --- /dev/null +++ b/evadb/functions/norfair/drawing/__init__.py @@ -0,0 +1,8 @@ +"Collection of drawing functions" +from .absolute_grid import draw_absolute_grid +from .color import Color, ColorType, Palette +from .draw_boxes import draw_boxes, draw_tracked_boxes +from .draw_points import draw_points, draw_tracked_objects +from .drawer import Drawable +from .fixed_camera import FixedCamera +from .path import AbsolutePaths, Paths diff --git a/evadb/functions/norfair/drawing/absolute_grid.py b/evadb/functions/norfair/drawing/absolute_grid.py new file mode 100644 index 0000000000..50e2ecc5fa --- /dev/null +++ b/evadb/functions/norfair/drawing/absolute_grid.py @@ -0,0 +1,103 @@ +from functools import lru_cache + +import numpy as np + +from norfair.camera_motion import CoordinatesTransformation + +from .color import Color, ColorType +from .drawer import Drawer + + +@lru_cache(maxsize=4) +def _get_grid(size, w, h, polar=False): + """ + Construct the grid of points. + + Points are choosen + Results are cached since the grid in absolute coordinates doesn't change. + """ + # We need to get points on a semi-sphere of radious 1 centered around (0, 0) + + # First step is to get a grid of angles, theta and phi ∈ (-pi/2, pi/2) + step = np.pi / size + start = -np.pi / 2 + step / 2 + end = np.pi / 2 + theta, fi = np.mgrid[start:end:step, start:end:step] + + if polar: + # if polar=True the first frame will show points as if + # you are on the center of the earth looking at one of the poles. + # Points on the sphere are defined as [sin(theta) * cos(fi), sin(theta) * sin(fi), cos(theta)] + # Then we need to intersect the line defined by the point above with the + # plane z=1 which is the "absolute plane", we do so by dividing by cos(theta), the result becomes + # [tan(theta) * cos(fi), tan(theta) * sin(phi), 1] + # note that the z=1 is implied by the coord_transformation so there is no need to add it. + tan_theta = np.tan(theta) + + X = tan_theta * np.cos(fi) + Y = tan_theta * np.sin(fi) + else: + # otherwhise will show as if you were looking at the equator + X = np.tan(fi) + Y = np.divide(np.tan(theta), np.cos(fi)) + # construct the points as x, y coordinates + points = np.vstack((X.flatten(), Y.flatten())).T + # scale and center the points + return points * max(h, w) + np.array([w // 2, h // 2]) + + +def draw_absolute_grid( + frame: np.ndarray, + coord_transformations: CoordinatesTransformation, + grid_size: int = 20, + radius: int = 2, + thickness: int = 1, + color: ColorType = Color.black, + polar: bool = False, +): + """ + Draw a grid of points in absolute coordinates. + + Useful for debugging camera motion. + + The points are drawn as if the camera were in the center of a sphere and points are drawn in the intersection + of latitude and longitude lines over the surface of the sphere. + + Parameters + ---------- + frame : np.ndarray + The OpenCV frame to draw on. + coord_transformations : CoordinatesTransformation + The coordinate transformation as returned by the [`MotionEstimator`][norfair.camera_motion.MotionEstimator] + grid_size : int, optional + How many points to draw. + radius : int, optional + Size of each point. + thickness : int, optional + Thickness of each point + color : ColorType, optional + Color of the points. + polar : Bool, optional + If True, the points on the first frame are drawn as if the camera were pointing to a pole (viewed from the center of the earth). + By default, False is used which means the points are drawn as if the camera were pointing to the Equator. + """ + h, w, _ = frame.shape + + # get absolute points grid + points = _get_grid(grid_size, w, h, polar=polar) + + # transform the points to relative coordinates + if coord_transformations is None: + points_transformed = points + else: + points_transformed = coord_transformations.abs_to_rel(points) + + # filter points that are not visible + visible_points = points_transformed[ + (points_transformed <= np.array([w, h])).all(axis=1) + & (points_transformed >= 0).all(axis=1) + ] + for point in visible_points: + Drawer.cross( + frame, point.astype(int), radius=radius, thickness=thickness, color=color + ) diff --git a/evadb/functions/norfair/drawing/color.py b/evadb/functions/norfair/drawing/color.py new file mode 100644 index 0000000000..40fc4d9d8b --- /dev/null +++ b/evadb/functions/norfair/drawing/color.py @@ -0,0 +1,371 @@ +import re +from typing import Any, Hashable, Iterable, Tuple, Union + +# types + +ColorType = Tuple[int, int, int] +ColorLike = Union[ColorType, str] + + +def hex_to_bgr(hex_value: str) -> ColorType: + """Converts conventional 6 digits hex colors to BGR tuples + + Parameters + ---------- + hex_value : str + hex value with leading `#` for instance `"#ff0000"` + + Returns + ------- + Tuple[int, int, int] + BGR values + + Raises + ------ + ValueError + if the string is invalid + """ + if re.match("#[a-f0-9]{6}$", hex_value): + return ( + int(hex_value[5:7], 16), + int(hex_value[3:5], 16), + int(hex_value[1:3], 16), + ) + + if re.match("#[a-f0-9]{3}$", hex_value): + return ( + int(hex_value[3] * 2, 16), + int(hex_value[2] * 2, 16), + int(hex_value[1] * 2, 16), + ) + raise ValueError(f"'{hex_value}' is not a valid color") + + +class Color: + """ + Contains predefined colors. + + Colors are defined as a Tuple of integers between 0 and 255 expressing the values in BGR + This is the format opencv uses. + """ + + # from PIL.ImageColors.colormap + aliceblue = hex_to_bgr("#f0f8ff") + antiquewhite = hex_to_bgr("#faebd7") + aqua = hex_to_bgr("#00ffff") + aquamarine = hex_to_bgr("#7fffd4") + azure = hex_to_bgr("#f0ffff") + beige = hex_to_bgr("#f5f5dc") + bisque = hex_to_bgr("#ffe4c4") + black = hex_to_bgr("#000000") + blanchedalmond = hex_to_bgr("#ffebcd") + blue = hex_to_bgr("#0000ff") + blueviolet = hex_to_bgr("#8a2be2") + brown = hex_to_bgr("#a52a2a") + burlywood = hex_to_bgr("#deb887") + cadetblue = hex_to_bgr("#5f9ea0") + chartreuse = hex_to_bgr("#7fff00") + chocolate = hex_to_bgr("#d2691e") + coral = hex_to_bgr("#ff7f50") + cornflowerblue = hex_to_bgr("#6495ed") + cornsilk = hex_to_bgr("#fff8dc") + crimson = hex_to_bgr("#dc143c") + cyan = hex_to_bgr("#00ffff") + darkblue = hex_to_bgr("#00008b") + darkcyan = hex_to_bgr("#008b8b") + darkgoldenrod = hex_to_bgr("#b8860b") + darkgray = hex_to_bgr("#a9a9a9") + darkgrey = hex_to_bgr("#a9a9a9") + darkgreen = hex_to_bgr("#006400") + darkkhaki = hex_to_bgr("#bdb76b") + darkmagenta = hex_to_bgr("#8b008b") + darkolivegreen = hex_to_bgr("#556b2f") + darkorange = hex_to_bgr("#ff8c00") + darkorchid = hex_to_bgr("#9932cc") + darkred = hex_to_bgr("#8b0000") + darksalmon = hex_to_bgr("#e9967a") + darkseagreen = hex_to_bgr("#8fbc8f") + darkslateblue = hex_to_bgr("#483d8b") + darkslategray = hex_to_bgr("#2f4f4f") + darkslategrey = hex_to_bgr("#2f4f4f") + darkturquoise = hex_to_bgr("#00ced1") + darkviolet = hex_to_bgr("#9400d3") + deeppink = hex_to_bgr("#ff1493") + deepskyblue = hex_to_bgr("#00bfff") + dimgray = hex_to_bgr("#696969") + dimgrey = hex_to_bgr("#696969") + dodgerblue = hex_to_bgr("#1e90ff") + firebrick = hex_to_bgr("#b22222") + floralwhite = hex_to_bgr("#fffaf0") + forestgreen = hex_to_bgr("#228b22") + fuchsia = hex_to_bgr("#ff00ff") + gainsboro = hex_to_bgr("#dcdcdc") + ghostwhite = hex_to_bgr("#f8f8ff") + gold = hex_to_bgr("#ffd700") + goldenrod = hex_to_bgr("#daa520") + gray = hex_to_bgr("#808080") + grey = hex_to_bgr("#808080") + green = (0, 128, 0) + greenyellow = hex_to_bgr("#adff2f") + honeydew = hex_to_bgr("#f0fff0") + hotpink = hex_to_bgr("#ff69b4") + indianred = hex_to_bgr("#cd5c5c") + indigo = hex_to_bgr("#4b0082") + ivory = hex_to_bgr("#fffff0") + khaki = hex_to_bgr("#f0e68c") + lavender = hex_to_bgr("#e6e6fa") + lavenderblush = hex_to_bgr("#fff0f5") + lawngreen = hex_to_bgr("#7cfc00") + lemonchiffon = hex_to_bgr("#fffacd") + lightblue = hex_to_bgr("#add8e6") + lightcoral = hex_to_bgr("#f08080") + lightcyan = hex_to_bgr("#e0ffff") + lightgoldenrodyellow = hex_to_bgr("#fafad2") + lightgreen = hex_to_bgr("#90ee90") + lightgray = hex_to_bgr("#d3d3d3") + lightgrey = hex_to_bgr("#d3d3d3") + lightpink = hex_to_bgr("#ffb6c1") + lightsalmon = hex_to_bgr("#ffa07a") + lightseagreen = hex_to_bgr("#20b2aa") + lightskyblue = hex_to_bgr("#87cefa") + lightslategray = hex_to_bgr("#778899") + lightslategrey = hex_to_bgr("#778899") + lightsteelblue = hex_to_bgr("#b0c4de") + lightyellow = hex_to_bgr("#ffffe0") + lime = hex_to_bgr("#00ff00") + limegreen = hex_to_bgr("#32cd32") + linen = hex_to_bgr("#faf0e6") + magenta = hex_to_bgr("#ff00ff") + maroon = hex_to_bgr("#800000") + mediumaquamarine = hex_to_bgr("#66cdaa") + mediumblue = hex_to_bgr("#0000cd") + mediumorchid = hex_to_bgr("#ba55d3") + mediumpurple = hex_to_bgr("#9370db") + mediumseagreen = hex_to_bgr("#3cb371") + mediumslateblue = hex_to_bgr("#7b68ee") + mediumspringgreen = hex_to_bgr("#00fa9a") + mediumturquoise = hex_to_bgr("#48d1cc") + mediumvioletred = hex_to_bgr("#c71585") + midnightblue = hex_to_bgr("#191970") + mintcream = hex_to_bgr("#f5fffa") + mistyrose = hex_to_bgr("#ffe4e1") + moccasin = hex_to_bgr("#ffe4b5") + navajowhite = hex_to_bgr("#ffdead") + navy = hex_to_bgr("#000080") + oldlace = hex_to_bgr("#fdf5e6") + olive = hex_to_bgr("#808000") + olivedrab = hex_to_bgr("#6b8e23") + orange = hex_to_bgr("#ffa500") + orangered = hex_to_bgr("#ff4500") + orchid = hex_to_bgr("#da70d6") + palegoldenrod = hex_to_bgr("#eee8aa") + palegreen = hex_to_bgr("#98fb98") + paleturquoise = hex_to_bgr("#afeeee") + palevioletred = hex_to_bgr("#db7093") + papayawhip = hex_to_bgr("#ffefd5") + peachpuff = hex_to_bgr("#ffdab9") + peru = hex_to_bgr("#cd853f") + pink = hex_to_bgr("#ffc0cb") + plum = hex_to_bgr("#dda0dd") + powderblue = hex_to_bgr("#b0e0e6") + purple = hex_to_bgr("#800080") + rebeccapurple = hex_to_bgr("#663399") + red = hex_to_bgr("#ff0000") + rosybrown = hex_to_bgr("#bc8f8f") + royalblue = hex_to_bgr("#4169e1") + saddlebrown = hex_to_bgr("#8b4513") + salmon = hex_to_bgr("#fa8072") + sandybrown = hex_to_bgr("#f4a460") + seagreen = hex_to_bgr("#2e8b57") + seashell = hex_to_bgr("#fff5ee") + sienna = hex_to_bgr("#a0522d") + silver = hex_to_bgr("#c0c0c0") + skyblue = hex_to_bgr("#87ceeb") + slateblue = hex_to_bgr("#6a5acd") + slategray = hex_to_bgr("#708090") + slategrey = hex_to_bgr("#708090") + snow = hex_to_bgr("#fffafa") + springgreen = hex_to_bgr("#00ff7f") + steelblue = hex_to_bgr("#4682b4") + tan = hex_to_bgr("#d2b48c") + teal = hex_to_bgr("#008080") + thistle = hex_to_bgr("#d8bfd8") + tomato = hex_to_bgr("#ff6347") + turquoise = hex_to_bgr("#40e0d0") + violet = hex_to_bgr("#ee82ee") + wheat = hex_to_bgr("#f5deb3") + white = hex_to_bgr("#ffffff") + whitesmoke = hex_to_bgr("#f5f5f5") + yellow = hex_to_bgr("#ffff00") + yellowgreen = hex_to_bgr("#9acd32") + + # seaborn tab20 colors + tab1 = hex_to_bgr("#1f77b4") + tab2 = hex_to_bgr("#aec7e8") + tab3 = hex_to_bgr("#ff7f0e") + tab4 = hex_to_bgr("#ffbb78") + tab5 = hex_to_bgr("#2ca02c") + tab6 = hex_to_bgr("#98df8a") + tab7 = hex_to_bgr("#d62728") + tab8 = hex_to_bgr("#ff9896") + tab9 = hex_to_bgr("#9467bd") + tab10 = hex_to_bgr("#c5b0d5") + tab11 = hex_to_bgr("#8c564b") + tab12 = hex_to_bgr("#c49c94") + tab13 = hex_to_bgr("#e377c2") + tab14 = hex_to_bgr("#f7b6d2") + tab15 = hex_to_bgr("#7f7f7f") + tab16 = hex_to_bgr("#c7c7c7") + tab17 = hex_to_bgr("#bcbd22") + tab18 = hex_to_bgr("#dbdb8d") + tab19 = hex_to_bgr("#17becf") + tab20 = hex_to_bgr("#9edae5") + # seaborn colorblind + cb1 = hex_to_bgr("#0173b2") + cb2 = hex_to_bgr("#de8f05") + cb3 = hex_to_bgr("#029e73") + cb4 = hex_to_bgr("#d55e00") + cb5 = hex_to_bgr("#cc78bc") + cb6 = hex_to_bgr("#ca9161") + cb7 = hex_to_bgr("#fbafe4") + cb8 = hex_to_bgr("#949494") + cb9 = hex_to_bgr("#ece133") + cb10 = hex_to_bgr("#56b4e9") + + +def parse_color(color_like: ColorLike) -> ColorType: + """Makes best effort to parse the given value to a Color + + Parameters + ---------- + color_like : ColorLike + Can be one of: + + 1. a string with the 6 digits hex value (`"#ff0000"`) + 2. a string with one of the names defined in Colors (`"red"`) + 3. a BGR tuple (`(0, 0, 255)`) + + Returns + ------- + Color + The BGR tuple. + """ + if isinstance(color_like, str): + if color_like.startswith("#"): + return hex_to_bgr(color_like) + else: + return getattr(Color, color_like) + # TODO: validate? + return tuple([int(v) for v in color_like]) + + +PALETTES = { + "tab10": [ + Color.tab1, + Color.tab3, + Color.tab5, + Color.tab7, + Color.tab9, + Color.tab11, + Color.tab13, + Color.tab15, + Color.tab17, + Color.tab19, + ], + "tab20": [ + Color.tab1, + Color.tab2, + Color.tab3, + Color.tab4, + Color.tab5, + Color.tab6, + Color.tab7, + Color.tab8, + Color.tab9, + Color.tab10, + Color.tab11, + Color.tab12, + Color.tab13, + Color.tab14, + Color.tab15, + Color.tab16, + Color.tab17, + Color.tab18, + Color.tab19, + Color.tab20, + ], + "colorblind": [ + Color.cb1, + Color.cb2, + Color.cb3, + Color.cb4, + Color.cb5, + Color.cb6, + Color.cb7, + Color.cb8, + Color.cb9, + Color.cb10, + ], +} + + +class Palette: + """ + Class to control the color pallete for drawing. + + Examples + -------- + Change palette: + >>> from norfair import Palette + >>> Palette.set("colorblind") + >>> # or a custom palette + >>> from norfair import Color + >>> Palette.set([Color.red, Color.blue, "#ffeeff"]) + """ + + _colors = PALETTES["tab10"] + _default_color = Color.black + + @classmethod + def set(cls, palette: Union[str, Iterable[ColorLike]]): + """ + Selects a color palette. + + Parameters + ---------- + palette : Union[str, Iterable[ColorLike]] + can be either + - the name of one of the predefined palettes `tab10`, `tab20`, or `colorblind` + - a list of ColorLike objects that can be parsed by [`parse_color`][norfair.drawing.color.parse_color] + """ + if isinstance(palette, str): + try: + cls._colors = PALETTES[palette] + except KeyError as e: + raise ValueError( + f"Invalid palette name '{palette}', valid values are {PALETTES.keys()}" + ) from e + else: + colors = [] + for c in palette: + colors.append(parse_color(c)) + + cls._colors = colors + + @classmethod + def set_default_color(cls, color: ColorLike): + """ + Selects the default color of `choose_color` when hashable is None. + + Parameters + ---------- + color : ColorLike + The new default color. + """ + cls._default_color = parse_color(color) + + @classmethod + def choose_color(cls, hashable: Hashable) -> ColorType: + if hashable is None: + return cls._default_color + return cls._colors[abs(hash(hashable)) % len(cls._colors)] diff --git a/evadb/functions/norfair/drawing/draw_boxes.py b/evadb/functions/norfair/drawing/draw_boxes.py new file mode 100644 index 0000000000..ef44e78285 --- /dev/null +++ b/evadb/functions/norfair/drawing/draw_boxes.py @@ -0,0 +1,209 @@ +from typing import Optional, Sequence, Tuple, Union + +import numpy as np + +from norfair.tracker import Detection, TrackedObject +from norfair.utils import warn_once + +from .color import ColorLike, Palette, parse_color +from .drawer import Drawable, Drawer +from .utils import _build_text + + +def draw_boxes( + frame: np.ndarray, + drawables: Union[Sequence[Detection], Sequence[TrackedObject]] = None, + color: ColorLike = "by_id", + thickness: Optional[int] = None, + random_color: bool = None, # Deprecated + color_by_label: bool = None, # Deprecated + draw_labels: bool = False, + text_size: Optional[float] = None, + draw_ids: bool = False, + text_color: Optional[ColorLike] = None, + text_thickness: Optional[int] = None, + draw_box: bool = True, + detections: Sequence["Detection"] = None, # Deprecated + line_color: Optional[ColorLike] = None, # Deprecated + line_width: Optional[int] = None, # Deprecated + label_size: Optional[int] = None, # Deprecated´ + draw_scores: bool = False, +) -> np.ndarray: + """ + Draw bounding boxes corresponding to Detections or TrackedObjects. + + Parameters + ---------- + frame : np.ndarray + The OpenCV frame to draw on. Modified in place. + drawables : Union[Sequence[Detection], Sequence[TrackedObject]], optional + List of objects to draw, Detections and TrackedObjects are accepted. + This objects are assumed to contain 2 bi-dimensional points defining + the bounding box as `[[x0, y0], [x1, y1]]`. + color : ColorLike, optional + This parameter can take: + + 1. A color as a tuple of ints describing the BGR `(0, 0, 255)` + 2. A 6-digit hex string `"#FF0000"` + 3. One of the defined color names `"red"` + 4. A string defining the strategy to choose colors from the Palette: + + 1. based on the id of the objects `"by_id"` + 2. based on the label of the objects `"by_label"` + 3. random choice `"random"` + + If using `by_id` or `by_label` strategy but your objects don't + have that field defined (Detections never have ids) the + selected color will be the same for all objects (Palette's default Color). + thickness : Optional[int], optional + Thickness or width of the line. + random_color : bool, optional + **Deprecated**. Set color="random". + color_by_label : bool, optional + **Deprecated**. Set color="by_label". + draw_labels : bool, optional + If set to True, the label is added to a title that is drawn on top of the box. + If an object doesn't have a label this parameter is ignored. + draw_scores : bool, optional + If set to True, the score is added to a title that is drawn on top of the box. + If an object doesn't have a label this parameter is ignored. + text_size : Optional[float], optional + Size of the title, the value is used as a multiplier of the base size of the font. + By default the size is scaled automatically based on the frame size. + draw_ids : bool, optional + If set to True, the id is added to a title that is drawn on top of the box. + If an object doesn't have an id this parameter is ignored. + text_color : Optional[ColorLike], optional + Color of the text. By default the same color as the box is used. + text_thickness : Optional[int], optional + Thickness of the font. By default it's scaled with the `text_size`. + draw_box : bool, optional + Set to False to hide the box and just draw the text. + detections : Sequence[Detection], optional + **Deprecated**. Use drawables. + line_color: Optional[ColorLike], optional + **Deprecated**. Use color. + line_width: Optional[int], optional + **Deprecated**. Use thickness. + label_size: Optional[int], optional + **Deprecated**. Use text_size. + + Returns + ------- + np.ndarray + The resulting frame. + """ + # + # handle deprecated parameters + # + if random_color is not None: + warn_once( + 'Parameter "random_color" is deprecated, set `color="random"` instead' + ) + color = "random" + if color_by_label is not None: + warn_once( + 'Parameter "color_by_label" is deprecated, set `color="by_label"` instead' + ) + color = "by_label" + if detections is not None: + warn_once('Parameter "detections" is deprecated, use "drawables" instead') + drawables = detections + if line_color is not None: + warn_once('Parameter "line_color" is deprecated, use "color" instead') + color = line_color + if line_width is not None: + warn_once('Parameter "line_width" is deprecated, use "thickness" instead') + thickness = line_width + if label_size is not None: + warn_once('Parameter "label_size" is deprecated, use "text_size" instead') + text_size = label_size + # end + + if color is None: + color = "by_id" + if thickness is None: + thickness = int(max(frame.shape) / 500) + + if drawables is None: + return frame + + if text_color is not None: + text_color = parse_color(text_color) + + for obj in drawables: + if not isinstance(obj, Drawable): + d = Drawable(obj) + else: + d = obj + + if color == "by_id": + obj_color = Palette.choose_color(d.id) + elif color == "by_label": + obj_color = Palette.choose_color(d.label) + elif color == "random": + obj_color = Palette.choose_color(np.random.rand()) + else: + obj_color = parse_color(color) + + points = d.points.astype(int) + if draw_box: + Drawer.rectangle( + frame, + tuple(points), + color=obj_color, + thickness=thickness, + ) + + text = _build_text( + d, draw_labels=draw_labels, draw_ids=draw_ids, draw_scores=draw_scores + ) + if text: + if text_color is None: + obj_text_color = obj_color + else: + obj_text_color = text_color + # the anchor will become the bottom-left of the text, + # we select-top left of the bbox compensating for the thickness of the box + text_anchor = ( + points[0, 0] - thickness // 2, + points[0, 1] - thickness // 2 - 1, + ) + frame = Drawer.text( + frame, + text, + position=text_anchor, + size=text_size, + color=obj_text_color, + thickness=text_thickness, + ) + + return frame + + +def draw_tracked_boxes( + frame: np.ndarray, + objects: Sequence["TrackedObject"], + border_colors: Optional[Tuple[int, int, int]] = None, + border_width: Optional[int] = None, + id_size: Optional[int] = None, + id_thickness: Optional[int] = None, + draw_box: bool = True, + color_by_label: bool = False, + draw_labels: bool = False, + label_size: Optional[int] = None, + label_width: Optional[int] = None, +) -> np.array: + "**Deprecated**. Use [`draw_box`][norfair.drawing.draw_boxes.draw_boxes]" + warn_once("draw_tracked_boxes is deprecated, use draw_box instead") + return draw_boxes( + frame=frame, + drawables=objects, + color="by_label" if color_by_label else border_colors, + thickness=border_width, + text_size=label_size or id_size, + text_thickness=id_thickness or label_width, + draw_labels=draw_labels, + draw_ids=id_size is not None and id_size > 0, + draw_box=draw_box, + ) diff --git a/evadb/functions/norfair/drawing/draw_points.py b/evadb/functions/norfair/drawing/draw_points.py new file mode 100644 index 0000000000..e513e3016f --- /dev/null +++ b/evadb/functions/norfair/drawing/draw_points.py @@ -0,0 +1,333 @@ +from typing import Optional, Sequence, Union + +import numpy as np + +from norfair.tracker import Detection, TrackedObject +from norfair.utils import warn_once + +from .color import ColorLike, Palette, parse_color +from .drawer import Drawable, Drawer +from .utils import _build_text + + +def draw_points( + frame: np.ndarray, + drawables: Union[Sequence[Detection], Sequence[TrackedObject]] = None, + radius: Optional[int] = None, + thickness: Optional[int] = None, + color: ColorLike = "by_id", + color_by_label: bool = None, # deprecated + draw_labels: bool = True, + text_size: Optional[int] = None, + draw_ids: bool = True, + draw_points: bool = True, # pylint: disable=redefined-outer-name + text_thickness: Optional[int] = None, + text_color: Optional[ColorLike] = None, + hide_dead_points: bool = True, + detections: Sequence["Detection"] = None, # deprecated + label_size: Optional[int] = None, # deprecated + draw_scores: bool = False, +) -> np.ndarray: + """ + Draw the points included in a list of Detections or TrackedObjects. + + Parameters + ---------- + frame : np.ndarray + The OpenCV frame to draw on. Modified in place. + drawables : Union[Sequence[Detection], Sequence[TrackedObject]], optional + List of objects to draw, Detections and TrackedObjects are accepted. + radius : Optional[int], optional + Radius of the circles representing each point. + By default a sensible value is picked considering the frame size. + thickness : Optional[int], optional + Thickness or width of the line. + color : ColorLike, optional + This parameter can take: + + 1. A color as a tuple of ints describing the BGR `(0, 0, 255)` + 2. A 6-digit hex string `"#FF0000"` + 3. One of the defined color names `"red"` + 4. A string defining the strategy to choose colors from the Palette: + + 1. based on the id of the objects `"by_id"` + 2. based on the label of the objects `"by_label"` + 3. random choice `"random"` + + If using `by_id` or `by_label` strategy but your objects don't + have that field defined (Detections never have ids) the + selected color will be the same for all objects (Palette's default Color). + color_by_label : bool, optional + **Deprecated**. set `color="by_label"`. + draw_labels : bool, optional + If set to True, the label is added to a title that is drawn on top of the box. + If an object doesn't have a label this parameter is ignored. + draw_scores : bool, optional + If set to True, the score is added to a title that is drawn on top of the box. + If an object doesn't have a label this parameter is ignored. + text_size : Optional[int], optional + Size of the title, the value is used as a multiplier of the base size of the font. + By default the size is scaled automatically based on the frame size. + draw_ids : bool, optional + If set to True, the id is added to a title that is drawn on top of the box. + If an object doesn't have an id this parameter is ignored. + draw_points : bool, optional + Set to False to hide the points and just draw the text. + text_thickness : Optional[int], optional + Thickness of the font. By default it's scaled with the `text_size`. + text_color : Optional[ColorLike], optional + Color of the text. By default the same color as the box is used. + hide_dead_points : bool, optional + Set this param to False to always draw all points, even the ones considered "dead". + A point is "dead" when the corresponding value of `TrackedObject.live_points` + is set to False. If all objects are dead the object is not drawn. + All points of a detection are considered to be alive. + detections : Sequence[Detection], optional + **Deprecated**. use drawables. + label_size : Optional[int], optional + **Deprecated**. text_size. + + Returns + ------- + np.ndarray + The resulting frame. + """ + # + # handle deprecated parameters + # + if color_by_label is not None: + warn_once( + 'Parameter "color_by_label" on function draw_points is deprecated, set `color="by_label"` instead' + ) + color = "by_label" + if detections is not None: + warn_once( + "Parameter 'detections' on function draw_points is deprecated, use 'drawables' instead" + ) + drawables = detections + if label_size is not None: + warn_once( + "Parameter 'label_size' on function draw_points is deprecated, use 'text_size' instead" + ) + text_size = label_size + # end + + if drawables is None: + return + + if text_color is not None: + text_color = parse_color(text_color) + + if color is None: + color = "by_id" + if thickness is None: + thickness = -1 + if radius is None: + radius = int(round(max(max(frame.shape) * 0.002, 1))) + + for o in drawables: + if not isinstance(o, Drawable): + d = Drawable(o) + else: + d = o + + if hide_dead_points and not d.live_points.any(): + continue + + if color == "by_id": + obj_color = Palette.choose_color(d.id) + elif color == "by_label": + obj_color = Palette.choose_color(d.label) + elif color == "random": + obj_color = Palette.choose_color(np.random.rand()) + else: + obj_color = parse_color(color) + + if text_color is None: + obj_text_color = obj_color + else: + obj_text_color = text_color + + if draw_points: + for point, live in zip(d.points, d.live_points): + if live or not hide_dead_points: + Drawer.circle( + frame, + tuple(point.astype(int)), + radius=radius, + color=obj_color, + thickness=thickness, + ) + + if draw_labels or draw_ids or draw_scores: + position = d.points[d.live_points].mean(axis=0) + position -= radius + text = _build_text( + d, draw_labels=draw_labels, draw_ids=draw_ids, draw_scores=draw_scores + ) + + Drawer.text( + frame, + text, + tuple(position.astype(int)), + size=text_size, + color=obj_text_color, + thickness=text_thickness, + ) + + return frame + + +# HACK add an alias to prevent error in the function below +# the deprecated draw_tracked_objects accepts a parameter called +# "draw_points" which overwrites the function "draw_points" from above +# since draw_tracked_objects needs to call this function, an alias +# is defined that can be used to call draw_points +_draw_points_alias = draw_points + + +def draw_tracked_objects( + frame: np.ndarray, + objects: Sequence["TrackedObject"], + radius: Optional[int] = None, + color: Optional[ColorLike] = None, + id_size: Optional[float] = None, + id_thickness: Optional[int] = None, + draw_points: bool = True, # pylint: disable=redefined-outer-name + color_by_label: bool = False, + draw_labels: bool = False, + label_size: Optional[int] = None, +): + """ + **Deprecated** use [`draw_points`][norfair.drawing.draw_points.draw_points] + """ + warn_once("draw_tracked_objects is deprecated, use draw_points instead") + + frame_scale = frame.shape[0] / 100 + if radius is None: + radius = int(frame_scale * 0.5) + if id_size is None: + id_size = frame_scale / 10 + if id_thickness is None: + id_thickness = int(frame_scale / 5) + if label_size is None: + label_size = int(max(frame_scale / 100, 1)) + + _draw_points_alias( + frame=frame, + drawables=objects, + color="by_label" if color_by_label else color, + radius=radius, + thickness=None, + draw_labels=draw_labels, + draw_ids=id_size is not None and id_size > 0, + draw_points=draw_points, + text_size=label_size or id_size, + text_thickness=id_thickness, + text_color=None, + hide_dead_points=True, + ) + + +# TODO: We used to have this function to debug +# migrate it to use Drawer and clean it up +# if possible maybe merge this functionality to the function above + +# def draw_debug_metrics( +# frame: np.ndarray, +# objects: Sequence["TrackedObject"], +# text_size: Optional[float] = None, +# text_thickness: Optional[int] = None, +# color: Optional[Tuple[int, int, int]] = None, +# only_ids=None, +# only_initializing_ids=None, +# draw_score_threshold: float = 0, +# color_by_label: bool = False, +# draw_labels: bool = False, +# ): +# """Draw objects with their debug information + +# It is recommended to set the input variable `objects` to `your_tracker_object.objects` +# so you can also debug objects wich haven't finished initializing, and you get a more +# complete view of what your tracker is doing on each step. +# """ +# frame_scale = frame.shape[0] / 100 +# if text_size is None: +# text_size = frame_scale / 10 +# if text_thickness is None: +# text_thickness = int(frame_scale / 5) +# radius = int(frame_scale * 0.5) + +# for obj in objects: +# if ( +# not (obj.last_detection.scores is None) +# and not (obj.last_detection.scores > draw_score_threshold).any() +# ): +# continue +# if only_ids is not None: +# if obj.id not in only_ids: +# continue +# if only_initializing_ids is not None: +# if obj.initializing_id not in only_initializing_ids: +# continue +# if color_by_label: +# text_color = Color.random(abs(hash(obj.label))) +# elif color is None: +# text_color = Color.random(obj.initializing_id) +# else: +# text_color = color +# draw_position = _centroid( +# obj.estimate[obj.last_detection.scores > draw_score_threshold] +# if obj.last_detection.scores is not None +# else obj.estimate +# ) + +# for point in obj.estimate: +# cv2.circle( +# frame, +# tuple(point.astype(int)), +# radius=radius, +# color=text_color, +# thickness=-1, +# ) + +# # Distance to last matched detection +# if obj.last_distance is None: +# last_dist = "-" +# elif obj.last_distance > 999: +# last_dist = ">" +# else: +# last_dist = "{:.2f}".format(obj.last_distance) + +# # Distance to currently closest detection +# if obj.current_min_distance is None: +# current_min_dist = "-" +# else: +# current_min_dist = "{:.2f}".format(obj.current_min_distance) + +# # No support for multiline text in opencv :facepalm: +# lines_to_draw = [ +# "{}|{}".format(obj.id, obj.initializing_id), +# "a:{}".format(obj.age), +# "h:{}".format(obj.hit_counter), +# "ld:{}".format(last_dist), +# "cd:{}".format(current_min_dist), +# ] +# if draw_labels: +# lines_to_draw.append("l:{}".format(obj.label)) + +# for i, line in enumerate(lines_to_draw): +# draw_position = ( +# int(draw_position[0]), +# int(draw_position[1] + i * text_size * 7 + 15), +# ) +# cv2.putText( +# frame, +# line, +# draw_position, +# cv2.FONT_HERSHEY_SIMPLEX, +# text_size, +# text_color, +# text_thickness, +# cv2.LINE_AA, +# ) diff --git a/evadb/functions/norfair/drawing/drawer.py b/evadb/functions/norfair/drawing/drawer.py new file mode 100644 index 0000000000..8365714322 --- /dev/null +++ b/evadb/functions/norfair/drawing/drawer.py @@ -0,0 +1,363 @@ +from typing import Any, Optional, Sequence, Tuple, Union + +import numpy as np + +from norfair.drawing.color import Color, ColorType +from norfair.tracker import Detection, TrackedObject + +try: + import cv2 +except ImportError: + from norfair.utils import DummyOpenCVImport + + cv2 = DummyOpenCVImport() + + +class Drawer: + """ + Basic drawing functionality. + + This class encapsulates opencv drawing functions allowing for + different backends to be implemented following the same interface. + """ + + @classmethod + def circle( + cls, + frame: np.ndarray, + position: Tuple[int, int], + radius: Optional[int] = None, + thickness: Optional[int] = None, + color: ColorType = None, + ) -> np.ndarray: + """ + Draw a circle. + + Parameters + ---------- + frame : np.ndarray + The OpenCV frame to draw on. Modified in place. + position : Tuple[int, int] + Position of the point. This will become the center of the circle. + radius : Optional[int], optional + Radius of the circle. + thickness : Optional[int], optional + Thickness or width of the line. + color : Color, optional + A tuple of ints describing the BGR color `(0, 0, 255)`. + + Returns + ------- + np.ndarray + The resulting frame. + """ + if radius is None: + radius = int(max(max(frame.shape) * 0.005, 1)) + if thickness is None: + thickness = radius - 1 + + return cv2.circle( + frame, + position, + radius=radius, + color=color, + thickness=thickness, + ) + + @classmethod + def text( + cls, + frame: np.ndarray, + text: str, + position: Tuple[int, int], + size: Optional[float] = None, + color: Optional[ColorType] = None, + thickness: Optional[int] = None, + shadow: bool = True, + shadow_color: ColorType = Color.black, + shadow_offset: int = 1, + ) -> np.ndarray: + """ + Draw text + + Parameters + ---------- + frame : np.ndarray + The OpenCV frame to draw on. Modified in place. + text : str + The text to be written. + position : Tuple[int, int] + Position of the bottom-left corner of the text. + This value is adjusted considering the thickness automatically. + size : Optional[float], optional + Scale of the font, by default chooses a sensible value is picked based on the size of the frame. + color : Optional[ColorType], optional + Color of the text, by default is black. + thickness : Optional[int], optional + Thickness of the lines, by default a sensible value is picked based on the size. + shadow : bool, optional + If True, a shadow of the text is added which improves legibility. + shadow_color : Color, optional + Color of the shadow. + shadow_offset : int, optional + Offset of the shadow. + + Returns + ------- + np.ndarray + The resulting frame. + """ + if size is None: + size = min(max(max(frame.shape) / 4000, 0.5), 1.5) + if thickness is None: + thickness = int(round(size) + 1) + + if thickness is None and size is not None: + thickness = int(round(size) + 1) + # adjust position based on the thickness + anchor = (position[0] + thickness // 2, position[1] - thickness // 2) + if shadow: + frame = cv2.putText( + frame, + text, + (anchor[0] + shadow_offset, anchor[1] + shadow_offset), + cv2.FONT_HERSHEY_SIMPLEX, + size, + shadow_color, + thickness, + cv2.LINE_AA, + ) + return cv2.putText( + frame, + text, + anchor, + cv2.FONT_HERSHEY_SIMPLEX, + size, + color, + thickness, + cv2.LINE_AA, + ) + + @classmethod + def rectangle( + cls, + frame: np.ndarray, + points: Sequence[Tuple[int, int]], + color: Optional[ColorType] = None, + thickness: Optional[int] = None, + ) -> np.ndarray: + """ + Draw a rectangle + + Parameters + ---------- + frame : np.ndarray + The OpenCV frame to draw on. Modified in place. + points : Sequence[Tuple[int, int]] + Points describing the rectangle in the format `[[x0, y0], [x1, y1]]`. + color : Optional[ColorType], optional + Color of the lines, by default Black. + thickness : Optional[int], optional + Thickness of the lines, by default 1. + + Returns + ------- + np.ndarray + The resulting frame. + """ + frame = cv2.rectangle( + frame, + tuple(points[0]), + tuple(points[1]), + color=color, + thickness=thickness, + ) + return frame + + @classmethod + def cross( + cls, + frame: np.ndarray, + center: Tuple[int, int], + radius: int, + color: ColorType, + thickness: int, + ) -> np.ndarray: + """ + Draw a cross + + Parameters + ---------- + frame : np.ndarray + The OpenCV frame to draw on. Modified in place. + center : Tuple[int, int] + Center of the cross. + radius : int + Size or radius of the cross. + color : Color + Color of the lines. + thickness : int + Thickness of the lines. + + Returns + ------- + np.ndarray + The resulting frame. + """ + middle_x, middle_y = center + left, top = center - radius + right, bottom = center + radius + frame = cls.line( + frame, + start=(middle_x, top), + end=(middle_x, bottom), + color=color, + thickness=thickness, + ) + frame = cls.line( + frame, + start=(left, middle_y), + end=(right, middle_y), + color=color, + thickness=thickness, + ) + return frame + + @classmethod + def line( + cls, + frame: np.ndarray, + start: Tuple[int, int], + end: Tuple[int, int], + color: ColorType = Color.black, + thickness: int = 1, + ) -> np.ndarray: + """ + Draw a line. + + Parameters + ---------- + frame : np.ndarray + The OpenCV frame to draw on. Modified in place. + start : Tuple[int, int] + Starting point. + end : Tuple[int, int] + End point. + color : ColorType, optional + Line color. + thickness : int, optional + Line width. + + Returns + ------- + np.ndarray + The resulting frame. + """ + return cv2.line( + frame, + pt1=start, + pt2=end, + color=color, + thickness=thickness, + ) + + @classmethod + def alpha_blend( + cls, + frame1: np.ndarray, + frame2: np.ndarray, + alpha: float = 0.5, + beta: Optional[float] = None, + gamma: float = 0, + ) -> np.ndarray: + """ + Blend 2 frame as a wheigthted sum. + + Parameters + ---------- + frame1 : np.ndarray + An OpenCV frame. + frame2 : np.ndarray + An OpenCV frame. + alpha : float, optional + Weight of frame1. + beta : Optional[float], optional + Weight of frame2, by default `1 - alpha` + gamma : float, optional + Scalar to add to the sum. + + Returns + ------- + np.ndarray + The resulting frame. + """ + if beta is None: + beta = 1 - alpha + return cv2.addWeighted( + src1=frame1, src2=frame2, alpha=alpha, beta=beta, gamma=gamma + ) + + +class Drawable: + """ + Class to standardize Drawable objects like Detections and TrackedObjects + + Parameters + ---------- + obj : Union[Detection, TrackedObject], optional + A [Detection][norfair.tracker.Detection] or a [TrackedObject][norfair.tracker.TrackedObject] + that will be used to initialized the drawable. + If this parameter is passed, all other arguments are ignored + points : np.ndarray, optional + Points included in the drawable, shape is `(N_points, N_dimensions)`. Ignored if `obj` is passed + id : Any, optional + Id of this object. Ignored if `obj` is passed + label : Any, optional + Label specifying the class of the object. Ignored if `obj` is passed + scores : np.ndarray, optional + Confidence scores of each point, shape is `(N_points,)`. Ignored if `obj` is passed + live_points : np.ndarray, optional + Bolean array indicating which points are alive, shape is `(N_points,)`. Ignored if `obj` is passed + + Raises + ------ + ValueError + If obj is not an instance of the supported classes. + """ + + def __init__( + self, + obj: Union[Detection, TrackedObject] = None, + points: np.ndarray = None, + id: Any = None, + label: Any = None, + scores: np.ndarray = None, + live_points: np.ndarray = None, + ) -> None: + if isinstance(obj, Detection): + self.points = obj.points + self.id = None + self.label = obj.label + self.scores = obj.scores + # TODO: alive points for detections could be the ones over the threshold + # but that info is not available here + self.live_points = np.ones(obj.points.shape[0]).astype(bool) + + elif isinstance(obj, TrackedObject): + self.points = obj.estimate + self.id = obj.id + self.label = obj.label + # TODO: TrackedObject.scores could be an interesting thing to have + # it could be the scores of the last detection or some kind of moving average + self.scores = None + self.live_points = obj.live_points + elif obj is None: + self.points = points + self.id = id + self.label = label + self.scores = scores + self.live_points = live_points + else: + raise ValueError( + f"Extecting a Detection or a TrackedObject but received {type(obj)}" + ) diff --git a/evadb/functions/norfair/drawing/fixed_camera.py b/evadb/functions/norfair/drawing/fixed_camera.py new file mode 100644 index 0000000000..6c04906323 --- /dev/null +++ b/evadb/functions/norfair/drawing/fixed_camera.py @@ -0,0 +1,141 @@ +import numpy as np + +from norfair.camera_motion import TranslationTransformation +from norfair.utils import warn_once + + +class FixedCamera: + """ + Class used to stabilize video based on the camera motion. + + Starts with a larger frame, where the original frame is drawn on top of a black background. + As the camera moves, the smaller frame moves in the opposite direction, stabilizing the objects in it. + + Useful for debugging or demoing the camera motion. + ![Example GIF](../../videos/camera_stabilization.gif) + + !!! Warning + This only works with [`TranslationTransformation`][norfair.camera_motion.TranslationTransformation], + using [`HomographyTransformation`][norfair.camera_motion.HomographyTransformation] will result in + unexpected behaviour. + + !!! Warning + If using other drawers, always apply this one last. Using other drawers on the scaled up frame will not work as expected. + + !!! Note + Sometimes the camera moves so far from the original point that the result won't fit in the scaled-up frame. + In this case, a warning will be logged and the frames will be cropped to avoid errors. + + Parameters + ---------- + scale : float, optional + The resulting video will have a resolution of `scale * (H, W)` where HxW is the resolution of the original video. + Use a bigger scale if the camera is moving too much. + attenuation : float, optional + Controls how fast the older frames fade to black. + + Examples + -------- + >>> # setup + >>> tracker = Tracker("frobenious", 100) + >>> motion_estimator = MotionEstimator() + >>> video = Video(input_path="video.mp4") + >>> fixed_camera = FixedCamera() + >>> # process video + >>> for frame in video: + >>> coord_transformations = motion_estimator.update(frame) + >>> detections = get_detections(frame) + >>> tracked_objects = tracker.update(detections, coord_transformations) + >>> draw_tracked_objects(frame, tracked_objects) # fixed_camera should always be the last drawer + >>> bigger_frame = fixed_camera.adjust_frame(frame, coord_transformations) + >>> video.write(bigger_frame) + """ + + def __init__(self, scale: float = 2, attenuation: float = 0.05): + self.scale = scale + self._background = None + self._attenuation_factor = 1 - attenuation + + def adjust_frame( + self, frame: np.ndarray, coord_transformation: TranslationTransformation + ) -> np.ndarray: + """ + Render scaled up frame. + + Parameters + ---------- + frame : np.ndarray + The OpenCV frame. + coord_transformation : TranslationTransformation + The coordinate transformation as returned by the [`MotionEstimator`][norfair.camera_motion.MotionEstimator] + + Returns + ------- + np.ndarray + The new bigger frame with the original frame drawn on it. + """ + + # initialize background if necessary + if self._background is None: + original_size = ( + frame.shape[1], + frame.shape[0], + ) # OpenCV format is (width, height) + + scaled_size = tuple( + (np.array(original_size) * np.array(self.scale)).round().astype(int) + ) + self._background = np.zeros( + [scaled_size[1], scaled_size[0], frame.shape[-1]], + frame.dtype, + ) + else: + self._background = (self._background * self._attenuation_factor).astype( + frame.dtype + ) + + # top_left is the anchor coordinate from where we start drawing the fame on top of the background + # aim to draw it in the center of the background but transformations will move this point + top_left = ( + np.array(self._background.shape[:2]) // 2 - np.array(frame.shape[:2]) // 2 + ) + top_left = ( + coord_transformation.rel_to_abs(top_left[::-1]).round().astype(int)[::-1] + ) + # box of the background that will be updated and the limits of it + background_y0, background_y1 = (top_left[0], top_left[0] + frame.shape[0]) + background_x0, background_x1 = (top_left[1], top_left[1] + frame.shape[1]) + background_size_y, background_size_x = self._background.shape[:2] + + # define box of the frame that will be used + # if the scale is not enough to support the movement, warn the user but keep drawing + # cropping the frame so that the operation doesn't fail + frame_y0, frame_y1, frame_x0, frame_x1 = (0, frame.shape[0], 0, frame.shape[1]) + if ( + background_y0 < 0 + or background_x0 < 0 + or background_y1 > background_size_y + or background_x1 > background_size_x + ): + warn_once( + "moving_camera_scale is not enough to cover the range of camera movement, frame will be cropped" + ) + # crop left or top of the frame if necessary + frame_y0 = max(-background_y0, 0) + frame_x0 = max(-background_x0, 0) + # crop right or bottom of the frame if necessary + frame_y1 = max( + min(background_size_y - background_y0, background_y1 - background_y0), 0 + ) + frame_x1 = max( + min(background_size_x - background_x0, background_x1 - background_x0), 0 + ) + # handle cases where the limits of the background become negative which numpy will interpret incorrectly + background_y0 = max(background_y0, 0) + background_x0 = max(background_x0, 0) + background_y1 = max(background_y1, 0) + background_x1 = max(background_x1, 0) + self._background[ + background_y0:background_y1, background_x0:background_x1, : + ] = frame[frame_y0:frame_y1, frame_x0:frame_x1, :] + return self._background diff --git a/evadb/functions/norfair/drawing/path.py b/evadb/functions/norfair/drawing/path.py new file mode 100644 index 0000000000..7f97f3dd57 --- /dev/null +++ b/evadb/functions/norfair/drawing/path.py @@ -0,0 +1,232 @@ +from collections import defaultdict +from typing import Callable, Optional, Sequence, Tuple + +import numpy as np + +from norfair.drawing.color import Palette +from norfair.drawing.drawer import Drawer +from norfair.tracker import TrackedObject +from norfair.utils import warn_once + + +class Paths: + """ + Class that draws the paths taken by a set of points of interest defined from the coordinates of each tracker estimation. + + Parameters + ---------- + get_points_to_draw : Optional[Callable[[np.array], np.array]], optional + Function that takes a list of points (the `.estimate` attribute of a [`TrackedObject`][norfair.tracker.TrackedObject]) + and returns a list of points for which we want to draw their paths. + + By default it is the mean point of all the points in the tracker. + thickness : Optional[int], optional + Thickness of the circles representing the paths of interest. + color : Optional[Tuple[int, int, int]], optional + [Color][norfair.drawing.Color] of the circles representing the paths of interest. + radius : Optional[int], optional + Radius of the circles representing the paths of interest. + attenuation : float, optional + A float number in [0, 1] that dictates the speed at which the path is erased. + if it is `0` then the path is never erased. + + Examples + -------- + >>> from norfair import Tracker, Video, Path + >>> video = Video("video.mp4") + >>> tracker = Tracker(...) + >>> path_drawer = Path() + >>> for frame in video: + >>> detections = get_detections(frame) # runs detector and returns Detections + >>> tracked_objects = tracker.update(detections) + >>> frame = path_drawer.draw(frame, tracked_objects) + >>> video.write(frame) + """ + + def __init__( + self, + get_points_to_draw: Optional[Callable[[np.array], np.array]] = None, + thickness: Optional[int] = None, + color: Optional[Tuple[int, int, int]] = None, + radius: Optional[int] = None, + attenuation: float = 0.01, + ): + if get_points_to_draw is None: + + def get_points_to_draw(points): + return [np.mean(np.array(points), axis=0)] + + self.get_points_to_draw = get_points_to_draw + + self.radius = radius + self.thickness = thickness + self.color = color + self.mask = None + self.attenuation_factor = 1 - attenuation + + def draw( + self, frame: np.ndarray, tracked_objects: Sequence[TrackedObject] + ) -> np.array: + """ + Draw the paths of the points interest on a frame. + + !!! warning + This method does **not** draw frames in place as other drawers do, the resulting frame is returned. + + Parameters + ---------- + frame : np.ndarray + The OpenCV frame to draw on. + tracked_objects : Sequence[TrackedObject] + List of [`TrackedObject`][norfair.tracker.TrackedObject] to get the points of interest in order to update the paths. + + Returns + ------- + np.array + The resulting frame. + """ + if self.mask is None: + frame_scale = frame.shape[0] / 100 + + if self.radius is None: + self.radius = int(max(frame_scale * 0.7, 1)) + if self.thickness is None: + self.thickness = int(max(frame_scale / 7, 1)) + + self.mask = np.zeros(frame.shape, np.uint8) + + self.mask = (self.mask * self.attenuation_factor).astype("uint8") + + for obj in tracked_objects: + if obj.abs_to_rel is not None: + warn_once( + "It seems that your using the Path drawer together with MotionEstimator. This is not fully supported and the results will not be what's expected" + ) + + if self.color is None: + color = Palette.choose_color(obj.id) + else: + color = self.color + + points_to_draw = self.get_points_to_draw(obj.estimate) + + for point in points_to_draw: + frame = Drawer.circle( + self.mask, + position=tuple(point.astype(int)), + radius=self.radius, + color=color, + thickness=self.thickness, + ) + + return Drawer.alpha_blend(self.mask, frame, alpha=1, beta=1) + + +class AbsolutePaths: + """ + Class that draws the absolute paths taken by a set of points. + + Works just like [`Paths`][norfair.drawing.Paths] but supports camera motion. + + !!! warning + This drawer is not optimized so it can be stremely slow. Performance degrades linearly with + `max_history * number_of_tracked_objects`. + + Parameters + ---------- + get_points_to_draw : Optional[Callable[[np.array], np.array]], optional + Function that takes a list of points (the `.estimate` attribute of a [`TrackedObject`][norfair.tracker.TrackedObject]) + and returns a list of points for which we want to draw their paths. + + By default it is the mean point of all the points in the tracker. + thickness : Optional[int], optional + Thickness of the circles representing the paths of interest. + color : Optional[Tuple[int, int, int]], optional + [Color][norfair.drawing.Color] of the circles representing the paths of interest. + radius : Optional[int], optional + Radius of the circles representing the paths of interest. + max_history : int, optional + Number of past points to include in the path. High values make the drawing slower + + Examples + -------- + >>> from norfair import Tracker, Video, Path + >>> video = Video("video.mp4") + >>> tracker = Tracker(...) + >>> path_drawer = Path() + >>> for frame in video: + >>> detections = get_detections(frame) # runs detector and returns Detections + >>> tracked_objects = tracker.update(detections) + >>> frame = path_drawer.draw(frame, tracked_objects) + >>> video.write(frame) + """ + + def __init__( + self, + get_points_to_draw: Optional[Callable[[np.array], np.array]] = None, + thickness: Optional[int] = None, + color: Optional[Tuple[int, int, int]] = None, + radius: Optional[int] = None, + max_history=20, + ): + + if get_points_to_draw is None: + + def get_points_to_draw(points): + return [np.mean(np.array(points), axis=0)] + + self.get_points_to_draw = get_points_to_draw + + self.radius = radius + self.thickness = thickness + self.color = color + self.past_points = defaultdict(lambda: []) + self.max_history = max_history + self.alphas = np.linspace(0.99, 0.01, max_history) + + def draw(self, frame, tracked_objects, coord_transform=None): + frame_scale = frame.shape[0] / 100 + + if self.radius is None: + self.radius = int(max(frame_scale * 0.7, 1)) + if self.thickness is None: + self.thickness = int(max(frame_scale / 7, 1)) + for obj in tracked_objects: + if not obj.live_points.any(): + continue + + if self.color is None: + color = Palette.choose_color(obj.id) + else: + color = self.color + + points_to_draw = self.get_points_to_draw(obj.get_estimate(absolute=True)) + + for point in coord_transform.abs_to_rel(points_to_draw): + Drawer.circle( + frame, + position=tuple(point.astype(int)), + radius=self.radius, + color=color, + thickness=self.thickness, + ) + + last = points_to_draw + for i, past_points in enumerate(self.past_points[obj.id]): + overlay = frame.copy() + last = coord_transform.abs_to_rel(last) + for j, point in enumerate(coord_transform.abs_to_rel(past_points)): + Drawer.line( + overlay, + tuple(last[j].astype(int)), + tuple(point.astype(int)), + color=color, + thickness=self.thickness, + ) + last = past_points + + alpha = self.alphas[i] + frame = Drawer.alpha_blend(overlay, frame, alpha=alpha) + self.past_points[obj.id].insert(0, points_to_draw) + self.past_points[obj.id] = self.past_points[obj.id][: self.max_history] + return frame diff --git a/evadb/functions/norfair/drawing/utils.py b/evadb/functions/norfair/drawing/utils.py new file mode 100644 index 0000000000..6b227b74cd --- /dev/null +++ b/evadb/functions/norfair/drawing/utils.py @@ -0,0 +1,28 @@ +from typing import TYPE_CHECKING, Optional, Sequence, Tuple + +import numpy as np + +if TYPE_CHECKING: + from .drawer import Drawable + + +def _centroid(tracked_points: np.ndarray) -> Tuple[int, int]: + num_points = tracked_points.shape[0] + sum_x = np.sum(tracked_points[:, 0]) + sum_y = np.sum(tracked_points[:, 1]) + return int(sum_x / num_points), int(sum_y / num_points) + + +def _build_text(drawable: "Drawable", draw_labels, draw_ids, draw_scores): + text = "" + if draw_labels and drawable.label is not None: + text = str(drawable.label) + if draw_ids and drawable.id is not None: + if len(text) > 0: + text += "-" + text += str(drawable.id) + if draw_scores and drawable.scores is not None: + if len(text) > 0: + text += "-" + text += str(np.round(np.mean(drawable.scores), 4)) + return text diff --git a/evadb/functions/norfair/filter.py b/evadb/functions/norfair/filter.py new file mode 100644 index 0000000000..b086eebd4e --- /dev/null +++ b/evadb/functions/norfair/filter.py @@ -0,0 +1,283 @@ +from abc import ABC, abstractmethod + +import numpy as np +from filterpy.kalman import KalmanFilter + + +class FilterFactory(ABC): + """Abstract class representing a generic Filter factory + + Subclasses must implement the method `create_filter` + """ + + @abstractmethod + def create_filter(self, initial_detection: np.ndarray): + pass + + +class FilterPyKalmanFilterFactory(FilterFactory): + """ + This class can be used either to change some parameters of the [KalmanFilter](https://filterpy.readthedocs.io/en/latest/kalman/KalmanFilter.html) + that the tracker uses, or to fully customize the predictive filter implementation to use (as long as the methods and properties are compatible). + + The former case only requires changing the default parameters upon tracker creation: `tracker = Tracker(..., filter_factory=FilterPyKalmanFilterFactory(R=100))`, + while the latter requires creating your own class extending `FilterPyKalmanFilterFactory`, and rewriting its `create_filter` method to return your own customized filter. + + Parameters + ---------- + R : float, optional + Multiplier for the sensor measurement noise matrix, by default 4.0 + Q : float, optional + Multiplier for the process uncertainty, by default 0.1 + P : float, optional + Multiplier for the initial covariance matrix estimation, only in the entries that correspond to position (not speed) variables, by default 10.0 + + See Also + -------- + [`filterpy.KalmanFilter`](https://filterpy.readthedocs.io/en/latest/kalman/KalmanFilter.html). + """ + + def __init__(self, R: float = 4.0, Q: float = 0.1, P: float = 10.0): + self.R = R + self.Q = Q + self.P = P + + def create_filter(self, initial_detection: np.ndarray) -> KalmanFilter: + """ + This method returns a new predictive filter instance with the current setup, to be used by each new [`TrackedObject`][norfair.tracker.TrackedObject] that is created. + This predictive filter will be used to estimate speed and future positions of the object, to better match the detections during its trajectory. + + Parameters + ---------- + initial_detection : np.ndarray + numpy array of shape `(number of points per object, 2)`, corresponding to the [`Detection.points`][norfair.tracker.Detection] of the tracked object being born, + which shall be used as initial position estimation for it. + + Returns + ------- + KalmanFilter + The kalman filter + """ + num_points = initial_detection.shape[0] + dim_points = initial_detection.shape[1] + dim_z = dim_points * num_points + dim_x = 2 * dim_z # We need to accommodate for velocities + + filter = KalmanFilter(dim_x=dim_x, dim_z=dim_z) + + # State transition matrix (models physics): numpy.array() + filter.F = np.eye(dim_x) + dt = 1 # At each step we update pos with v * dt + + filter.F[:dim_z, dim_z:] = dt * np.eye(dim_z) + + # Measurement function: numpy.array(dim_z, dim_x) + filter.H = np.eye( + dim_z, + dim_x, + ) + + # Measurement uncertainty (sensor noise): numpy.array(dim_z, dim_z) + filter.R *= self.R + + # Process uncertainty: numpy.array(dim_x, dim_x) + # Don't decrease it too much or trackers pay too little attention to detections + filter.Q[dim_z:, dim_z:] *= self.Q + + # Initial state: numpy.array(dim_x, 1) + filter.x[:dim_z] = np.expand_dims(initial_detection.flatten(), 0).T + filter.x[dim_z:] = 0 + + # Estimation uncertainty: numpy.array(dim_x, dim_x) + filter.P[dim_z:, dim_z:] *= self.P + + return filter + + +class NoFilter: + def __init__(self, dim_x, dim_z): + self.dim_z = dim_z + self.x = np.zeros((dim_x, 1)) + + def predict(self): + return + + def update(self, detection_points_flatten, R=None, H=None): + + if H is not None: + diagonal = np.diagonal(H).reshape((self.dim_z, 1)) + one_minus_diagonal = 1 - diagonal + + detection_points_flatten = np.multiply( + diagonal, detection_points_flatten + ) + np.multiply(one_minus_diagonal, self.x[: self.dim_z]) + + self.x[: self.dim_z] = detection_points_flatten + + +class NoFilterFactory(FilterFactory): + """ + This class allows the user to try Norfair without any predictive filter or velocity estimation. + + This track only by comparing the position of the previous detections to the ones in the current frame. + + The throughput of this class in FPS is similar to the one achieved by the + [`OptimizedKalmanFilterFactory`](#optimizedkalmanfilterfactory) class, so this class exists only for + comparative purposes and it is not advised to use it for tracking on a real application. + + Parameters + ---------- + FilterFactory : _type_ + _description_ + """ + + def create_filter(self, initial_detection: np.ndarray): + num_points = initial_detection.shape[0] + dim_points = initial_detection.shape[1] + dim_z = dim_points * num_points # flattened positions + dim_x = 2 * dim_z # We need to accommodate for velocities + + no_filter = NoFilter( + dim_x, + dim_z, + ) + no_filter.x[:dim_z] = np.expand_dims(initial_detection.flatten(), 0).T + return no_filter + + +class OptimizedKalmanFilter: + def __init__( + self, + dim_x, + dim_z, + pos_variance=10, + pos_vel_covariance=0, + vel_variance=1, + q=0.1, + r=4, + ): + self.dim_z = dim_z + self.x = np.zeros((dim_x, 1)) + + # matrix P from Kalman + self.pos_variance = np.zeros((dim_z, 1)) + pos_variance + self.pos_vel_covariance = np.zeros((dim_z, 1)) + pos_vel_covariance + self.vel_variance = np.zeros((dim_z, 1)) + vel_variance + + self.q_Q = q + + self.default_r = r * np.ones((dim_z, 1)) + + def predict(self): + self.x[: self.dim_z] += self.x[self.dim_z :] + + def update(self, detection_points_flatten, R=None, H=None): + + if H is not None: + diagonal = np.diagonal(H).reshape((self.dim_z, 1)) + one_minus_diagonal = 1 - diagonal + else: + diagonal = np.ones((self.dim_z, 1)) + one_minus_diagonal = np.zeros((self.dim_z, 1)) + + if R is not None: + kalman_r = np.diagonal(R).reshape((self.dim_z, 1)) + else: + kalman_r = self.default_r + + error = np.multiply(detection_points_flatten - self.x[: self.dim_z], diagonal) + + vel_var_plus_pos_vel_cov = self.pos_vel_covariance + self.vel_variance + added_variances = ( + self.pos_variance + + self.pos_vel_covariance + + vel_var_plus_pos_vel_cov + + self.q_Q + + kalman_r + ) + + kalman_r_over_added_variances = np.divide(kalman_r, added_variances) + vel_var_plus_pos_vel_cov_over_added_variances = np.divide( + vel_var_plus_pos_vel_cov, added_variances + ) + + added_variances_or_kalman_r = np.multiply( + added_variances, one_minus_diagonal + ) + np.multiply(kalman_r, diagonal) + + self.x[: self.dim_z] += np.multiply( + diagonal, np.multiply(1 - kalman_r_over_added_variances, error) + ) + self.x[self.dim_z :] += np.multiply( + diagonal, np.multiply(vel_var_plus_pos_vel_cov_over_added_variances, error) + ) + + self.pos_variance = np.multiply( + 1 - kalman_r_over_added_variances, added_variances_or_kalman_r + ) + self.pos_vel_covariance = np.multiply( + vel_var_plus_pos_vel_cov_over_added_variances, added_variances_or_kalman_r + ) + self.vel_variance += self.q_Q - np.multiply( + diagonal, + np.multiply( + np.square(vel_var_plus_pos_vel_cov_over_added_variances), + added_variances, + ), + ) + + +class OptimizedKalmanFilterFactory(FilterFactory): + """ + Creates faster Filters than [`FilterPyKalmanFilterFactory`][norfair.filter.FilterPyKalmanFilterFactory]. + + It allows the user to create Kalman Filter optimized for tracking and set its parameters. + + Parameters + ---------- + R : float, optional + Multiplier for the sensor measurement noise matrix. + Q : float, optional + Multiplier for the process uncertainty. + pos_variance : float, optional + Multiplier for the initial covariance matrix estimation, only in the entries that correspond to position (not speed) variables. + pos_vel_covariance : float, optional + Multiplier for the initial covariance matrix estimation, only in the entries that correspond to the covariance between position and speed. + vel_variance : float, optional + Multiplier for the initial covariance matrix estimation, only in the entries that correspond to velocity (not position) variables. + """ + + def __init__( + self, + R: float = 4.0, + Q: float = 0.1, + pos_variance: float = 10, + pos_vel_covariance: float = 0, + vel_variance: float = 1, + ): + self.R = R + self.Q = Q + + # entrances P matrix of KF + self.pos_variance = pos_variance + self.pos_vel_covariance = pos_vel_covariance + self.vel_variance = vel_variance + + def create_filter(self, initial_detection: np.ndarray): + num_points = initial_detection.shape[0] + dim_points = initial_detection.shape[1] + dim_z = dim_points * num_points # flattened positions + dim_x = 2 * dim_z # We need to accommodate for velocities + + custom_filter = OptimizedKalmanFilter( + dim_x, + dim_z, + pos_variance=self.pos_variance, + pos_vel_covariance=self.pos_vel_covariance, + vel_variance=self.vel_variance, + q=self.Q, + r=self.R, + ) + custom_filter.x[:dim_z] = np.expand_dims(initial_detection.flatten(), 0).T + + return custom_filter diff --git a/evadb/functions/norfair/metrics.py b/evadb/functions/norfair/metrics.py new file mode 100644 index 0000000000..b1e056de47 --- /dev/null +++ b/evadb/functions/norfair/metrics.py @@ -0,0 +1,348 @@ +import os + +import numpy as np +from rich import print +from rich.progress import track + +from norfair import Detection + +try: + import motmetrics as mm + import pandas as pd +except ImportError: + from .utils import DummyMOTMetricsImport + + mm = DummyMOTMetricsImport() + pandas = DummyMOTMetricsImport() +from collections import OrderedDict + + +class InformationFile: + def __init__(self, file_path): + self.path = file_path + with open(file_path, "r") as myfile: + file = myfile.read() + self.lines = file.splitlines() + + def search(self, variable_name): + for line in self.lines: + if line[: len(variable_name)] == variable_name: + result = line[len(variable_name) + 1 :] + break + else: + raise ValueError(f"Couldn't find '{variable_name}' in {self.path}") + if result.isdigit(): + return int(result) + else: + return result + + +class PredictionsTextFile: + """Generates a text file with your predicted tracked objects, in the MOTChallenge format. + It needs the 'input_path', which is the path to the sequence being processed, + the 'save_path', and optionally the 'information_file' (in case you don't give an + 'information_file', is assumed there is one in the input_path folder). + """ + + def __init__(self, input_path, save_path=".", information_file=None): + + file_name = os.path.split(input_path)[1] + + if information_file is None: + seqinfo_path = os.path.join(input_path, "seqinfo.ini") + information_file = InformationFile(file_path=seqinfo_path) + + self.length = information_file.search(variable_name="seqLength") + + predictions_folder = os.path.join(save_path, "predictions") + if not os.path.exists(predictions_folder): + os.makedirs(predictions_folder) + + out_file_name = os.path.join(predictions_folder, file_name + ".txt") + self.text_file = open(out_file_name, "w+") + + self.frame_number = 1 + + def update(self, predictions, frame_number=None): + if frame_number is None: + frame_number = self.frame_number + """ + Write tracked object information in the output file (for this frame), in the format + frame_number, id, bb_left, bb_top, bb_width, bb_height, -1, -1, -1, -1 + """ + for obj in predictions: + frame_str = str(int(frame_number)) + id_str = str(int(obj.id)) + bb_left_str = str((obj.estimate[0, 0])) + bb_top_str = str((obj.estimate[0, 1])) # [0,1] + bb_width_str = str((obj.estimate[1, 0] - obj.estimate[0, 0])) + bb_height_str = str((obj.estimate[1, 1] - obj.estimate[0, 1])) + row_text_out = ( + frame_str + + "," + + id_str + + "," + + bb_left_str + + "," + + bb_top_str + + "," + + bb_width_str + + "," + + bb_height_str + + ",-1,-1,-1,-1" + ) + self.text_file.write(row_text_out) + self.text_file.write("\n") + + self.frame_number += 1 + + if self.frame_number > self.length: + self.text_file.close() + + +class DetectionFileParser: + """Get Norfair detections from MOTChallenge text files containing detections""" + + def __init__(self, input_path, information_file=None): + self.frame_number = 1 + + # Get detecions matrix data with rows corresponding to: + # frame, id, bb_left, bb_top, bb_right, bb_down, conf, x, y, z + detections_path = os.path.join(input_path, "det/det.txt") + + self.matrix_detections = np.loadtxt(detections_path, dtype="f", delimiter=",") + row_order = np.argsort(self.matrix_detections[:, 0]) + self.matrix_detections = self.matrix_detections[row_order] + # Coordinates refer to box corners + self.matrix_detections[:, 4] = ( + self.matrix_detections[:, 2] + self.matrix_detections[:, 4] + ) + self.matrix_detections[:, 5] = ( + self.matrix_detections[:, 3] + self.matrix_detections[:, 5] + ) + + if information_file is None: + seqinfo_path = os.path.join(input_path, "seqinfo.ini") + information_file = InformationFile(file_path=seqinfo_path) + self.length = information_file.search(variable_name="seqLength") + + self.sorted_by_frame = [] + for frame_number in range(1, self.length + 1): + self.sorted_by_frame.append(self.get_dets_from_frame(frame_number)) + + def get_dets_from_frame(self, frame_number): + """this function returns a list of norfair Detections class, corresponding to frame=frame_number""" + + indexes = np.argwhere(self.matrix_detections[:, 0] == frame_number) + detections = [] + if len(indexes) > 0: + actual_det = self.matrix_detections[indexes] + actual_det.shape = [actual_det.shape[0], actual_det.shape[2]] + for det in actual_det: + points = np.array([[det[2], det[3]], [det[4], det[5]]]) + conf = det[6] + new_detection = Detection(points, np.array([conf, conf])) + detections.append(new_detection) + self.actual_detections = detections + return detections + + def __iter__(self): + self.frame_number = 1 + return self + + def __next__(self): + if self.frame_number <= self.length: + self.frame_number += 1 + # Frame_number is always 1 unit bigger than the corresponding index in self.sorted_by_frame, and + # also we just incremented the frame_number, so now is 2 units bigger than the corresponding index + return self.sorted_by_frame[self.frame_number - 2] + + raise StopIteration() + + +class Accumulators: + def __init__(self): + self.matrixes_predictions = [] + self.paths = [] + + def create_accumulator(self, input_path, information_file=None): + # Check that motmetrics is installed here, so we don't have to process + # the whole dataset before failing out if we don't. + mm.metrics + + file_name = os.path.split(input_path)[1] + + self.frame_number = 1 + # Save the path of this video in a list + self.paths = np.hstack((self.paths, input_path)) + # Initialize a matrix where we will save our predictions for this video (in the MOTChallenge format) + self.matrix_predictions = [] + + # Initialize progress bar + if information_file is None: + seqinfo_path = os.path.join(input_path, "seqinfo.ini") + information_file = InformationFile(file_path=seqinfo_path) + length = information_file.search(variable_name="seqLength") + self.progress_bar_iter = track( + range(length - 1), description=file_name, transient=False + ) + + def update(self, predictions=None): + # Get the tracked boxes from this frame in an array + for obj in predictions: + new_row = [ + self.frame_number, + obj.id, + obj.estimate[0, 0], + obj.estimate[0, 1], + obj.estimate[1, 0] - obj.estimate[0, 0], + obj.estimate[1, 1] - obj.estimate[0, 1], + -1, + -1, + -1, + -1, + ] + if np.shape(self.matrix_predictions)[0] == 0: + self.matrix_predictions = new_row + else: + self.matrix_predictions = np.vstack((self.matrix_predictions, new_row)) + self.frame_number += 1 + # Advance in progress bar + try: + next(self.progress_bar_iter) + except StopIteration: + self.matrixes_predictions.append(self.matrix_predictions) + return + + def compute_metrics( + self, + metrics=None, + generate_overall=True, + ): + if metrics is None: + metrics = list(mm.metrics.motchallenge_metrics) + + self.summary_text, self.summary_dataframe = eval_motChallenge( + matrixes_predictions=self.matrixes_predictions, + paths=self.paths, + metrics=metrics, + generate_overall=generate_overall, + ) + + return self.summary_dataframe + + def save_metrics(self, save_path=".", file_name="metrics.txt"): + if not os.path.exists(save_path): + os.makedirs(save_path) + + metrics_path = os.path.join(save_path, file_name) + metrics_file = open(metrics_path, "w+") + metrics_file.write(self.summary_text) + metrics_file.close() + + def print_metrics(self): + print(self.summary_text) + + +def load_motchallenge(matrix_data, min_confidence=-1): + """Load MOT challenge data. + + This is a modification of the function load_motchallenge from the py-motmetrics library, defined in io.py + In this version, the pandas dataframe is generated from a numpy array (matrix_data) instead of a text file. + + Params + ------ + matrix_data : array of float that has [frame, id, X, Y, width, height, conf, cassId, visibility] in each row, for each prediction on a particular video + + min_confidence : float + Rows with confidence less than this threshold are removed. + Defaults to -1. You should set this to 1 when loading + ground truth MOTChallenge data, so that invalid rectangles in + the ground truth are not considered during matching. + + Returns + ------ + df : pandas.DataFrame + The returned dataframe has the following columns + 'X', 'Y', 'Width', 'Height', 'Confidence', 'ClassId', 'Visibility' + The dataframe is indexed by ('FrameId', 'Id') + """ + + df = pd.DataFrame( + data=matrix_data, + columns=[ + "FrameId", + "Id", + "X", + "Y", + "Width", + "Height", + "Confidence", + "ClassId", + "Visibility", + "unused", + ], + ) + df = df.set_index(["FrameId", "Id"]) + # Account for matlab convention. + df[["X", "Y"]] -= (1, 1) + + # Removed trailing column + del df["unused"] + + # Remove all rows without sufficient confidence + return df[df["Confidence"] >= min_confidence] + + +def compare_dataframes(gts, ts): + """Builds accumulator for each sequence.""" + accs = [] + names = [] + for k, tsacc in ts.items(): + print("Comparing ", k, "...") + if k in gts: + accs.append( + mm.utils.compare_to_groundtruth(gts[k], tsacc, "iou", distth=0.5) + ) + names.append(k) + + return accs, names + + +def eval_motChallenge(matrixes_predictions, paths, metrics=None, generate_overall=True): + gt = OrderedDict( + [ + ( + os.path.split(p)[1], + mm.io.loadtxt( + os.path.join(p, "gt/gt.txt"), fmt="mot15-2D", min_confidence=1 + ), + ) + for p in paths + ] + ) + + ts = OrderedDict( + [ + (os.path.split(paths[n])[1], load_motchallenge(matrixes_predictions[n])) + for n in range(len(paths)) + ] + ) + + mh = mm.metrics.create() + + accs, names = compare_dataframes(gt, ts) + + if metrics is None: + metrics = list(mm.metrics.motchallenge_metrics) + mm.lap.default_solver = "scipy" + print("Computing metrics...") + summary_dataframe = mh.compute_many( + accs, names=names, metrics=metrics, generate_overall=generate_overall + ) + summary_text = mm.io.render_summary( + summary_dataframe, + formatters=mh.formatters, + namemap=mm.io.motchallenge_metric_names, + ) + return summary_text, summary_dataframe diff --git a/evadb/functions/norfair/tracker.py b/evadb/functions/norfair/tracker.py new file mode 100644 index 0000000000..63f5e4a69c --- /dev/null +++ b/evadb/functions/norfair/tracker.py @@ -0,0 +1,794 @@ +from logging import warning +from typing import Any, Callable, Hashable, List, Optional, Sequence, Tuple, Union + +import numpy as np +from rich import print + +from norfair.camera_motion import CoordinatesTransformation + +from .distances import ( + AVAILABLE_VECTORIZED_DISTANCES, + ScalarDistance, + get_distance_by_name, +) +from .filter import FilterFactory, OptimizedKalmanFilterFactory +from .utils import validate_points + + +class Tracker: + """ + The class in charge of performing the tracking of the detections produced by a detector. + + Parameters + ---------- + distance_function : Union[str, Callable[[Detection, TrackedObject], float]] + Function used by the tracker to determine the distance between newly detected objects and the objects that are currently being tracked. + This function should take 2 input arguments, the first being a [Detection][norfair.tracker.Detection], and the second a [TrackedObject][norfair.tracker.TrackedObject]. + It has to return a `float` with the distance it calculates. + Some common distances are implemented in [distances][], as a shortcut the tracker accepts the name of these [predefined distances][norfair.distances.get_distance_by_name]. + Scipy's predefined distances are also accepted. A `str` with one of the available metrics in + [`scipy.spatial.distance.cdist`](https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.cdist.html). + distance_threshold : float + Defines what is the maximum distance that can constitute a match. + Detections and tracked objects whose distances are above this threshold won't be matched by the tracker. + hit_counter_max : int, optional + Each tracked objects keeps an internal hit counter which tracks how often it's getting matched to a detection, + each time it gets a match this counter goes up, and each time it doesn't it goes down. + + If it goes below 0 the object gets destroyed. This argument defines how large this inertia can grow, + and therefore defines how long an object can live without getting matched to any detections, before it is displaced as a dead object, if no ReID distance function is implemented it will be destroyed. + initialization_delay : Optional[int], optional + Determines how large the object's hit counter must be in order to be considered as initialized, and get returned to the user as a real object. + It must be smaller than `hit_counter_max` or otherwise the object would never be initialized. + + If set to 0, objects will get returned to the user as soon as they are detected for the first time, + which can be problematic as this can result in objects appearing and immediately dissapearing. + + Defaults to `hit_counter_max / 2` + pointwise_hit_counter_max : int, optional + Each tracked object keeps track of how often the points it's tracking have been getting matched. + Points that are getting matched (`pointwise_hit_counter > 0`) are said to be live, and points which aren't (`pointwise_hit_counter = 0`) + are said to not be live. + + This is used to determine things like which individual points in a tracked object get drawn by [`draw_tracked_objects`][norfair.drawing.draw_tracked_objects] and which don't. + This argument defines how large the inertia for each point of a tracker can grow. + detection_threshold : float, optional + Sets the threshold at which the scores of the points in a detection being fed into the tracker must dip below to be ignored by the tracker. + filter_factory : FilterFactory, optional + This parameter can be used to change what filter the [`TrackedObject`][norfair.tracker.TrackedObject] instances created by the tracker will use. + Defaults to [`OptimizedKalmanFilterFactory()`][norfair.filter.OptimizedKalmanFilterFactory] + past_detections_length : int, optional + How many past detections to save for each tracked object. + Norfair tries to distribute these past detections uniformly through the object's lifetime so they're more representative. + Very useful if you want to add metric learning to your model, as you can associate an embedding to each detection and access them in your distance function. + reid_distance_function: Optional[Callable[["TrackedObject", "TrackedObject"], float]] + Function used by the tracker to determine the ReID distance between newly detected trackers and unmatched trackers by the distance function. + + This function should take 2 input arguments, the first being tracked objects in the initialization phase of type [`TrackedObject`][norfair.tracker.TrackedObject], + and the second being tracked objects that have been unmatched of type [`TrackedObject`][norfair.tracker.TrackedObject]. It returns a `float` with the distance it + calculates. + reid_distance_threshold: float + Defines what is the maximum ReID distance that can constitute a match. + + Tracked objects whose distance is above this threshold won't be merged, if they are the oldest tracked object will be maintained + with the position of the new tracked object. + reid_hit_counter_max: Optional[int] + Each tracked object keeps an internal ReID hit counter which tracks how often it's getting recognized by another tracker, + each time it gets a match this counter goes up, and each time it doesn't it goes down. If it goes below 0 the object gets destroyed. + If used, this argument (`reid_hit_counter_max`) defines how long an object can live without getting matched to any detections, before it is destroyed. + """ + + def __init__( + self, + distance_function: Union[str, Callable[["Detection", "TrackedObject"], float]], + distance_threshold: float, + hit_counter_max: int = 15, + initialization_delay: Optional[int] = None, + pointwise_hit_counter_max: int = 4, + detection_threshold: float = 0, + filter_factory: FilterFactory = OptimizedKalmanFilterFactory(), + past_detections_length: int = 4, + reid_distance_function: Optional[ + Callable[["TrackedObject", "TrackedObject"], float] + ] = None, + reid_distance_threshold: float = 0, + reid_hit_counter_max: Optional[int] = None, + ): + self.tracked_objects: Sequence["TrackedObject"] = [] + + if isinstance(distance_function, str): + distance_function = get_distance_by_name(distance_function) + elif isinstance(distance_function, Callable): + warning( + "You are using a scalar distance function. If you want to speed up the" + " tracking process please consider using a vectorized distance" + f" function such as {AVAILABLE_VECTORIZED_DISTANCES}." + ) + distance_function = ScalarDistance(distance_function) + else: + raise ValueError( + "Argument `distance_function` should be a string or function but is" + f" {type(distance_function)} instead." + ) + self.distance_function = distance_function + + self.hit_counter_max = hit_counter_max + self.reid_hit_counter_max = reid_hit_counter_max + self.pointwise_hit_counter_max = pointwise_hit_counter_max + self.filter_factory = filter_factory + if past_detections_length >= 0: + self.past_detections_length = past_detections_length + else: + raise ValueError( + f"Argument `past_detections_length` is {past_detections_length} and should be larger than 0." + ) + + if initialization_delay is None: + self.initialization_delay = int(self.hit_counter_max / 2) + elif initialization_delay < 0 or initialization_delay >= self.hit_counter_max: + raise ValueError( + f"Argument 'initialization_delay' for 'Tracker' class should be an int between 0 and (hit_counter_max = {hit_counter_max}). The selected value is {initialization_delay}.\n" + ) + else: + self.initialization_delay = initialization_delay + + self.distance_threshold = distance_threshold + self.detection_threshold = detection_threshold + if reid_distance_function is not None: + self.reid_distance_function = ScalarDistance(reid_distance_function) + else: + self.reid_distance_function = reid_distance_function + self.reid_distance_threshold = reid_distance_threshold + self._obj_factory = _TrackedObjectFactory() + + def update( + self, + detections: Optional[List["Detection"]] = None, + period: int = 1, + coord_transformations: Optional[CoordinatesTransformation] = None, + ) -> List["TrackedObject"]: + """ + Process detections found in each frame. + + The detections can be matched to previous tracked objects or new ones will be created + according to the configuration of the Tracker. + The currently alive and initialized tracked objects are returned + + Parameters + ---------- + detections : Optional[List[Detection]], optional + A list of [`Detection`][norfair.tracker.Detection] which represent the detections found in the current frame being processed. + + If no detections have been found in the current frame, or the user is purposely skipping frames to improve video processing time, + this argument should be set to None or ignored, as the update function is needed to advance the state of the Kalman Filters inside the tracker. + period : int, optional + The user can chose not to run their detector on all frames, so as to process video faster. + This parameter sets every how many frames the detector is getting ran, + so that the tracker is aware of this situation and can handle it properly. + + This argument can be reset on each frame processed, + which is useful if the user is dynamically changing how many frames the detector is skipping on a video when working in real-time. + coord_transformations: Optional[CoordinatesTransformation] + The coordinate transformation calculated by the [MotionEstimator][norfair.camera_motion.MotionEstimator]. + + Returns + ------- + List[TrackedObject] + The list of active tracked objects. + """ + if coord_transformations is not None: + for det in detections: + det.update_coordinate_transformation(coord_transformations) + + # Remove stale trackers and make candidate object real if the hit counter is positive + alive_objects = [] + dead_objects = [] + if self.reid_hit_counter_max is None: + self.tracked_objects = [ + o for o in self.tracked_objects if o.hit_counter_is_positive + ] + alive_objects = self.tracked_objects + else: + tracked_objects = [] + for o in self.tracked_objects: + if o.reid_hit_counter_is_positive: + tracked_objects.append(o) + if o.hit_counter_is_positive: + alive_objects.append(o) + else: + dead_objects.append(o) + self.tracked_objects = tracked_objects + + # Update tracker + for obj in self.tracked_objects: + obj.tracker_step() + obj.update_coordinate_transformation(coord_transformations) + + # Update initialized tracked objects with detections + ( + unmatched_detections, + _, + unmatched_init_trackers, + ) = self._update_objects_in_place( + self.distance_function, + self.distance_threshold, + [o for o in alive_objects if not o.is_initializing], + detections, + period, + ) + + # Update not yet initialized tracked objects with yet unmatched detections + ( + unmatched_detections, + matched_not_init_trackers, + _, + ) = self._update_objects_in_place( + self.distance_function, + self.distance_threshold, + [o for o in alive_objects if o.is_initializing], + unmatched_detections, + period, + ) + + if self.reid_distance_function is not None: + # Match unmatched initialized tracked objects with not yet initialized tracked objects + _, _, _ = self._update_objects_in_place( + self.reid_distance_function, + self.reid_distance_threshold, + unmatched_init_trackers + dead_objects, + matched_not_init_trackers, + period, + ) + + # Create new tracked objects from remaining unmatched detections + for detection in unmatched_detections: + self.tracked_objects.append( + self._obj_factory.create( + initial_detection=detection, + hit_counter_max=self.hit_counter_max, + initialization_delay=self.initialization_delay, + pointwise_hit_counter_max=self.pointwise_hit_counter_max, + detection_threshold=self.detection_threshold, + period=period, + filter_factory=self.filter_factory, + past_detections_length=self.past_detections_length, + reid_hit_counter_max=self.reid_hit_counter_max, + coord_transformations=coord_transformations, + ) + ) + + return self.get_active_objects() + + @property + def current_object_count(self) -> int: + """Number of active TrackedObjects""" + return len(self.get_active_objects()) + + @property + def total_object_count(self) -> int: + """Total number of TrackedObjects initialized in the by this Tracker""" + return self._obj_factory.count + + def get_active_objects(self) -> List["TrackedObject"]: + """Get the list of active objects + + Returns + ------- + List["TrackedObject"] + The list of active objects + """ + return [ + o + for o in self.tracked_objects + if not o.is_initializing and o.hit_counter_is_positive + ] + + def _update_objects_in_place( + self, + distance_function, + distance_threshold, + objects: Sequence["TrackedObject"], + candidates: Optional[Union[List["Detection"], List["TrackedObject"]]], + period: int, + ): + if candidates is not None and len(candidates) > 0: + distance_matrix = distance_function.get_distances(objects, candidates) + if np.isnan(distance_matrix).any(): + print( + "\nReceived nan values from distance function, please check your distance function for errors!" + ) + exit() + + # Used just for debugging distance function + if distance_matrix.any(): + for i, minimum in enumerate(distance_matrix.min(axis=0)): + objects[i].current_min_distance = ( + minimum if minimum < distance_threshold else None + ) + + matched_cand_indices, matched_obj_indices = self.match_dets_and_objs( + distance_matrix, distance_threshold + ) + if len(matched_cand_indices) > 0: + unmatched_candidates = [ + d for i, d in enumerate(candidates) if i not in matched_cand_indices + ] + unmatched_objects = [ + d for i, d in enumerate(objects) if i not in matched_obj_indices + ] + matched_objects = [] + + # Handle matched people/detections + for (match_cand_idx, match_obj_idx) in zip( + matched_cand_indices, matched_obj_indices + ): + match_distance = distance_matrix[match_cand_idx, match_obj_idx] + matched_candidate = candidates[match_cand_idx] + matched_object = objects[match_obj_idx] + if match_distance < distance_threshold: + if isinstance(matched_candidate, Detection): + matched_object.hit(matched_candidate, period=period) + matched_object.last_distance = match_distance + matched_objects.append(matched_object) + elif isinstance(matched_candidate, TrackedObject): + # Merge new TrackedObject with the old one + matched_object.merge(matched_candidate) + # If we are matching TrackedObject instances we want to get rid of the + # already matched candidate to avoid matching it again in future frames + self.tracked_objects.remove(matched_candidate) + else: + unmatched_candidates.append(matched_candidate) + unmatched_objects.append(matched_object) + else: + unmatched_candidates, matched_objects, unmatched_objects = ( + candidates, + [], + objects, + ) + else: + unmatched_candidates, matched_objects, unmatched_objects = [], [], objects + + return unmatched_candidates, matched_objects, unmatched_objects + + def match_dets_and_objs(self, distance_matrix: np.ndarray, distance_threshold): + """Matches detections with tracked_objects from a distance matrix + + I used to match by minimizing the global distances, but found several + cases in which this was not optimal. So now I just match by starting + with the global minimum distance and matching the det-obj corresponding + to that distance, then taking the second minimum, and so on until we + reach the distance_threshold. + + This avoids the the algorithm getting cute with us and matching things + that shouldn't be matching just for the sake of minimizing the global + distance, which is what used to happen + """ + # NOTE: This implementation is terribly inefficient, but it doesn't + # seem to affect the fps at all. + distance_matrix = distance_matrix.copy() + if distance_matrix.size > 0: + det_idxs = [] + obj_idxs = [] + current_min = distance_matrix.min() + + while current_min < distance_threshold: + flattened_arg_min = distance_matrix.argmin() + det_idx = flattened_arg_min // distance_matrix.shape[1] + obj_idx = flattened_arg_min % distance_matrix.shape[1] + det_idxs.append(det_idx) + obj_idxs.append(obj_idx) + distance_matrix[det_idx, :] = distance_threshold + 1 + distance_matrix[:, obj_idx] = distance_threshold + 1 + current_min = distance_matrix.min() + + return det_idxs, obj_idxs + else: + return [], [] + + +class _TrackedObjectFactory: + global_count = 0 + + def __init__(self) -> None: + self.count = 0 + self.initializing_count = 0 + + def create( + self, + initial_detection: "Detection", + hit_counter_max: int, + initialization_delay: int, + pointwise_hit_counter_max: int, + detection_threshold: float, + period: int, + filter_factory: "FilterFactory", + past_detections_length: int, + reid_hit_counter_max: Optional[int], + coord_transformations: CoordinatesTransformation, + ) -> "TrackedObject": + obj = TrackedObject( + obj_factory=self, + initial_detection=initial_detection, + hit_counter_max=hit_counter_max, + initialization_delay=initialization_delay, + pointwise_hit_counter_max=pointwise_hit_counter_max, + detection_threshold=detection_threshold, + period=period, + filter_factory=filter_factory, + past_detections_length=past_detections_length, + reid_hit_counter_max=reid_hit_counter_max, + coord_transformations=coord_transformations, + ) + return obj + + def get_initializing_id(self) -> int: + self.initializing_count += 1 + return self.initializing_count + + def get_ids(self) -> Tuple[int, int]: + self.count += 1 + _TrackedObjectFactory.global_count += 1 + return self.count, _TrackedObjectFactory.global_count + + +class TrackedObject: + """ + The objects returned by the tracker's `update` function on each iteration. + + They represent the objects currently being tracked by the tracker. + + Users should not instantiate TrackedObjects manually; + the Tracker will be in charge of creating them. + + Attributes + ---------- + estimate : np.ndarray + Where the tracker predicts the point will be in the current frame based on past detections. + A numpy array with the same shape as the detections being fed to the tracker that produced it. + id : Optional[int] + The unique identifier assigned to this object by the tracker. Set to `None` if the object is initializing. + global_id : Optional[int] + The globally unique identifier assigned to this object. Set to `None` if the object is initializing + last_detection : Detection + The last detection that matched with this tracked object. + Useful if you are storing embeddings in your detections and want to do metric learning, or for debugging. + last_distance : Optional[float] + The distance the tracker had with the last object it matched with. + age : int + The age of this object measured in number of frames. + live_points : + A boolean mask with shape `(n_points,)`. Points marked as `True` have recently been matched with detections. + Points marked as `False` haven't and are to be considered stale, and should be ignored. + + Functions like [`draw_tracked_objects`][norfair.drawing.draw_tracked_objects] use this property to determine which points not to draw. + initializing_id : int + On top of `id`, objects also have an `initializing_id` which is the id they are given internally by the `Tracker`; + this id is used solely for debugging. + + Each new object created by the `Tracker` starts as an uninitialized `TrackedObject`, + which needs to reach a certain match rate to be converted into a full blown `TrackedObject`. + `initializing_id` is the id temporarily assigned to `TrackedObject` while they are getting initialized. + """ + + def __init__( + self, + obj_factory: _TrackedObjectFactory, + initial_detection: "Detection", + hit_counter_max: int, + initialization_delay: int, + pointwise_hit_counter_max: int, + detection_threshold: float, + period: int, + filter_factory: "FilterFactory", + past_detections_length: int, + reid_hit_counter_max: Optional[int], + coord_transformations: Optional[CoordinatesTransformation] = None, + ): + if not isinstance(initial_detection, Detection): + print( + f"\n[red]ERROR[/red]: The detection list fed into `tracker.update()` should be composed of {Detection} objects not {type(initial_detection)}.\n" + ) + exit() + self._obj_factory = obj_factory + self.dim_points = initial_detection.absolute_points.shape[1] + self.num_points = initial_detection.absolute_points.shape[0] + self.hit_counter_max: int = hit_counter_max + self.pointwise_hit_counter_max: int = max(pointwise_hit_counter_max, period) + self.initialization_delay = initialization_delay + self.detection_threshold: float = detection_threshold + self.initial_period: int = period + self.hit_counter: int = period + self.reid_hit_counter_max = reid_hit_counter_max + self.reid_hit_counter: Optional[int] = None + self.last_distance: Optional[float] = None + self.current_min_distance: Optional[float] = None + self.last_detection: "Detection" = initial_detection + self.age: int = 0 + self.is_initializing: bool = self.hit_counter <= self.initialization_delay + + self.initializing_id: Optional[int] = self._obj_factory.get_initializing_id() + self.id: Optional[int] = None + self.global_id: Optional[int] = None + if not self.is_initializing: + self._acquire_ids() + + if initial_detection.scores is None: + self.detected_at_least_once_points = np.array([True] * self.num_points) + else: + self.detected_at_least_once_points = ( + initial_detection.scores > self.detection_threshold + ) + self.point_hit_counter: np.ndarray = self.detected_at_least_once_points.astype( + int + ) + initial_detection.age = self.age + self.past_detections_length = past_detections_length + if past_detections_length > 0: + self.past_detections: Sequence["Detection"] = [initial_detection] + else: + self.past_detections: Sequence["Detection"] = [] + + # Create Kalman Filter + self.filter = filter_factory.create_filter(initial_detection.absolute_points) + self.dim_z = self.dim_points * self.num_points + self.label = initial_detection.label + self.abs_to_rel = None + if coord_transformations is not None: + self.update_coordinate_transformation(coord_transformations) + + def tracker_step(self): + if self.reid_hit_counter is None: + if self.hit_counter <= 0: + self.reid_hit_counter = self.reid_hit_counter_max + else: + self.reid_hit_counter -= 1 + self.hit_counter -= 1 + self.point_hit_counter -= 1 + self.age += 1 + # Advances the tracker's state + self.filter.predict() + + @property + def hit_counter_is_positive(self): + return self.hit_counter >= 0 + + @property + def reid_hit_counter_is_positive(self): + return self.reid_hit_counter is None or self.reid_hit_counter >= 0 + + @property + def estimate_velocity(self) -> np.ndarray: + """Get the velocity estimate of the object from the Kalman filter. This velocity is in the absolute coordinate system. + + Returns + ------- + np.ndarray + An array of shape (self.num_points, self.dim_points) containing the velocity estimate of the object on each axis. + """ + return self.filter.x.T.flatten()[self.dim_z :].reshape(-1, self.dim_points) + + @property + def estimate(self) -> np.ndarray: + """Get the position estimate of the object from the Kalman filter. + + Returns + ------- + np.ndarray + An array of shape (self.num_points, self.dim_points) containing the position estimate of the object on each axis. + """ + return self.get_estimate() + + def get_estimate(self, absolute=False) -> np.ndarray: + """Get the position estimate of the object from the Kalman filter in an absolute or relative format. + + Parameters + ---------- + absolute : bool, optional + If true the coordinates are returned in absolute format, by default False, by default False. + + Returns + ------- + np.ndarray + An array of shape (self.num_points, self.dim_points) containing the position estimate of the object on each axis. + + Raises + ------ + ValueError + Alert if the coordinates are requested in absolute format but the tracker has no coordinate transformation. + """ + positions = self.filter.x.T.flatten()[: self.dim_z].reshape(-1, self.dim_points) + if self.abs_to_rel is None: + if not absolute: + return positions + else: + raise ValueError( + "You must provide 'coord_transformations' to the tracker to get absolute coordinates" + ) + else: + if absolute: + return positions + else: + return self.abs_to_rel(positions) + + @property + def live_points(self): + return self.point_hit_counter > 0 + + def hit(self, detection: "Detection", period: int = 1): + """Update tracked object with a new detection + + Parameters + ---------- + detection : Detection + the new detection matched to this tracked object + period : int, optional + frames corresponding to the period of time since last update. + """ + self._conditionally_add_to_past_detections(detection) + + self.last_detection = detection + self.hit_counter = min(self.hit_counter + 2 * period, self.hit_counter_max) + + if self.is_initializing and self.hit_counter > self.initialization_delay: + self.is_initializing = False + self._acquire_ids() + + # We use a kalman filter in which we consider each coordinate on each point as a sensor. + # This is a hacky way to update only certain sensors (only x, y coordinates for + # points which were detected). + # TODO: Use keypoint confidence information to change R on each sensor instead? + if detection.scores is not None: + assert len(detection.scores.shape) == 1 + points_over_threshold_mask = detection.scores > self.detection_threshold + matched_sensors_mask = np.array( + [(m,) * self.dim_points for m in points_over_threshold_mask] + ).flatten() + H_pos = np.diag(matched_sensors_mask).astype( + float + ) # We measure x, y positions + self.point_hit_counter[points_over_threshold_mask] += 2 * period + else: + points_over_threshold_mask = np.array([True] * self.num_points) + H_pos = np.identity(self.num_points * self.dim_points) + self.point_hit_counter += 2 * period + self.point_hit_counter[ + self.point_hit_counter >= self.pointwise_hit_counter_max + ] = self.pointwise_hit_counter_max + self.point_hit_counter[self.point_hit_counter < 0] = 0 + H_vel = np.zeros(H_pos.shape) # But we don't directly measure velocity + H = np.hstack([H_pos, H_vel]) + self.filter.update( + np.expand_dims(detection.absolute_points.flatten(), 0).T, None, H + ) + + detected_at_least_once_mask = np.array( + [(m,) * self.dim_points for m in self.detected_at_least_once_points] + ).flatten() + now_detected_mask = np.hstack( + (points_over_threshold_mask,) * self.dim_points + ).flatten() + first_detection_mask = np.logical_and( + now_detected_mask, np.logical_not(detected_at_least_once_mask) + ) + + self.filter.x[: self.dim_z][first_detection_mask] = np.expand_dims( + detection.absolute_points.flatten(), 0 + ).T[first_detection_mask] + + # Force points being detected for the first time to have velocity = 0 + # This is needed because some detectors (like OpenPose) set points with + # low confidence to coordinates (0, 0). And when they then get their first + # real detection this creates a huge velocity vector in our KalmanFilter + # and causes the tracker to start with wildly inaccurate estimations which + # eventually coverge to the real detections. + self.filter.x[self.dim_z :][np.logical_not(detected_at_least_once_mask)] = 0 + self.detected_at_least_once_points = np.logical_or( + self.detected_at_least_once_points, points_over_threshold_mask + ) + + def __repr__(self): + if self.last_distance is None: + placeholder_text = "\033[1mObject_{}\033[0m(age: {}, hit_counter: {}, last_distance: {}, init_id: {})" + else: + placeholder_text = "\033[1mObject_{}\033[0m(age: {}, hit_counter: {}, last_distance: {:.2f}, init_id: {})" + return placeholder_text.format( + self.id, + self.age, + self.hit_counter, + self.last_distance, + self.initializing_id, + ) + + def _conditionally_add_to_past_detections(self, detection): + """Adds detections into (and pops detections away) from `past_detections` + + It does so by keeping a fixed amount of past detections saved into each + TrackedObject, while maintaining them distributed uniformly through the object's + lifetime. + """ + if self.past_detections_length == 0: + return + if len(self.past_detections) < self.past_detections_length: + detection.age = self.age + self.past_detections.append(detection) + elif self.age >= self.past_detections[0].age * self.past_detections_length: + self.past_detections.pop(0) + detection.age = self.age + self.past_detections.append(detection) + + def merge(self, tracked_object): + """Merge with a not yet initialized TrackedObject instance""" + self.reid_hit_counter = None + self.hit_counter = self.initial_period * 2 + self.point_hit_counter = tracked_object.point_hit_counter + self.last_distance = tracked_object.last_distance + self.current_min_distance = tracked_object.current_min_distance + self.last_detection = tracked_object.last_detection + self.detected_at_least_once_points = ( + tracked_object.detected_at_least_once_points + ) + self.filter = tracked_object.filter + + for past_detection in tracked_object.past_detections: + self._conditionally_add_to_past_detections(past_detection) + + def update_coordinate_transformation( + self, coordinate_transformation: CoordinatesTransformation + ): + if coordinate_transformation is not None: + self.abs_to_rel = coordinate_transformation.abs_to_rel + + def _acquire_ids(self): + self.id, self.global_id = self._obj_factory.get_ids() + + +class Detection: + """Detections returned by the detector must be converted to a `Detection` object before being used by Norfair. + + Parameters + ---------- + points : np.ndarray + Points detected. Must be a rank 2 array with shape `(n_points, n_dimensions)` where n_dimensions is 2 or 3. + scores : np.ndarray, optional + An array of length `n_points` which assigns a score to each of the points defined in `points`. + + This is used to inform the tracker of which points to ignore; + any point with a score below `detection_threshold` will be ignored. + + This useful for cases in which detections don't always have every point present, as is often the case in pose estimators. + data : Any, optional + The place to store any extra data which may be useful when calculating the distance function. + Anything stored here will be available to use inside the distance function. + + This enables the development of more interesting trackers which can do things like assign an appearance embedding to each + detection to aid in its tracking. + label : Hashable, optional + When working with multiple classes the detection's label can be stored to be used as a matching condition when associating + tracked objects with new detections. Label's type must be hashable for drawing purposes. + embedding : Any, optional + The embedding for the reid_distance. + """ + + def __init__( + self, + points: np.ndarray, + scores: np.ndarray = None, + data: Any = None, + label: Hashable = None, + embedding=None, + ): + self.points = validate_points(points) + self.scores = scores + self.data = data + self.label = label + self.absolute_points = self.points.copy() + self.embedding = embedding + self.age = None + + def update_coordinate_transformation( + self, coordinate_transformation: CoordinatesTransformation + ): + if coordinate_transformation is not None: + self.absolute_points = coordinate_transformation.rel_to_abs( + self.absolute_points + ) diff --git a/evadb/functions/norfair/utils.py b/evadb/functions/norfair/utils.py new file mode 100644 index 0000000000..d04b6227e0 --- /dev/null +++ b/evadb/functions/norfair/utils.py @@ -0,0 +1,100 @@ +import os +from functools import lru_cache +from logging import warn +from typing import Sequence, Tuple + +import numpy as np +from rich import print +from rich.console import Console +from rich.table import Table + + +def validate_points(points: np.ndarray) -> np.array: + # If the user is tracking only a single point, reformat it slightly. + if len(points.shape) == 1: + points = points[np.newaxis, ...] + elif len(points.shape) > 2: + print_detection_error_message_and_exit(points) + return points + + +def print_detection_error_message_and_exit(points): + print("\n[red]INPUT ERROR:[/red]") + print( + f"Each `Detection` object should have a property `points` of shape (num_of_points_to_track, 2), not {points.shape}. Check your `Detection` list creation code." + ) + print("You can read the documentation for the `Detection` class here:") + print( + "https://tryolabs.github.io/norfair/reference/tracker/#norfair.tracker.Detection\n" + ) + exit() + + +def print_objects_as_table(tracked_objects: Sequence): + """Used for helping in debugging""" + print() + console = Console() + table = Table(show_header=True, header_style="bold magenta") + table.add_column("Id", style="yellow", justify="center") + table.add_column("Age", justify="right") + table.add_column("Hit Counter", justify="right") + table.add_column("Last distance", justify="right") + table.add_column("Init Id", justify="center") + for obj in tracked_objects: + table.add_row( + str(obj.id), + str(obj.age), + str(obj.hit_counter), + f"{obj.last_distance:.4f}", + str(obj.initializing_id), + ) + console.print(table) + + +def get_terminal_size(default: Tuple[int, int] = (80, 24)) -> Tuple[int, int]: + columns, lines = default + for fd in range(0, 3): # First in order 0=Std In, 1=Std Out, 2=Std Error + try: + columns, lines = os.get_terminal_size(fd) + except OSError: + continue + break + return columns, lines + + +def get_cutout(points, image): + """Returns a rectangular cut-out from a set of points on an image""" + max_x = int(max(points[:, 0])) + min_x = int(min(points[:, 0])) + max_y = int(max(points[:, 1])) + min_y = int(min(points[:, 1])) + return image[min_y:max_y, min_x:max_x] + + +class DummyOpenCVImport: + def __getattribute__(self, name): + print( + r"""[bold red]Missing dependency:[/bold red] You are trying to use Norfair's video features. However, OpenCV is not installed. + +Please, make sure there is an existing installation of OpenCV or install Norfair with `pip install norfair\[video]`.""" + ) + exit() + + +class DummyMOTMetricsImport: + def __getattribute__(self, name): + print( + r"""[bold red]Missing dependency:[/bold red] You are trying to use Norfair's metrics features without the required dependencies. + +Please, install Norfair with `pip install norfair\[metrics]`, or `pip install norfair\[metrics,video]` if you also want video features.""" + ) + exit() + + +# lru_cache will prevent re-run the function if the message is the same +@lru_cache(maxsize=None) +def warn_once(message): + """ + Write a warning message only once. + """ + warn(message) diff --git a/evadb/functions/norfair/video.py b/evadb/functions/norfair/video.py new file mode 100644 index 0000000000..0f3cd3c3da --- /dev/null +++ b/evadb/functions/norfair/video.py @@ -0,0 +1,361 @@ +import os +import time +from typing import List, Optional, Union + +try: + import cv2 +except ImportError: + from .utils import DummyOpenCVImport + + cv2 = DummyOpenCVImport() +import numpy as np +from rich import print +from rich.progress import BarColumn, Progress, ProgressColumn, TimeRemainingColumn + +from norfair import metrics + +from .utils import get_terminal_size + + +class Video: + """ + Class that provides a simple and pythonic way to interact with video. + + It returns regular OpenCV frames which enables the usage of the huge number of tools OpenCV provides to modify images. + + Parameters + ---------- + camera : Optional[int], optional + An integer representing the device id of the camera to be used as the video source. + + Webcams tend to have an id of `0`. Arguments `camera` and `input_path` can't be used at the same time, one must be chosen. + input_path : Optional[str], optional + A string consisting of the path to the video file to be used as the video source. + + Arguments `camera` and `input_path` can't be used at the same time, one must be chosen. + output_path : str, optional + The path to the output video to be generated. + Can be a folder were the file will be created or a full path with a file name. + output_fps : Optional[float], optional + The frames per second at which to encode the output video file. + + If not provided it is set to be equal to the input video source's fps. + This argument is useful when using live video cameras as a video source, + where the user may know the input fps, + but where the frames are being fed to the output video at a rate that is lower than the video source's fps, + due to the latency added by the detector. + label : str, optional + Label to add to the progress bar that appears when processing the current video. + output_fourcc : Optional[str], optional + OpenCV encoding for output video file. + By default we use `mp4v` for `.mp4` and `XVID` for `.avi`. This is a combination that works on most systems but + it results in larger files. To get smaller files use `avc1` or `H264` if available. + Notice that some fourcc are not compatible with some extensions. + output_extension : str, optional + File extension used for the output video. Ignored if `output_path` is not a folder. + + Examples + -------- + >>> video = Video(input_path="video.mp4") + >>> for frame in video: + >>> # << Your modifications to the frame would go here >> + >>> video.write(frame) + """ + + def __init__( + self, + camera: Optional[int] = None, + input_path: Optional[str] = None, + output_path: str = ".", + output_fps: Optional[float] = None, + label: str = "", + output_fourcc: Optional[str] = None, + output_extension: str = "mp4", + ): + self.camera = camera + self.input_path = input_path + self.output_path = output_path + self.label = label + self.output_fourcc = output_fourcc + self.output_extension = output_extension + self.output_video: Optional[cv2.VideoWriter] = None + + # Input validation + if (input_path is None and camera is None) or ( + input_path is not None and camera is not None + ): + raise ValueError( + "You must set either 'camera' or 'input_path' arguments when setting 'Video' class" + ) + if camera is not None and type(camera) is not int: + raise ValueError( + "Argument 'camera' refers to the device-id of your camera, and must be an int. Setting it to 0 usually works if you don't know the id." + ) + + # Read Input Video + if self.input_path is not None: + if "~" in self.input_path: + self.input_path = os.path.expanduser(self.input_path) + if not os.path.isfile(self.input_path): + self._fail( + f"[bold red]Error:[/bold red] File '{self.input_path}' does not exist." + ) + self.video_capture = cv2.VideoCapture(self.input_path) + total_frames = int(self.video_capture.get(cv2.CAP_PROP_FRAME_COUNT)) + if total_frames == 0: + self._fail( + f"[bold red]Error:[/bold red] '{self.input_path}' does not seem to be a video file supported by OpenCV. If the video file is not the problem, please check that your OpenCV installation is working correctly." + ) + description = os.path.basename(self.input_path) + else: + self.video_capture = cv2.VideoCapture(self.camera) + total_frames = 0 + description = f"Camera({self.camera})" + self.output_fps = ( + output_fps + if output_fps is not None + else self.video_capture.get(cv2.CAP_PROP_FPS) + ) + self.input_height = self.video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT) + self.input_width = self.video_capture.get(cv2.CAP_PROP_FRAME_WIDTH) + self.frame_counter = 0 + + # Setup progressbar + if self.label: + description += f" | {self.label}" + progress_bar_fields: List[Union[str, ProgressColumn]] = [ + "[progress.description]{task.description}", + BarColumn(), + "[yellow]{task.fields[process_fps]:.2f}fps[/yellow]", + ] + if self.input_path is not None: + progress_bar_fields.insert( + 2, "[progress.percentage]{task.percentage:>3.0f}%" + ) + progress_bar_fields.insert( + 3, + TimeRemainingColumn(), + ) + self.progress_bar = Progress( + *progress_bar_fields, + auto_refresh=False, + redirect_stdout=False, + redirect_stderr=False, + ) + self.task = self.progress_bar.add_task( + self.abbreviate_description(description), + total=total_frames, + start=self.input_path is not None, + process_fps=0, + ) + + # This is a generator, note the yield keyword below. + def __iter__(self): + with self.progress_bar as progress_bar: + start = time.time() + + # Iterate over video + while True: + self.frame_counter += 1 + ret, frame = self.video_capture.read() + if ret is False or frame is None: + break + process_fps = self.frame_counter / (time.time() - start) + progress_bar.update( + self.task, advance=1, refresh=True, process_fps=process_fps + ) + yield frame + + # Cleanup + if self.output_video is not None: + self.output_video.release() + print( + f"[white]Output video file saved to: {self.get_output_file_path()}[/white]" + ) + self.video_capture.release() + cv2.destroyAllWindows() + + def _fail(self, msg: str): + print(msg) + exit() + + def write(self, frame: np.ndarray) -> int: + """ + Write one frame to the output video. + + Parameters + ---------- + frame : np.ndarray + The OpenCV frame to write to file. + + Returns + ------- + int + _description_ + """ + if self.output_video is None: + # The user may need to access the output file path on their code + output_file_path = self.get_output_file_path() + fourcc = cv2.VideoWriter_fourcc(*self.get_codec_fourcc(output_file_path)) + # Set on first frame write in case the user resizes the frame in some way + output_size = ( + frame.shape[1], + frame.shape[0], + ) # OpenCV format is (width, height) + self.output_video = cv2.VideoWriter( + output_file_path, + fourcc, + self.output_fps, + output_size, + ) + + self.output_video.write(frame) + return cv2.waitKey(1) + + def show(self, frame: np.ndarray, downsample_ratio: float = 1.0) -> int: + """ + Display a frame through a GUI. Usually used inside a video inference loop to show the output video. + + Parameters + ---------- + frame : np.ndarray + The OpenCV frame to be displayed. + downsample_ratio : float, optional + How much to downsample the frame being show. + + Useful when streaming the GUI video display through a slow internet connection using something like X11 forwarding on an ssh connection. + + Returns + ------- + int + _description_ + """ + # Resize to lower resolution for faster streaming over slow connections + if downsample_ratio != 1.0: + frame = cv2.resize( + frame, + ( + frame.shape[1] // downsample_ratio, + frame.shape[0] // downsample_ratio, + ), + ) + cv2.imshow("Output", frame) + return cv2.waitKey(1) + + def get_output_file_path(self) -> str: + """ + Calculate the output path being used in case you are writing your frames to a video file. + + Useful if you didn't set `output_path`, and want to know what the autogenerated output file path by Norfair will be. + + Returns + ------- + str + The path to the file. + """ + if not os.path.isdir(self.output_path): + return self.output_path + + if self.input_path is not None: + file_name = self.input_path.split("/")[-1].split(".")[0] + else: + file_name = "camera_{self.camera}" + file_name = f"{file_name}_out.{self.output_extension}" + + return os.path.join(self.output_path, file_name) + + def get_codec_fourcc(self, filename: str) -> Optional[str]: + if self.output_fourcc is not None: + return self.output_fourcc + + # Default codecs for each extension + extension = filename[-3:].lower() + if "avi" == extension: + return "XVID" + elif "mp4" == extension: + return "mp4v" # When available, "avc1" is better + else: + self._fail( + f"[bold red]Could not determine video codec for the provided output filename[/bold red]: " + f"[yellow]{filename}[/yellow]\n" + f"Please use '.mp4', '.avi', or provide a custom OpenCV fourcc codec name." + ) + return ( + None # Had to add this return to make mypya happy. I don't like this. + ) + + def abbreviate_description(self, description: str) -> str: + """Conditionally abbreviate description so that progress bar fits in small terminals""" + terminal_columns, _ = get_terminal_size() + space_for_description = ( + int(terminal_columns) - 25 + ) # Leave 25 space for progressbar + if len(description) < space_for_description: + return description + else: + return "{} ... {}".format( + description[: space_for_description // 2 - 3], + description[-space_for_description // 2 + 3 :], + ) + + +class VideoFromFrames: + def __init__( + self, input_path, save_path=".", information_file=None, make_video=True + ): + + if information_file is None: + information_file = metrics.InformationFile( + file_path=os.path.join(input_path, "seqinfo.ini") + ) + if make_video: + file_name = os.path.split(input_path)[1] + + # Search framerate on seqinfo.ini + fps = information_file.search(variable_name="frameRate") + + # Search resolution in seqinfo.ini + horizontal_resolution = information_file.search(variable_name="imWidth") + vertical_resolution = information_file.search(variable_name="imHeight") + image_size = (horizontal_resolution, vertical_resolution) + + videos_folder = os.path.join(save_path, "videos") + if not os.path.exists(videos_folder): + os.makedirs(videos_folder) + + video_path = os.path.join(videos_folder, file_name + ".mp4") + fourcc = cv2.VideoWriter_fourcc(*"mp4v") + + self.file_name = file_name + # Video file + self.video = cv2.VideoWriter(video_path, fourcc, fps, image_size) + + self.length = information_file.search(variable_name="seqLength") + self.input_path = input_path + self.frame_number = 1 + self.image_extension = information_file.search("imExt") + self.image_directory = information_file.search("imDir") + + def __iter__(self): + self.frame_number = 1 + return self + + def __next__(self): + if self.frame_number <= self.length: + frame_path = os.path.join( + self.input_path, + self.image_directory, + str(self.frame_number).zfill(6) + self.image_extension, + ) + self.frame_number += 1 + + return cv2.imread(frame_path) + raise StopIteration() + + def update(self, frame): + self.video.write(frame) + cv2.waitKey(1) + + if self.frame_number > self.length: + cv2.destroyAllWindows() + self.video.release() diff --git a/evadb/functions/sklearn.py b/evadb/functions/sklearn.py index 1333004b09..4ab2b0abfe 100644 --- a/evadb/functions/sklearn.py +++ b/evadb/functions/sklearn.py @@ -17,7 +17,7 @@ import pandas as pd from evadb.functions.abstract.abstract_function import AbstractFunction -from evadb.utils.generic_utils import try_to_import_flaml_automl +from evadb.utils.generic_utils import try_to_import_sklearn class GenericSklearnModel(AbstractFunction): @@ -26,7 +26,7 @@ def name(self) -> str: return "GenericSklearnModel" def setup(self, model_path: str, predict_col: str, **kwargs): - try_to_import_flaml_automl() + try_to_import_sklearn() self.model = pickle.load(open(model_path, "rb")) self.predict_col = predict_col diff --git a/evadb/functions/stable_diffusion.py b/evadb/functions/stable_diffusion.py index 1ceaefe7e3..0441955476 100644 --- a/evadb/functions/stable_diffusion.py +++ b/evadb/functions/stable_diffusion.py @@ -22,6 +22,7 @@ from PIL import Image from evadb.catalog.catalog_type import NdArrayType +from evadb.configuration.configuration_manager import ConfigurationManager from evadb.functions.abstract.abstract_function import AbstractFunction from evadb.functions.decorators.decorators import forward from evadb.functions.decorators.io_descriptors.data_types import PandasDataframe @@ -33,8 +34,10 @@ class StableDiffusion(AbstractFunction): def name(self) -> str: return "StableDiffusion" - def setup(self, replicate_api_token="") -> None: - self.replicate_api_token = replicate_api_token + def setup( + self, + ) -> None: + pass @forward( input_signatures=[ @@ -61,13 +64,16 @@ def forward(self, text_df): try_to_import_replicate() import replicate - replicate_api_key = self.replicate_api_token + # Register API key, try configuration manager first + replicate_api_key = ConfigurationManager().get_value( + "third_party", "REPLICATE_API_TOKEN" + ) # If not found, try OS Environment Variable if replicate_api_key is None: replicate_api_key = os.environ.get("REPLICATE_API_TOKEN", "") assert ( len(replicate_api_key) != 0 - ), "Please set your Replicate API key using SET REPLICATE_API_TOKEN = '' or set the environment variable (REPLICATE_API_TOKEN)" + ), "Please set your Replicate API key in evadb.yml file (third_party, replicate_api_token) or environment variable (REPLICATE_API_TOKEN)" os.environ["REPLICATE_API_TOKEN"] = replicate_api_key model_id = ( diff --git a/evadb/functions/trackers/nor_fair.py b/evadb/functions/trackers/nor_fair.py index 0468a3c997..8cc3bf1561 100644 --- a/evadb/functions/trackers/nor_fair.py +++ b/evadb/functions/trackers/nor_fair.py @@ -14,6 +14,8 @@ # limitations under the License. import numpy as np +#import sys + from evadb.functions.abstract.tracker_abstract_function import ( EvaDBTrackerAbstractFunction, ) @@ -31,6 +33,8 @@ def name(self) -> str: def setup(self, distance_threshold=DISTANCE_THRESHOLD_CENTROID) -> None: # https://github.com/tryolabs/norfair/blob/74b11edde83941dd6e32bcccd5fa849e16bf8564/norfair/tracker.py#L18 try_to_import_norfair() + #sys.path.append('../norfair') + #from norfair.tracker import Tracker from norfair import Tracker self.tracker = Tracker( diff --git a/evadb/functions/xgboost.py b/evadb/functions/xgboost.py index 9705d09e4b..0635294116 100644 --- a/evadb/functions/xgboost.py +++ b/evadb/functions/xgboost.py @@ -17,7 +17,7 @@ import pandas as pd from evadb.functions.abstract.abstract_function import AbstractFunction -from evadb.utils.generic_utils import try_to_import_flaml_automl +from evadb.utils.generic_utils import try_to_import_xgboost class GenericXGBoostModel(AbstractFunction): @@ -26,7 +26,7 @@ def name(self) -> str: return "GenericXGBoostModel" def setup(self, model_path: str, predict_col: str, **kwargs): - try_to_import_flaml_automl() + try_to_import_xgboost() self.model = pickle.load(open(model_path, "rb")) self.predict_col = predict_col diff --git a/evadb/parser/create_function_statement.py b/evadb/parser/create_function_statement.py index eb35fcffaa..e8ada39984 100644 --- a/evadb/parser/create_function_statement.py +++ b/evadb/parser/create_function_statement.py @@ -15,12 +15,12 @@ from pathlib import Path from typing import List, Tuple +from evadb.configuration.constants import EvaDB_INSTALLATION_DIR from evadb.parser.create_statement import ColumnDefinition from evadb.parser.select_statement import SelectStatement from evadb.parser.statement import AbstractStatement from evadb.parser.types import StatementType - class CreateFunctionStatement(AbstractStatement): """CreateFunctionStatement constructed after parsing the input query @@ -64,7 +64,15 @@ def __init__( self._if_not_exists = if_not_exists self._inputs = inputs self._outputs = outputs - self._impl_path = Path(impl_path) if impl_path else None + if impl_path: + if "DBFUNCTIONS" == impl_path[0:11] and (impl_path[11:12] == "." or impl_path[11:12] == "\\" or impl_path[11:12] == "/"): + self._impl_path = Path(str(EvaDB_INSTALLATION_DIR) + "\\functions\\" + impl_path[12:]) + elif "ENVFUNCTIONS" == impl_path[0:12] and (impl_path[12:13] == "." or impl_path[12:13] == "\\" or impl_path[12:13] == "/"): + self._impl_path = Path( "..\\functions\\" + impl_path[13:]) + else: + self._impl_path = Path(impl_path) + else: + self._impl_path = None self._function_type = function_type self._query = query self._metadata = metadata diff --git a/evadb/parser/create_statement.py b/evadb/parser/create_statement.py index 89aee64cfa..ca79a9eac4 100644 --- a/evadb/parser/create_statement.py +++ b/evadb/parser/create_statement.py @@ -12,8 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from dataclasses import dataclass -from typing import List, Optional, Tuple +from typing import List, Tuple from evadb.catalog.catalog_type import ColumnType, NdArrayType from evadb.parser.select_statement import SelectStatement @@ -227,44 +226,3 @@ def __str__(self) -> str: f"WITH ENGINE '{self.engine}' , \n" f"PARAMETERS = {self.param_dict};" ) - - -@dataclass -class CreateJobStatement(AbstractStatement): - job_name: str - queries: list - if_not_exists: bool - start_time: Optional[str] = None - end_time: Optional[str] = None - repeat_interval: Optional[int] = None - repeat_period: Optional[str] = None - - def __hash__(self): - return hash( - ( - super().__hash__(), - self.job_name, - tuple(self.queries), - self.start_time, - self.end_time, - self.repeat_interval, - self.repeat_period, - ) - ) - - def __post_init__(self): - super().__init__(StatementType.CREATE_JOB) - - def __str__(self): - start_str = f"\nSTART {self.start_time}" if self.start_time is not None else "" - end_str = f"\nEND {self.end_time}" if self.end_time is not None else "" - repeat_str = ( - f"\nEVERY {self.repeat_interval} {self.repeat_period}" - if self.repeat_interval is not None - else "" - ) - return ( - f"CREATE JOB {self.job_name} AS\n" - f"({(str(q) for q in self.queries)})" - f"{start_str} {end_str} {repeat_str}" - ) diff --git a/evadb/parser/evadb.lark b/evadb/parser/evadb.lark index 86798df6c0..e834d1a7d0 100644 --- a/evadb/parser/evadb.lark +++ b/evadb/parser/evadb.lark @@ -1,15 +1,11 @@ // Top Level Description -// create_job is intentionally not treated as an sql_statement to keep the parser clean -// because we assume that inside the job, the user can specify multiple sql_statements -// but not a create_job within a create_job. - -start: (sql_statement? ";")+ | (create_job ";") +start: (sql_statement? ";")+ sql_statement: ddl_statement | dml_statement | utility_statement | context_statement - -ddl_statement: create_database | create_table | create_index | create_function | drop_database - | drop_table | drop_function | drop_index | drop_job | rename_table + +ddl_statement: create_database | create_table | create_index | create_function + | drop_database | drop_table | drop_function | drop_index | rename_table dml_statement: select_statement | insert_statement | update_statement | delete_statement | load_statement | set_statement @@ -18,9 +14,6 @@ utility_statement: describe_statement | show_statement | help_statement | explai context_statement: use_statement -job_sql_statements: query_string (";" query_string)* ";"? - - // Data Definition Language // Create statements @@ -36,18 +29,7 @@ create_database_engine_clause: WITH ENGINE "=" string_literal "," PARAMETERS "=" create_index: CREATE INDEX if_not_exists? uid ON table_name index_elem vector_store_type? create_table: CREATE TABLE if_not_exists? table_name (create_definitions | (AS select_statement)) - -create_job: CREATE JOB if_not_exists? uid AS "{" job_sql_statements "}" (start_time)? (end_time)? (repeat_clause)? - -start_time: START string_literal - -end_time: END string_literal - -repeat_clause: EVERY decimal_literal simple_id - - - - + // Rename statements rename_table: RENAME TABLE table_name TO table_name @@ -71,7 +53,7 @@ function_metadata_key: uid function_metadata_value: constant -vector_store_type: USING (FAISS | QDRANT | PINECONE | PGVECTOR | CHROMADB | MILVUS) +vector_store_type: USING (FAISS | QDRANT | PINECONE | PGVECTOR | CHROMADB) index_elem: ("(" uid_list ")" | "(" function_call ")") @@ -96,15 +78,13 @@ drop_index: DROP INDEX if_exists? uid drop_table: DROP TABLE if_exists? uid drop_function: DROP FUNCTION if_exists? uid - -drop_job: DROP JOB if_exists? uid // SET statements (configuration) set_statement: SET config_name (EQUAL_SYMBOL | TO) config_value config_name: uid -config_value: constant +config_value: (string_literal | decimal_literal | boolean_literal | real_literal) // Data Manipulation Language @@ -368,9 +348,7 @@ DESC: "DESC"i DESCRIBE: "DESCRIBE"i DISTINCT: "DISTINCT"i DROP: "DROP"i -END: "END"i ENGINE: "ENGINE"i -EVERY: "EVERY"i EXIT: "EXIT"i EXISTS: "EXISTS"i EXPLAIN: "EXPLAIN"i @@ -386,7 +364,6 @@ INTO: "INTO"i INDEX: "INDEX"i INSERT: "INSERT"i IS: "IS"i -JOB: "JOB"i JOIN: "JOIN"i KEY: "KEY"i LATERAL: "LATERAL"i @@ -415,7 +392,6 @@ SET: "SET"i SHUTDOWN: "SHUTDOWN"i SHOW: "SHOW"i SOME: "SOME"i -START: "START"i TABLE: "TABLE"i TABLES: "TABLES"i TO: "TO"i @@ -448,7 +424,6 @@ QDRANT: "QDRANT"i PINECONE: "PINECONE"i PGVECTOR: "PGVECTOR"i CHROMADB: "CHROMADB"i -MILVUS: "MILVUS"i // Computer vision tasks @@ -591,6 +566,7 @@ REAL_LITERAL: (DEC_DIGIT+)? "." DEC_DIGIT+ DOT_ID: "." ID_LITERAL + // Identifiers ID: ID_LITERAL @@ -603,8 +579,8 @@ GLOBAL_ID: "@" "@" (/[A-Z0-9._$]+/ | BQUOTA_STRING) EXPONENT_NUM_PART: /"E" "-"? DEC_DIGIT+/ ID_LITERAL: /[A-Za-z_$0-9]*?[A-Za-z_$]+?[A-Za-z_$0-9]*/ -DQUOTA_STRING: "\"" /(?:[^"\\]|\\.)*"/ -SQUOTA_STRING: "'" /(?:[^'\\]|\\.)*'/ +DQUOTA_STRING: /"[^";]*"/ +SQUOTA_STRING: /'[^';]*'/ BQUOTA_STRING: /`[^'`]*`/ QUERY_STRING: /[^{};]+/ DEC_DIGIT: /[0-9]/ diff --git a/evadb/parser/lark_visitor/__init__.py b/evadb/parser/lark_visitor/__init__.py index 911e886a2d..9ed0a1b6fe 100644 --- a/evadb/parser/lark_visitor/__init__.py +++ b/evadb/parser/lark_visitor/__init__.py @@ -20,7 +20,6 @@ from evadb.parser.lark_visitor._create_statements import ( CreateDatabase, CreateIndex, - CreateJob, CreateTable, ) from evadb.parser.lark_visitor._delete_statement import Delete @@ -67,7 +66,6 @@ class LarkInterpreter( CreateTable, CreateIndex, CreateDatabase, - CreateJob, Expressions, Functions, Insert, @@ -91,11 +89,3 @@ def start(self, tree): def sql_statement(self, tree): return self.visit(tree.children[0]) - - def job_sql_statements(self, tree): - sql_statements = [] - for child in tree.children: - if isinstance(child, Tree): - if child.data == "query_string": - sql_statements.append(self.visit(child)) - return sql_statements diff --git a/evadb/parser/lark_visitor/_create_statements.py b/evadb/parser/lark_visitor/_create_statements.py index 72066b294c..18e13ca3fc 100644 --- a/evadb/parser/lark_visitor/_create_statements.py +++ b/evadb/parser/lark_visitor/_create_statements.py @@ -22,7 +22,6 @@ ColConstraintInfo, ColumnDefinition, CreateDatabaseStatement, - CreateJobStatement, CreateTableStatement, ) from evadb.parser.table_ref import TableRef @@ -300,8 +299,6 @@ def vector_store_type(self, tree): vector_store_type = VectorStoreType.PGVECTOR elif str.upper(token) == "CHROMADB": vector_store_type = VectorStoreType.CHROMADB - elif str.upper(token) == "MILVUS": - vector_store_type = VectorStoreType.MILVUS return vector_store_type @@ -337,49 +334,3 @@ def create_database_engine_clause(self, tree): param_dict = self.visit(child) return engine, param_dict - - -class CreateJob: - def create_job(self, tree): - job_name = None - queries = [] - start_time = None - end_time = None - repeat_interval = None - repeat_period = None - if_not_exists = False - for child in tree.children: - if isinstance(child, Tree): - if child.data == "if_not_exists": - if_not_exists = True - if child.data == "uid": - job_name = self.visit(child) - if child.data == "job_sql_statements": - queries = self.visit(child) - elif child.data == "start_time": - start_time = self.visit(child) - elif child.data == "end_time": - end_time = self.visit(child) - elif child.data == "repeat_clause": - repeat_interval, repeat_period = self.visit(child) - - create_job = CreateJobStatement( - job_name, - queries, - if_not_exists, - start_time, - end_time, - repeat_interval, - repeat_period, - ) - - return create_job - - def start_time(self, tree): - return self.visit(tree.children[1]).value - - def end_time(self, tree): - return self.visit(tree.children[1]).value - - def repeat_clause(self, tree): - return self.visit(tree.children[1]), self.visit(tree.children[2]) diff --git a/evadb/parser/lark_visitor/_drop_statement.py b/evadb/parser/lark_visitor/_drop_statement.py index 7fc96298ed..0b397378ae 100644 --- a/evadb/parser/lark_visitor/_drop_statement.py +++ b/evadb/parser/lark_visitor/_drop_statement.py @@ -73,17 +73,3 @@ def drop_database(self, tree): database_name = self.visit(child) return DropObjectStatement(ObjectType.DATABASE, database_name, if_exists) - - # Drop Job - def drop_job(self, tree): - job_name = None - if_exists = False - - for child in tree.children: - if isinstance(child, Tree): - if child.data == "if_exists": - if_exists = True - elif child.data == "uid": - job_name = self.visit(child) - - return DropObjectStatement(ObjectType.JOB, job_name, if_exists) diff --git a/evadb/parser/lark_visitor/_expressions.py b/evadb/parser/lark_visitor/_expressions.py index ff53ed4e1a..6ec01cf991 100644 --- a/evadb/parser/lark_visitor/_expressions.py +++ b/evadb/parser/lark_visitor/_expressions.py @@ -20,7 +20,6 @@ from evadb.expression.comparison_expression import ComparisonExpression from evadb.expression.constant_value_expression import ConstantValueExpression from evadb.expression.logical_expression import LogicalExpression -from evadb.utils.generic_utils import string_comparison_case_insensitive ################################################################## @@ -102,12 +101,10 @@ def comparison_operator(self, tree): def logical_operator(self, tree): op = str(tree.children[0]) - if string_comparison_case_insensitive(op, "OR"): + if op == "OR": return ExpressionType.LOGICAL_OR - elif string_comparison_case_insensitive(op, "AND"): + elif op == "AND": return ExpressionType.LOGICAL_AND - else: - raise NotImplementedError("Unsupported logical operator: {}".format(op)) def expressions_with_defaults(self, tree): expr_list = [] diff --git a/evadb/parser/select_statement.py b/evadb/parser/select_statement.py index 69270a6b84..b04c7148bb 100644 --- a/evadb/parser/select_statement.py +++ b/evadb/parser/select_statement.py @@ -141,11 +141,7 @@ def __str__(self) -> str: orderby_list_str += str(expr[0]) + " " + sort_str + ", " orderby_list_str = orderby_list_str.rstrip(", ") - select_str = f"SELECT {target_list_str}" - - if self._from_table is not None: - select_str += " FROM " + str(self._from_table) - + select_str = f"SELECT {target_list_str} FROM {str(self._from_table)}" if self._where_clause is not None: select_str += " WHERE " + str(self._where_clause) diff --git a/evadb/parser/types.py b/evadb/parser/types.py index 227a768c7b..751d2b5f31 100644 --- a/evadb/parser/types.py +++ b/evadb/parser/types.py @@ -42,7 +42,6 @@ class StatementType(EvaDBEnum): CREATE_DATABASE # noqa: F821 USE # noqa: F821 SET # noqa: F821 - CREATE_JOB # noqa: F821 # add other types @@ -84,4 +83,3 @@ class ObjectType(EvaDBEnum): FUNCTION # noqa: F821 INDEX # noqa: F821 DATABASE # noqa: F821 - JOB # noqa: F821 diff --git a/evadb/parser/utils.py b/evadb/parser/utils.py index dd4567cf4b..a2be06ec16 100644 --- a/evadb/parser/utils.py +++ b/evadb/parser/utils.py @@ -13,11 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from evadb.parser.create_function_statement import CreateFunctionStatement -from evadb.parser.create_statement import ( - CreateDatabaseStatement, - CreateJobStatement, - CreateTableStatement, -) +from evadb.parser.create_statement import CreateDatabaseStatement, CreateTableStatement from evadb.parser.drop_object_statement import DropObjectStatement from evadb.parser.explain_statement import ExplainStatement from evadb.parser.insert_statement import InsertTableStatement @@ -34,7 +30,6 @@ # directly to the executor. SKIP_BINDER_AND_OPTIMIZER_STATEMENTS = ( CreateDatabaseStatement, - CreateJobStatement, UseStatement, SetStatement, )