diff --git a/.gitignore b/.gitignore index 9736baa80a3f..8300e608bd71 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,7 @@ cvat-core/reports # produced by cvat/apps/iam/rules/tests/generate_tests.py /cvat/apps/*/rules/*_test.gen.rego + +# Custom +rsa +rsa.pub \ No newline at end of file diff --git a/cvat/apps/engine/chunks.py b/cvat/apps/engine/chunks.py new file mode 100755 index 000000000000..230331230cab --- /dev/null +++ b/cvat/apps/engine/chunks.py @@ -0,0 +1,95 @@ +import os +import traceback +import subprocess + + +def get_video_duration(video_file): + result = subprocess.run( + ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', video_file], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + duration = float(result.stdout) + return duration + + +class MakeVideoChunks: + def make(task_id, chunk_duration=1): + try: + current_file_path = os.path.abspath(__file__) + print(f"Current file path: {current_file_path}") + + # Define the raw video directory + raw_video_dir = f"/home/vignesh/Desktop/Desktop/IIITD/BTP.02/cvat/data/data/{task_id}/raw" + print(f"Raw video directory: {raw_video_dir}") + + # Recursively search for .mp4 files in the raw video directory and its subdirectories + input_files = [] + for root, dirs, files in os.walk(raw_video_dir): + for file in files: + if file.endswith('.mp4'): + input_files.append(os.path.join(root, file)) + + # Check if any .mp4 files are found + if not input_files: + raise FileNotFoundError("No .mp4 files found in the specified directory or subdirectories.") + + print(f"Input files: {input_files}") + input_file = input_files[0] # Use the first .mp4 file found + output_folder = f"/home/vignesh/Desktop/Desktop/IIITD/BTP.02/cvat/data/data/{task_id}/compressed" + + # Create the output folder if it doesn't exist + os.makedirs(output_folder, exist_ok=True) + + print(f"Processing video: {input_file}") + + # Retrieve video duration + video_duration = get_video_duration(input_file) + print(f"Video duration: {video_duration} seconds") + + # Define start and end times + start_time = 0 # Start from the beginning of the video + end_time = int(video_duration) # Set end time to the duration of the video + + # Create chunks using a loop + for i in range(start_time, end_time, chunk_duration): + output_file = os.path.join(output_folder, f'{i}.mp4') + + # If the output file exists, remove it + if os.path.exists(output_file): + print(f"File {output_file} already exists. Removing it.") + os.remove(output_file) + + command = [ + 'ffmpeg', + '-ss', str(i), # Start time for the chunk + '-i', input_file, # Input file + '-c', 'copy', # Copy codec, no re-encoding + '-t', str(chunk_duration), # Duration of the chunk + output_file # Output file path + ] + + # Execute the command + print(' '.join(command)) + subprocess.run(command) + + response = { + "success": True, + "message": None, + "data": None, + "error": None + } + + return response + except Exception as e: + print(str(e)) + error = traceback.print_exc() + + response = { + "success": False, + "message": f"An unexpected error occurred, Error: {e}", + "data": None, + "error": error + } + + return response \ No newline at end of file diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index 12c94768afa3..e99718e34ab8 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -25,7 +25,6 @@ from cvat.apps.engine.utils import parse_specific_attributes from cvat.apps.events.utils import cache_deleted - class SafeCharField(models.CharField): def get_prep_value(self, value): value = super().get_prep_value(value) diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 4fe4c4b46039..ba228955c4dc 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -39,6 +39,7 @@ from utils.dataset_manifest.core import VideoManifestValidator, is_dataset_manifest from utils.dataset_manifest.utils import detect_related_images from .cloud_provider import db_storage_to_storage_instance +from .chunks import MakeVideoChunks slogger = ServerLogManager(__name__) @@ -105,6 +106,7 @@ def _copy_data_from_share_point( )) for path in filtered_server_files: + slogger.glob.info(f"Copying file: {path}") if server_dir is None: source_path = os.path.join(settings.SHARE_ROOT, os.path.normpath(path)) else: @@ -449,8 +451,10 @@ def _download_data_from_cloud_storage( files: List[str], upload_dir: str, ): + slogger.glob.info(f"Downloading data from cloud storage: {files}") cloud_storage_instance = db_storage_to_storage_instance(db_storage) cloud_storage_instance.bulk_download_to_dir(files, upload_dir) + slogger.glob.info(f"Downloaded data to {upload_dir}") def _get_manifest_frame_indexer(start_frame=0, frame_step=1): return lambda frame_id: start_frame + frame_id * frame_step @@ -559,6 +563,7 @@ def _create_thread( slogger.glob.info("create task #{}".format(db_task.id)) job_file_mapping = _validate_job_file_mapping(db_task, data) + slogger.glob.info(f"Job file mapping: {job_file_mapping}") db_data = db_task.data upload_dir = db_data.get_upload_dirname() if db_data.storage != models.StorageChoice.SHARE else settings.SHARE_ROOT @@ -700,24 +705,29 @@ def _update_status(msg: str) -> None: # count and validate uploaded files media = _count_files(data) + slogger.glob.info(f"Media: {media}") media, task_mode = _validate_data(media, manifest_files) is_media_sorted = False if is_data_in_cloud: # first we need to filter files and keep only supported ones + slogger.glob.info(f"Data in cloud") if any([v for k, v in media.items() if k != 'image']) and db_data.storage_method == models.StorageMethodChoice.CACHE: + slogger.glob.info(f"Storage method: {db_data.storage_method}") # FUTURE-FIXME: This is a temporary workaround for creating tasks # with unsupported cloud storage data (video, archive, pdf) when use_cache is enabled db_data.storage_method = models.StorageMethodChoice.FILE_SYSTEM - _update_status("The 'use cache' option is ignored") + # _update_status("The 'use cache' option is ignored") if db_data.storage_method == models.StorageMethodChoice.FILE_SYSTEM or not settings.USE_CACHE: + slogger.glob.info(f"Storage method: {db_data.storage_method}") filtered_data = [] for files in (i for i in media.values() if i): filtered_data.extend(files) media_to_download = filtered_data - if media['image']: + if 'image' in media and media['image']: + slogger.glob.info(f"Image in media") start_frame = db_data.start_frame stop_frame = len(filtered_data) - 1 if data['stop_frame'] is not None: @@ -726,40 +736,62 @@ def _update_status(msg: str) -> None: step = db_data.get_frame_step() if start_frame or step != 1 or stop_frame != len(filtered_data) - 1: media_to_download = filtered_data[start_frame : stop_frame + 1: step] + + slogger.glob.info(f"Downloading data from cloud storage: {media_to_download}") _download_data_from_cloud_storage(db_data.cloud_storage, media_to_download, upload_dir) del media_to_download del filtered_data is_data_in_cloud = False db_data.storage = models.StorageChoice.LOCAL + slogger.glob.info(f"DB Data Storage: {db_data.storage}") else: manifest = ImageManifestManager(db_data.get_manifest_path()) if job_file_mapping is not None and task_mode != 'annotation': raise ValidationError("job_file_mapping can't be used with sequence-based data like videos") + slogger.glob.info(f"Data: {data}") if data['server_files']: if db_data.storage == models.StorageChoice.LOCAL and not db_data.cloud_storage: # this means that the data has not been downloaded from the storage to the host + slogger.glob.info(f"Copying data from share point") _copy_data_from_share_point( (data['server_files'] + [manifest_file]) if manifest_file else data['server_files'], upload_dir, data.get('server_files_path'), data.get('server_files_exclude')) manifest_root = upload_dir + slogger.glob.info(f"Manifest Root: {manifest_root}") elif is_data_in_cloud: # we should sort media before sorting in the extractor because the manifest structure should match to the sorted media if job_file_mapping is not None: + slogger.glob.info(f"Job file mapping") + filtered_files = [] + for f in itertools.chain.from_iterable(job_file_mapping): + if f not in data['server_files']: + raise ValidationError(f"Job mapping file {f} is not specified in input files") + filtered_files.append(f) + data['server_files'] = filtered_files sorted_media = list(itertools.chain.from_iterable(job_file_mapping)) else: + slogger.glob.info(f"Sorting media") sorted_media = sort(media['image'], data['sorting_method']) media['image'] = sorted_media + + # Add logic to handle audio files from cloud storage + if db_data.storage == models.StorageChoice.CLOUD_STORAGE: + slogger.glob.info(f"Downloading data from cloud storage: {data['server_files']}") + _download_data_from_cloud_storage(db_data.cloud_storage, data['server_files'], upload_dir) + is_media_sorted = True if manifest_file: # Define task manifest content based on cloud storage manifest content and uploaded files + slogger.glob.info(f"Creating task manifest based on cloud storage manifest content and uploaded files") _create_task_manifest_based_on_cloud_storage_manifest( sorted_media, cloud_storage_manifest_prefix, cloud_storage_manifest, manifest) else: # without manifest file but with use_cache option # Define task manifest content based on list with uploaded files + slogger.glob.info(f"Creating task manifest from cloud data: {db_data.cloud_storage, sorted_media, manifest}") _create_task_manifest_from_cloud_data(db_data.cloud_storage, sorted_media, manifest) av_scan_paths(upload_dir) @@ -770,6 +802,7 @@ def _update_status(msg: str) -> None: # If upload from server_files image and directories # need to update images list by all found images in directories if (data['server_files']) and len(media['directory']) and len(media['image']): + slogger.glob.info(f"Updating images list by all found images in directories: {media['directory']}") media['image'].extend( [os.path.relpath(image, upload_dir) for image in MEDIA_TYPES['directory']['extractor']( @@ -1264,3 +1297,24 @@ def process_results(img_meta: list[tuple[str, int, tuple[int, int]]]): slogger.glob.info("Found frames {} for Data #{}".format(db_data.size, db_data.id)) _save_task_to_db(db_task, job_file_mapping=job_file_mapping) + + if MEDIA_TYPE == "video": + # Video Chunks overwrites + slogger.glob.info(f"Creating video chunks") + job_id_string = job.id + match = re.search(r'task-(\d+)', job_id_string) + + if match: + task_id = match.group(1) # Extracted '106' + response = MakeVideoChunks.make(task_id) + slogger.glob.info(response) + else: + response = { + "success" : False, + "message" : "No match found." + } + slogger.glob.error(response) + + # f = open( '/home/vignesh/Desktop/Desktop/IIITD/BTP.02/cvat/cvat/apps/engine/chunks.txt', 'w' ) + # f.write( 'dict = ' + repr(response) + '\n' ) + # f.close() \ No newline at end of file diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index 7a42f2986332..8f6d788f025c 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -2218,6 +2218,17 @@ def save_segments(self, request): job.save() self.send_annotation_email(request, 'annotation') + + ## Notification + from ..notifications.api import SendNotificationToSingleUser + + notification_response = SendNotificationToSingleUser( + request.user.id, + f"#{job.id} - Annotaion Completed", + f"This annotation was completed at {datetime.now()}. \nStatus: {job.ai_audio_annotation_status}", + "info" + ) + return Response({'success': True, 'segments': saved_segments}, status=status.HTTP_201_CREATED) except Exception as e: diff --git a/cvat/apps/notifications/__init__.py b/cvat/apps/notifications/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cvat/apps/notifications/admin.py b/cvat/apps/notifications/admin.py new file mode 100644 index 000000000000..eb02673b112b --- /dev/null +++ b/cvat/apps/notifications/admin.py @@ -0,0 +1,18 @@ +from django.contrib import admin + +from .models import * +# Register your models here. + + +class NotificationsAdmin(admin.ModelAdmin): + model = Notifications + + +admin.site.register(Notifications, NotificationsAdmin) + + +class NotificationStatusAdmin(admin.ModelAdmin): + model = NotificationStatus + + +admin.site.register(NotificationStatus, NotificationStatusAdmin) \ No newline at end of file diff --git a/cvat/apps/notifications/api.py b/cvat/apps/notifications/api.py new file mode 100644 index 000000000000..921f50ea3689 --- /dev/null +++ b/cvat/apps/notifications/api.py @@ -0,0 +1,126 @@ +## Send Notification +from ..notifications.views import NotificationsViewSet +from ..notifications.serializers import (SendNotificationSerializer, FetchUserNotificationsSerializer, MarkNotificationAsViewedSerializer) + + +# Send notification to specified user +def SendNotificationToSingleUser(user_id, title, message, noti_type): + viewset = NotificationsViewSet() + send_notification_data_user = { + "user" : f"{user_id}", + "title" : f"{title}", + "message" : f"{message}", + "notification_type" : f"{noti_type}", + } + + send_notification_serializer_user = SendNotificationSerializer( + data = send_notification_data_user + ) + + if send_notification_serializer_user.is_valid(): + response = viewset.SendNotification( + request = type( + 'Request', + ( + object, + ), + { + 'data': send_notification_serializer_user.validated_data + } + ) + ) + + return response + + return None + + +# Send notification to all the users of specified organizations +def SendNotificationToOrganisationUsers(org_id, title, message, noti_type): + viewset = NotificationsViewSet() + send_notification_data_org = { + "org" : f"{org_id}", + "title" : f"{title}", + "message" : f"{message}", + "notification_type" : f"{noti_type}", + } + + send_notification_serializer_org = SendNotificationSerializer( + data = send_notification_data_org + ) + + if send_notification_serializer_org.is_valid(): + response = viewset.SendNotification( + request = type( + 'Request', + ( + object, + ), + { + 'data': send_notification_serializer_org.validated_data + } + ) + ) + + return response + + return None + + +# Fetch all Notifications of the specified user +def FetchUserNotifications(user_id, current_page, items_per_page): + viewset = NotificationsViewSet() + fetch_user_notifications_data = { + "user": user_id, + "current_page" : current_page, + "items_per_page" : items_per_page + } + fetch_user_notifications_serializer = FetchUserNotificationsSerializer( + data = fetch_user_notifications_data + ) + + if fetch_user_notifications_serializer.is_valid(): + response = viewset.FetchUserNotifications( + request = type( + 'Request', + ( + object, + ), + { + 'data' : fetch_user_notifications_serializer.validated_data + } + ) + ) + + return response + + return None + + +# Mark user notification(s) as read +def MarkUserNotificationsAsRead(user_id, notification_ids = []): + viewset = NotificationsViewSet() + mark_notification_as_viewed_data = { + "user": user_id, + "notification_ids": notification_ids + } + mark_notification_as_viewed_serializer = MarkNotificationAsViewedSerializer( + data = mark_notification_as_viewed_data + ) + + if mark_notification_as_viewed_serializer.is_valid(): + response = viewset.MarkNotificationAsViewed( + request = type( + 'Request', + ( + object, + ), + { + 'data' : mark_notification_as_viewed_serializer.validated_data + } + ) + ) + + return response + + return None \ No newline at end of file diff --git a/cvat/apps/notifications/apps.py b/cvat/apps/notifications/apps.py new file mode 100644 index 000000000000..6d66d17830b5 --- /dev/null +++ b/cvat/apps/notifications/apps.py @@ -0,0 +1,9 @@ +from django.apps import AppConfig + + +class NotificationsConfig(AppConfig): + name = 'cvat.apps.notifications' + + def ready(self) -> None: + from cvat.apps.iam.permissions import load_app_permissions + load_app_permissions(self) diff --git a/cvat/apps/notifications/migrations/0001_initial.py b/cvat/apps/notifications/migrations/0001_initial.py new file mode 100644 index 000000000000..727b8f27ccea --- /dev/null +++ b/cvat/apps/notifications/migrations/0001_initial.py @@ -0,0 +1,54 @@ +# Generated by Django 4.2.14 on 2024-09-01 08:23 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="Notifications", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("title", models.CharField(max_length=255)), + ("message", models.TextField()), + ("extra_data", models.JSONField(blank=True, null=True)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("is_read", models.BooleanField(default=False)), + ("read_at", models.DateTimeField(blank=True, null=True)), + ( + "notification_type", + models.CharField( + choices=[ + ("info", "Info"), + ("warning", "Warning"), + ("success", "Success"), + ("error", "Error"), + ], + max_length=50, + ), + ), + ( + "recipient", + models.ManyToManyField( + related_name="notifications", to=settings.AUTH_USER_MODEL + ), + ), + ], + ), + ] diff --git a/cvat/apps/notifications/migrations/0002_alter_notifications_id.py b/cvat/apps/notifications/migrations/0002_alter_notifications_id.py new file mode 100644 index 000000000000..000fc93b1248 --- /dev/null +++ b/cvat/apps/notifications/migrations/0002_alter_notifications_id.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.14 on 2024-09-06 09:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("notifications", "0001_initial"), + ] + + operations = [ + migrations.AlterField( + model_name="notifications", + name="id", + field=models.AutoField(primary_key=True, serialize=False), + ), + ] diff --git a/cvat/apps/notifications/migrations/0003_remove_notifications_is_read_and_more.py b/cvat/apps/notifications/migrations/0003_remove_notifications_is_read_and_more.py new file mode 100644 index 000000000000..e9975f3a3b5a --- /dev/null +++ b/cvat/apps/notifications/migrations/0003_remove_notifications_is_read_and_more.py @@ -0,0 +1,61 @@ +# Generated by Django 4.2.14 on 2024-09-06 10:07 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("notifications", "0002_alter_notifications_id"), + ] + + operations = [ + migrations.RemoveField( + model_name="notifications", + name="is_read", + ), + migrations.RemoveField( + model_name="notifications", + name="read_at", + ), + migrations.RemoveField( + model_name="notifications", + name="recipient", + ), + migrations.CreateModel( + name="NotificationStatus", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("is_read", models.BooleanField(default=False)), + ("read_at", models.DateTimeField(blank=True, null=True)), + ( + "notification", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="notifications.notifications", + ), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "unique_together": {("notification", "user")}, + }, + ), + ] diff --git a/cvat/apps/notifications/migrations/__init__.py b/cvat/apps/notifications/migrations/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cvat/apps/notifications/models.py b/cvat/apps/notifications/models.py new file mode 100644 index 000000000000..9113d64d5d8f --- /dev/null +++ b/cvat/apps/notifications/models.py @@ -0,0 +1,35 @@ +from django.db.models import * + +from django.contrib.auth.models import User +from ..organizations.models import * +# Create your models here. + + +class Notifications(Model): + id = AutoField(primary_key=True) + title = CharField(max_length=255) + message = TextField() + extra_data = JSONField(blank=True, null=True) + created_at = DateTimeField(auto_now_add=True) + notification_type = CharField(max_length=50, choices=[ + ('info', 'Info'), + ('warning', 'Warning'), + ('success', 'Success'), + ('error', 'Error') + ]) + + def __str__(self): + return f"Notification - {self.title}" + + +class NotificationStatus(Model): + notification = ForeignKey(Notifications, on_delete=CASCADE) + user = ForeignKey(User, on_delete=CASCADE) + is_read = BooleanField(default=False) + read_at = DateTimeField(blank=True, null=True) + + class Meta: + unique_together = ('notification', 'user') + + def __str__(self): + return f"Status for {self.user.username} - {self.notification.title}" \ No newline at end of file diff --git a/cvat/apps/notifications/permissions.py b/cvat/apps/notifications/permissions.py new file mode 100644 index 000000000000..6915c6831e6d --- /dev/null +++ b/cvat/apps/notifications/permissions.py @@ -0,0 +1,37 @@ +from django.conf import settings +from cvat.apps.iam.permissions import OpenPolicyAgentPermission, StrEnum + +class NotificationPermission(OpenPolicyAgentPermission): + class Scopes(StrEnum): + VIEW = 'view' + SEND = 'send' + MARK_AS_READ = 'mark_as_read' + + @classmethod + def create(cls, request, view, obj, iam_context): + permissions = [] + + for scope in cls.get_scopes(request, view, obj): + perm = cls.create_base_perm(request, view, scope, iam_context, obj) + permissions.append(perm) + + return permissions + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.url = settings.IAM_OPA_DATA_URL + '/notifications/allow' + + @staticmethod + def get_scopes(request, view, obj): + Scopes = __class__.Scopes + if view.action == 'SendNotification': + return [Scopes.SEND] + elif view.action == 'FetchUserNotifications': + return [Scopes.VIEW] + elif view.action == 'MarkNotificationAsViewed': + return [Scopes.MARK_AS_READ] + + return [] + + def get_resource(self): + return None \ No newline at end of file diff --git a/cvat/apps/notifications/serializers.py b/cvat/apps/notifications/serializers.py new file mode 100644 index 000000000000..af3272acf727 --- /dev/null +++ b/cvat/apps/notifications/serializers.py @@ -0,0 +1,50 @@ +# serializers.py +from rest_framework import serializers +from .models import * + + +class NotificationSerializer(serializers.ModelSerializer): + class Meta: + model = Notifications + fields = ['id', 'title', 'message', 'notification_type', 'created_at'] + + +class UserNotificationStatusSerializer(serializers.ModelSerializer): + class Meta: + model = NotificationStatus + fields = ['is_read', 'read_at'] + + +class UserNotificationDetailSerializer(serializers.ModelSerializer): + status = UserNotificationStatusSerializer(source='notificationstatus_set.first') + + class Meta: + model = Notifications + fields = ['id', 'title', 'message', 'notification_type', 'created_at', 'status'] + + +class AddNotificationSerializer(serializers.Serializer): + title = serializers.CharField(max_length=255) + message = serializers.CharField() + notification_type = serializers.CharField(max_length=50) + + +class SendNotificationSerializer(serializers.Serializer): + user = serializers.IntegerField(required=False) + org = serializers.IntegerField(required=False) + title = serializers.CharField() + message = serializers.CharField() + notification_type = serializers.CharField(max_length=50) + + +class MarkNotificationAsViewedSerializer(serializers.Serializer): + user = serializers.IntegerField() + notification_ids = serializers.ListField( + child=serializers.IntegerField() + ) + + +class FetchUserNotificationsSerializer(serializers.Serializer): + user = serializers.IntegerField() + current_page = serializers.IntegerField() + items_per_page = serializers.IntegerField() \ No newline at end of file diff --git a/cvat/apps/notifications/tests.py b/cvat/apps/notifications/tests.py new file mode 100644 index 000000000000..7ce503c2dd97 --- /dev/null +++ b/cvat/apps/notifications/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/cvat/apps/notifications/urls.py b/cvat/apps/notifications/urls.py new file mode 100644 index 000000000000..1620e9232384 --- /dev/null +++ b/cvat/apps/notifications/urls.py @@ -0,0 +1,21 @@ +from django.urls import path +from .views import NotificationsViewSet + + +notifications_viewset = NotificationsViewSet.as_view({ + 'post': 'SendNotification' +}) + +fetch_notifications_viewset = NotificationsViewSet.as_view({ + 'post': 'FetchUserNotifications' +}) + +mark_all_read_viewset = NotificationsViewSet.as_view({ + 'post': 'MarkNotificationAsViewed' +}) + +urlpatterns = [ + path('notifications/send', notifications_viewset, name='send-notification'), + path('notifications/fetch', fetch_notifications_viewset, name='fetch-user-notifications'), + path('notifications/markallread', mark_all_read_viewset, name='mark-all-read'), +] \ No newline at end of file diff --git a/cvat/apps/notifications/views.py b/cvat/apps/notifications/views.py new file mode 100644 index 000000000000..5c34b8404dbe --- /dev/null +++ b/cvat/apps/notifications/views.py @@ -0,0 +1,329 @@ +from django.shortcuts import render +from django.utils import timezone +from django.core.paginator import EmptyPage + +from rest_framework import status, viewsets +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.pagination import PageNumberPagination + +from .models import * +from .serializers import * + +import json +import traceback + + +## Pagination +class CustomPagination(PageNumberPagination): + def paginate_queryset(self, queryset, request, view = None): + page_size = request.data.get('items_per_page', 10) + page_number = request.data.get('current_page', 1) + self.page_size = page_size + paginator = self.django_paginator_class(queryset, page_size) + + try: + self.page = paginator.page(page_number) + except EmptyPage: + return None + + if int(page_number) > paginator.num_pages: + return None + + return list(self.page) + + +## Notification +class NotificationsViewSet(viewsets.ViewSet): + isAuthorized = True + + def AddNotification(self, data): + serializer = AddNotificationSerializer(data=data) + if serializer.is_valid(): + try: + notification = Notifications.objects.create( + title=serializer.validated_data['title'], + message=serializer.validated_data['message'], + notification_type=serializer.validated_data['notification_type'] + ) + return Response( + { + "success": True, + "message": "Notification saved successfully.", + "data": { + "notification": UserNotificationDetailSerializer(notification).data + }, + "error": None + } + ) + except Exception as e: + error = traceback.format_exc() + return Response( + { + "success": False, + "message": "An error occurred while saving notification.", + "data": {}, + "error": error + }, + status = status.HTTP_500_INTERNAL_SERVER_ERROR + ) + else: + return Response( + { + "success": False, + "message": "Invalid data.", + "data": serializer.errors, + "error": None + }, + status = status.HTTP_400_BAD_REQUEST + ) + + def SendNotification(self, request: Request): + try: + data = request.data # Use request.data instead of json.loads(request.body) + serializer = SendNotificationSerializer(data=data) + if serializer.is_valid(): + response = self.AddNotification(serializer.validated_data) + if not response.data["success"]: + return response + + notification = response.data["data"]["notification"] + + if "user" in serializer.validated_data: + user = serializer.validated_data["user"] + response = self.SendUserNotifications(notification, user) + elif "org" in serializer.validated_data: + response = self.SendOrganizationNotifications(notification, serializer.validated_data) + + return response + else: + return Response( + { + "success": False, + "message": "Invalid request data.", + "data": serializer.errors, + "error": None + }, + status = status.HTTP_400_BAD_REQUEST + ) + except Exception as e: + error = traceback.format_exc() + return Response( + { + "success": False, + "message": "An error occurred while sending notification.", + "data": {}, + "error": error + }, + status = status.HTTP_500_INTERNAL_SERVER_ERROR + ) + + def SendUserNotifications(self, notification, user_id): + try: + user = User.objects.get(id=user_id) + notification = Notifications.objects.get(id=notification.get("id")) + NotificationStatus.objects.get_or_create( + notification=notification, + user=user, + defaults={'is_read': False} + ) + return Response( + { + "success": True, + "message": "Notification sent successfully.", + "data": {}, + "error": None + }, + status = status.HTTP_201_CREATED + ) + except User.DoesNotExist: + return Response( + { + "success": False, + "message": f"User with id {user_id} does not exist.", + "data": {}, + "error": None + }, + status = status.HTTP_404_NOT_FOUND + ) + except Exception as e: + error = traceback.format_exc() + return Response( + { + "success": False, + "message": "An error occurred while sending user notification.", + "data": {}, + "error": error + }, + status = status.HTTP_500_INTERNAL_SERVER_ERROR + ) + + def SendOrganizationNotifications(self, notification, data): + try: + organization = Organization.objects.get(id=data["org"]) + members = organization.members.filter(is_active=True) + errors = [] + + for member in members: + user = member.user + response = self.SendUserNotifications(notification, user.id) + if not response.data.get("success"): + errors.append(f"Error occurred while sending notification to user ({user.username}). Error: {response.data.get('error')}") + + if not errors: + return Response( + { + "success": True, + "message": "Notifications sent successfully.", + "data": {}, + "error": None + }, + status = status.HTTP_200_OK + ) + else: + return Response( + { + "success": False, + "message": "Unable to send notifications to one or more users.", + "data": {}, + "error": errors + }, + status = status.HTTP_504_GATEWAY_TIMEOUT + ) + except Organization.DoesNotExist: + return Response( + { + "success": False, + "message": f"Organization with id {data['org']} does not exist.", + "data": {}, + "error": None + }, + status = status.HTTP_404_NOT_FOUND + ) + except Exception as e: + error = traceback.format_exc() + return Response( + { + "success": False, + "message": "An error occurred while sending organization notifications.", + "data": {}, + "error": error + }, + status = status.HTTP_500_INTERNAL_SERVER_ERROR + ) + + + def FetchUserNotifications(self, request: Request): + try: + data = request.data + serializer = FetchUserNotificationsSerializer(data = data) + + if serializer.is_valid(): + user_id = serializer.validated_data["user"] + notifications_status = NotificationStatus.objects.filter(user_id=user_id).order_by('-notification__created_at') + unread_count = notifications_status.filter(is_read=False).count() + + # Set up pagination + paginator = CustomPagination() + paginated_notifications = paginator.paginate_queryset(notifications_status, request) + + if paginated_notifications is None: + return Response( + { + "success": False, + "message": "No notifications available on this page.", + "data": None, + "error": None + }, + status = status.HTTP_400_BAD_REQUEST + ) + + serialized_notifications = [UserNotificationDetailSerializer(noti_status.notification).data for noti_status in paginated_notifications] + + return Response( + { + "success": True, + "message": "User notifications fetched successfully.", + "data": { + "unread" : unread_count, + "notifications": serialized_notifications + }, + "error": None + }, + status = status.HTTP_200_OK + ) + else: + return Response( + { + "success": False, + "message": "Invalid request data.", + "data": serializer.errors, + "error": None + }, + status = status.HTTP_400_BAD_REQUEST + ) + except Exception as e: + error = traceback.format_exc() + print(error) + return Response( + { + "success": False, + "message": "An error occurred while fetching notifications.", + "data": {}, + "error": error + }, + status = status.HTTP_500_INTERNAL_SERVER_ERROR + ) + + def MarkNotificationAsViewed(self, request: Request): + try: + data = request.data # Use request.data instead of json.loads(request.body) + serializer = MarkNotificationAsViewedSerializer(data=data) + if serializer.is_valid(): + user_id = serializer.validated_data["user"] + notification_ids = serializer.validated_data["notification_ids"] + + notifications_status = NotificationStatus.objects.filter(notification_id__in=notification_ids, user_id=user_id) + updated_count = notifications_status.update(is_read=True, read_at=timezone.now()) + + if updated_count == 0: + return Response( + { + "success": False, + "message": "No notifications found or none belong to you.", + "data": {}, + "error": None + }, + status = status.HTTP_404_NOT_FOUND + ) + + return Response( + { + "success": True, + "message": f"{updated_count} notifications marked as viewed.", + "data": {}, + "error": None + }, + status = status.HTTP_200_OK + ) + else: + return Response( + { + "success": False, + "message": "Invalid data.", + "data": serializer.errors, + "error": None + }, + status = status.HTTP_400_BAD_REQUEST + ) + except Exception as e: + error = traceback.format_exc() + return Response( + { + "success": False, + "message": "An error occurred while marking notifications as viewed.", + "data": {}, + "error": error + }, + status = status.HTTP_500_INTERNAL_SERVER_ERROR + ) \ No newline at end of file diff --git a/cvat/apps/organizations/migrations/0003_notifications.py b/cvat/apps/organizations/migrations/0003_notifications.py new file mode 100644 index 000000000000..a6f2fbb48b94 --- /dev/null +++ b/cvat/apps/organizations/migrations/0003_notifications.py @@ -0,0 +1,53 @@ +# Generated by Django 4.2.13 on 2024-09-04 10:20 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("organizations", "0002_invitation_sent_date"), + ] + + operations = [ + migrations.CreateModel( + name="Notifications", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("title", models.CharField(max_length=255)), + ("message", models.TextField()), + ("extra_data", models.JSONField(blank=True, null=True)), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("is_read", models.BooleanField(default=False)), + ("read_at", models.DateTimeField(blank=True, null=True)), + ( + "notification_type", + models.CharField( + choices=[ + ("info", "Info"), + ("warning", "Warning"), + ("success", "Success"), + ("error", "Error"), + ], + max_length=50, + ), + ), + ( + "recipient", + models.ManyToManyField( + related_name="notifications", to=settings.AUTH_USER_MODEL + ), + ), + ], + ), + ] diff --git a/cvat/apps/organizations/migrations/0004_delete_notifications.py b/cvat/apps/organizations/migrations/0004_delete_notifications.py new file mode 100644 index 000000000000..d3b73a68abdd --- /dev/null +++ b/cvat/apps/organizations/migrations/0004_delete_notifications.py @@ -0,0 +1,16 @@ +# Generated by Django 4.2.14 on 2024-09-06 07:55 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("organizations", "0003_notifications"), + ] + + operations = [ + migrations.DeleteModel( + name="Notifications", + ), + ] diff --git a/cvat/apps/organizations/models.py b/cvat/apps/organizations/models.py index 3da77bafbebf..44c958e4392d 100644 --- a/cvat/apps/organizations/models.py +++ b/cvat/apps/organizations/models.py @@ -118,4 +118,4 @@ def accept(self, date=None): self.membership.save() class Meta: - default_permissions = () + default_permissions = () \ No newline at end of file diff --git a/cvat/apps/organizations/serializers.py b/cvat/apps/organizations/serializers.py index 9cfb467aa3b9..966c28c5fa1d 100644 --- a/cvat/apps/organizations/serializers.py +++ b/cvat/apps/organizations/serializers.py @@ -157,4 +157,4 @@ class Meta: read_only_fields = ['user', 'organization', 'is_active', 'joined_date'] class AcceptInvitationReadSerializer(serializers.Serializer): - organization_slug = serializers.CharField() + organization_slug = serializers.CharField() \ No newline at end of file diff --git a/cvat/apps/organizations/views.py b/cvat/apps/organizations/views.py index 11b92b29cad8..218bf6db8a15 100644 --- a/cvat/apps/organizations/views.py +++ b/cvat/apps/organizations/views.py @@ -239,6 +239,7 @@ def get_queryset(self): def create(self, request): serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) + try: self.perform_create(serializer) except ImproperlyConfigured: @@ -254,7 +255,11 @@ def perform_create(self, serializer): request=self.request, ) + + + def perform_update(self, serializer): + if 'accepted' in self.request.query_params: serializer.instance.accept() else: @@ -265,6 +270,7 @@ def perform_update(self, serializer): def accept(self, request, pk): try: invitation = self.get_object() # force to call check_object_permissions + if invitation.expired: return Response(status=status.HTTP_400_BAD_REQUEST, data="Your invitation is expired. Please contact organization owner to renew it.") if invitation.membership.is_active: @@ -272,6 +278,17 @@ def accept(self, request, pk): invitation.accept() response_serializer = AcceptInvitationReadSerializer(data={'organization_slug': invitation.membership.organization.slug}) response_serializer.is_valid(raise_exception=True) + + ## Notifications + from ..notifications.api import SendNotificationToOrganisationUsers + + notification_response = SendNotificationToOrganisationUsers( + self.request.iam_context['organization'].id, + f"{self.request.user.username} joined to {self.request.iam_context['organization'].name}", + "Hey guys an idiot joined the organization.", + "info" + ) + return Response(status=status.HTTP_200_OK, data=response_serializer.data) except Invitation.DoesNotExist: return Response(status=status.HTTP_404_NOT_FOUND, data="This invitation does not exist. Please contact organization owner.") @@ -297,4 +314,4 @@ def decline(self, request, pk): membership.delete() return Response(status=status.HTTP_204_NO_CONTENT) except Invitation.DoesNotExist: - return Response(status=status.HTTP_404_NOT_FOUND, data="This invitation does not exist.") + return Response(status=status.HTTP_404_NOT_FOUND, data="This invitation does not exist.") \ No newline at end of file diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 3e4d610915bd..72cbf0d3c4fd 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -117,6 +117,7 @@ def generate_secret_key(): 'cvat.apps.events', 'cvat.apps.quality_control', 'cvat.apps.analytics_report', + 'cvat.apps.notifications', ] SITE_ID = 1 @@ -723,4 +724,4 @@ class CVAT_QUEUES(Enum): update_started_job_registry_cleanup() CLOUD_DATA_DOWNLOADING_MAX_THREADS_NUMBER = 4 -CLOUD_DATA_DOWNLOADING_NUMBER_OF_FILES_PER_THREAD = 1000 +CLOUD_DATA_DOWNLOADING_NUMBER_OF_FILES_PER_THREAD = 1000 \ No newline at end of file diff --git a/cvat/urls.py b/cvat/urls.py index 144ed619f766..396f3a1b4f6b 100644 --- a/cvat/urls.py +++ b/cvat/urls.py @@ -51,3 +51,6 @@ if apps.is_installed('cvat.apps.analytics_report'): urlpatterns.append(path('api/', include('cvat.apps.analytics_report.urls'))) + +if apps.is_installed('cvat.apps.notifications'): + urlpatterns.append(path('api/', include('cvat.apps.notifications.urls'))) \ No newline at end of file diff --git a/datumaro b/datumaro new file mode 160000 index 000000000000..393cb6665290 --- /dev/null +++ b/datumaro @@ -0,0 +1 @@ +Subproject commit 393cb666529067060ff57e30cb6e448669274f35