From 7262cfa56d4df2492296e61b7bc1fa51265622b7 Mon Sep 17 00:00:00 2001 From: gaby kourie Date: Thu, 30 Nov 2023 23:12:02 +0100 Subject: [PATCH 001/100] Implement video sprite generator function the function generateSprites generates sprites from a video file, organizes them into a 5x5 sprite sheet, and saves them. --- app/Jobs/ProcessNewVideo.php | 56 ++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/app/Jobs/ProcessNewVideo.php b/app/Jobs/ProcessNewVideo.php index 0c830c4da..3a061194c 100644 --- a/app/Jobs/ProcessNewVideo.php +++ b/app/Jobs/ProcessNewVideo.php @@ -155,6 +155,8 @@ public function handleFile($file, $path) $buffer = $this->generateVideoThumbnail($path, $time, $width, $height, $format); $disk->put("{$fragment}/{$index}.{$format}", $buffer); } + // generate sprites + $this->generateSprites($path, $this->video->duration, $disk, $fragment); } catch (Exception $e) { // The video seems to be fine if it passed the previous checks. There may be // errors in the actual video data but we can ignore that and skip generating @@ -271,4 +273,58 @@ protected function getThumbnailTimes($duration) return $range; } + + /** + * Generate sprites from a video file and save them to a storage disk. + * + * This function takes a video file path, its duration, a storage disk instance, + * and a fragment identifier to generate sprites from the video at specified intervals. + * Sprites are created as thumbnails of frames and organized into sprite sheets. + * + * @param string $path path to the video file. + * @param float $duration duration of the video in seconds. + * @param mixed $disk storage disk instance (e.g., Laravel's Storage). + * @param string $fragment fragment identifier for organizing the sprites. + * + */ + protected function generateSprites($path, $duration, $disk, $fragment) + { + // Cache the video instance. + if (!isset($this->ffmpegVideo)) { + $this->ffmpegVideo = FFMpeg::create()->open($path); + } + + $MAX_FRAMES = 1500; + $MIN_FRAMES = 5; + $FRAMES_PER_SPRITE = 25; + + $durationRounded = round($duration); + $estimatedFrames = $durationRounded / 10; + // Adjust the frame time based on the number of generated frames + $seconds = ($estimatedFrames > $MAX_FRAMES) ? $durationRounded / $MAX_FRAMES : (($estimatedFrames < $MIN_FRAMES) ? $durationRounded / $MIN_FRAMES : 10); + + $frames = []; + for ($time = 0; $time <= $duration; $time += $seconds) { + // Capture a frame at the current time, save it to buffer + $buffer = $this->ffmpegVideo->frame(TimeCode::fromSeconds($time))->save(null, false, true); + // Create a thumbnail of the frame + $vipsImg = VipsImage::thumbnail_buffer($buffer, 240, ['height' => 138]); + // Add the VipsImage to the frames array + $frames[] = $vipsImg; + + if (count($frames) % $FRAMES_PER_SPRITE === 0 || $time > ($duration - $seconds)) { + // Join the frames into a 5x5 sprite + $sprite = VipsImage::arrayjoin($frames, ['across' => 5]); + // Write the sprite to buffer with quality 85 and stripped metadata + $spriteBuffer = $sprite->writeToBuffer(".jpg", [ + 'Q' => 85, + 'strip' => true, + ]); + $spritePath = "{$fragment}/sprite_{$time}.jpg"; + $disk->put($spritePath, $spriteBuffer); + + $frames = []; + } + } + } } From 5a37738d27be39ce32cb34c8f48653bf8a3aa5c4 Mon Sep 17 00:00:00 2001 From: gaby kourie Date: Thu, 30 Nov 2023 23:18:54 +0100 Subject: [PATCH 002/100] Add test for generating sprites. --- app/Jobs/ProcessNewVideo.php | 2 +- tests/php/Jobs/ProcessNewVideoTest.php | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/app/Jobs/ProcessNewVideo.php b/app/Jobs/ProcessNewVideo.php index 3a061194c..d12452204 100644 --- a/app/Jobs/ProcessNewVideo.php +++ b/app/Jobs/ProcessNewVideo.php @@ -285,7 +285,7 @@ protected function getThumbnailTimes($duration) * @param float $duration duration of the video in seconds. * @param mixed $disk storage disk instance (e.g., Laravel's Storage). * @param string $fragment fragment identifier for organizing the sprites. - * + * */ protected function generateSprites($path, $duration, $disk, $fragment) { diff --git a/tests/php/Jobs/ProcessNewVideoTest.php b/tests/php/Jobs/ProcessNewVideoTest.php index 1262b85ea..0a968c884 100644 --- a/tests/php/Jobs/ProcessNewVideoTest.php +++ b/tests/php/Jobs/ProcessNewVideoTest.php @@ -31,6 +31,22 @@ public function testHandleThumbnails() $this->assertTrue($disk->exists("{$fragment}/2.jpg")); } + public function testGenerateSprites() + { + Storage::fake('video-thumbs'); + config(['videos.thumbnail_count' => 1]); + $video = VideoTest::create(['filename' => 'test.mp4']); + $job = new ProcessNewVideoStub($video); + $job->duration = 10; + + $job->handle(); + $this->assertEquals(10, $video->fresh()->duration); + + $disk = Storage::disk('video-thumbs'); + $fragment = fragment_uuid_path($video->uuid); + $this->assertTrue($disk->exists("{$fragment}/sprite_10.jpg")); + } + public function testHandleNotFound() { $video = VideoTest::create(['filename' => 'abc.mp4']); From e06def8f4247e5e268e733a83a6dcd4e095df1d8 Mon Sep 17 00:00:00 2001 From: gaby kourie Date: Thu, 7 Dec 2023 01:39:42 +0100 Subject: [PATCH 003/100] Fix issues and add more test cases --- app/Jobs/ProcessNewVideo.php | 74 ++++++++-------- config/videos.php | 34 +++++++- tests/php/Jobs/ProcessNewVideoTest.php | 114 +++++++++++++++++++++++-- 3 files changed, 176 insertions(+), 46 deletions(-) diff --git a/app/Jobs/ProcessNewVideo.php b/app/Jobs/ProcessNewVideo.php index d12452204..4efa99580 100644 --- a/app/Jobs/ProcessNewVideo.php +++ b/app/Jobs/ProcessNewVideo.php @@ -152,7 +152,11 @@ public function handleFile($file, $path) try { foreach ($times as $index => $time) { - $buffer = $this->generateVideoThumbnail($path, $time, $width, $height, $format); + $buffer = $this->generateVideoThumbnail($path, $time, $width, $height) + ->writeToBuffer(".{$format}", [ + 'Q' => 85, + 'strip' => true, + ]); $disk->put("{$fragment}/{$index}.{$format}", $buffer); } // generate sprites @@ -223,11 +227,10 @@ protected function getVideoDimensions($url) * @param float $time Time for the thumbnail in seconds. * @param int $width Width of the thumbnail. * @param int $height Height of the thumbnail. - * @param string $format File format of the thumbnail (e.g. 'jpg'). * * @return string Vips image buffer string. */ - protected function generateVideoThumbnail($path, $time, $width, $height, $format) + protected function generateVideoThumbnail($path, $time, $width, $height) { // Cache the video instance. if (!isset($this->ffmpegVideo)) { @@ -238,10 +241,7 @@ protected function generateVideoThumbnail($path, $time, $width, $height, $format ->save(null, false, true); return VipsImage::thumbnail_buffer($buffer, $width, ['height' => $height]) - ->writeToBuffer(".{$format}", [ - 'Q' => 85, - 'strip' => true, - ]); + ; } /** @@ -289,41 +289,39 @@ protected function getThumbnailTimes($duration) */ protected function generateSprites($path, $duration, $disk, $fragment) { - // Cache the video instance. - if (!isset($this->ffmpegVideo)) { - $this->ffmpegVideo = FFMpeg::create()->open($path); - } - $MAX_FRAMES = 1500; - $MIN_FRAMES = 5; - $FRAMES_PER_SPRITE = 25; - - $durationRounded = round($duration); - $estimatedFrames = $durationRounded / 10; - // Adjust the frame time based on the number of generated frames - $seconds = ($estimatedFrames > $MAX_FRAMES) ? $durationRounded / $MAX_FRAMES : (($estimatedFrames < $MIN_FRAMES) ? $durationRounded / $MIN_FRAMES : 10); - - $frames = []; - for ($time = 0; $time <= $duration; $time += $seconds) { - // Capture a frame at the current time, save it to buffer - $buffer = $this->ffmpegVideo->frame(TimeCode::fromSeconds($time))->save(null, false, true); - // Create a thumbnail of the frame - $vipsImg = VipsImage::thumbnail_buffer($buffer, 240, ['height' => 138]); - // Add the VipsImage to the frames array - $frames[] = $vipsImg; - - if (count($frames) % $FRAMES_PER_SPRITE === 0 || $time > ($duration - $seconds)) { - // Join the frames into a 5x5 sprite - $sprite = VipsImage::arrayjoin($frames, ['across' => 5]); - // Write the sprite to buffer with quality 85 and stripped metadata - $spriteBuffer = $sprite->writeToBuffer(".jpg", [ - 'Q' => 85, + $maxFrames = config('videos.sprites_max_frames'); + $minFrames = config('videos.sprites_min_frames'); + $framesPerSprite = config('videos.frames_per_sprite'); + $thumbsPerRow = sqrt($framesPerSprite); + $thumbnailWidth = config('videos.sprites_thumbnail_width'); + $thumbnailHeight = config('videos.sprites_thumbnail_height'); + $spriteFormat = config('videos.sprites_format'); + $intervalSeconds = config('videos.sprites_interval_seconds'); + $durationRounded = floor($duration * 10) / 10; + $estimatedFrames = $durationRounded / $intervalSeconds; + // Adjust the frame time based on the number of estimated frames + $intervalSeconds = ($estimatedFrames > $maxFrames) ? $durationRounded / $maxFrames + : (($estimatedFrames < $minFrames) ? $durationRounded / $minFrames : 10); + + $thumbnails = []; + $spritesCounter = 0; + for ($time = 0.0; $time < $durationRounded && $durationRounded > 0; $time += $intervalSeconds) { + $time = round($time, 1); + // Create a thumbnail from the video at the specified time and add it to the thumbnails array. + $thumbnails[] = $this->generateVideoThumbnail($path, $time, $thumbnailWidth, $thumbnailHeight); + if (count($thumbnails) === $framesPerSprite || $time >= abs($durationRounded - $intervalSeconds)) { + // Join the thumbnails into a NxN sprite + $sprite = VipsImage::arrayjoin($thumbnails, ['across' => $thumbsPerRow]); + // Write the sprite to buffer with quality 75 and stripped metadata + $spriteBuffer = $sprite->writeToBuffer(".{$spriteFormat}", [ + 'Q' => 75, 'strip' => true, ]); - $spritePath = "{$fragment}/sprite_{$time}.jpg"; + $spritePath = "{$fragment}/sprite_{$spritesCounter}.{$spriteFormat}"; $disk->put($spritePath, $spriteBuffer); - - $frames = []; + $thumbnails = []; + $spritesCounter += 1; } } } diff --git a/config/videos.php b/config/videos.php index 17c9a1c4e..4cdcd6967 100644 --- a/config/videos.php +++ b/config/videos.php @@ -15,7 +15,7 @@ /* | Path to the object tracking script. */ - 'object_tracker_script' => __DIR__.'/../resources/scripts/ObjectTracker.py', + 'object_tracker_script' => __DIR__ . '/../resources/scripts/ObjectTracker.py', /* | Distance in pixels between the annotation center positions or circle radii of two @@ -52,4 +52,36 @@ */ 'track_object_max_jobs_per_user' => env('VIDEOS_TRACK_OBJECT_MAX_JOBS_PER_USER', 10), + /* + | Number of max frames to generate for sprites. + */ + 'sprites_max_frames' => 1500, + + /* + | Number of min frames to generate for sprites. + */ + 'sprites_min_frames' => 5, + + /* + | Number of frames per sprite. Default 5x5 = 25. + | The square root of the number must be an integer. + */ + 'frames_per_sprite' => 25, + + /* + | Dimensions of the thumbnail images to create in sprites. + */ + 'sprites_thumbnail_width' => 240, + 'sprites_thumbnail_height' => 138, + + /* + | Sprite file format. + */ + 'sprites_format' => 'webp', + + /* + | Standard time interval at which thumbnails should be sampled when generating sprites. + | It will be adjusted dynamically during the process. + */ + 'sprites_interval_seconds' => 10, ]; diff --git a/tests/php/Jobs/ProcessNewVideoTest.php b/tests/php/Jobs/ProcessNewVideoTest.php index 0a968c884..c7c76d957 100644 --- a/tests/php/Jobs/ProcessNewVideoTest.php +++ b/tests/php/Jobs/ProcessNewVideoTest.php @@ -7,8 +7,10 @@ use Biigle\Video; use Exception; use FileCache; +use Jcupitt\Vips\Extend; use Storage; use TestCase; +use VipsImage; class ProcessNewVideoTest extends TestCase { @@ -22,7 +24,7 @@ public function testHandleThumbnails() $job->handle(); $this->assertEquals(10, $video->fresh()->duration); - $this->assertEquals([0.5, 5, 9.5], $job->times); + $this->assertEquals([0.5, 5, 9.5], array_slice($job->times, 0, 3)); $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); @@ -34,17 +36,110 @@ public function testHandleThumbnails() public function testGenerateSprites() { Storage::fake('video-thumbs'); - config(['videos.thumbnail_count' => 1]); + config(['videos.frames_per_sprite' => 9]); $video = VideoTest::create(['filename' => 'test.mp4']); $job = new ProcessNewVideoStub($video); - $job->duration = 10; + $job->duration = 180; $job->handle(); - $this->assertEquals(10, $video->fresh()->duration); + $this->assertEquals(180, $video->fresh()->duration); + $expected_frames = 18; + // additional frames caused by generating regular thumbnails + $additional_frames = $job->frames - $expected_frames; + $this->assertEquals($expected_frames, $job->frames - $additional_frames); + $this->assertEquals([0.0, 10.0, 20.0, 30.0, 40.0, 50.0, + 60.0, 70.0, 80.0, 90.0, 100.0, 110.0, 120.0, 130.0, + 140.0, 150.0, 160.0, 170.0], array_slice($job->times, $additional_frames, $job->frames)); + + $disk = Storage::disk('video-thumbs'); + $fragment = fragment_uuid_path($video->uuid); + $this->assertTrue($disk->exists("{$fragment}/sprite_0.webp")); + $this->assertTrue($disk->exists("{$fragment}/sprite_1.webp")); + } + + public function testGenerateSpritesZeroDuration() + { + Storage::fake('video-thumbs'); + $video = VideoTest::create(['filename' => 'test.mp4']); + $job = new ProcessNewVideoStub($video); + $job->duration = 0; + + $job->handle(); + $expected_frames = 0; + // additional frames caused by generating regular thumbnails + $additional_frames = $job->frames - $expected_frames; + $this->assertEquals($expected_frames, $job->frames - $additional_frames); + $this->assertEquals([], array_slice($job->times, $additional_frames, $job->frames)); + + $disk = Storage::disk('video-thumbs'); + $fragment = fragment_uuid_path($video->uuid); + $this->assertFalse($disk->exists("{$fragment}/sprite_0.webp")); + } + + public function testGenerateSpritesOneSecondDuration() + { + Storage::fake('video-thumbs'); + $video = VideoTest::create(['filename' => 'test.mp4']); + $job = new ProcessNewVideoStub($video); + $job->duration = 1; + + $job->handle(); + $this->assertEquals(1, $video->fresh()->duration); + $expected_frames = 5; + // additional frames caused by generating regular thumbnails + $additional_frames = $job->frames - $expected_frames; + $this->assertEquals($expected_frames, $job->frames - $additional_frames); + $this->assertEquals([0.0, 0.2, 0.4, 0.6, 0.8], array_slice($job->times, $additional_frames, $job->frames)); + + $disk = Storage::disk('video-thumbs'); + $fragment = fragment_uuid_path($video->uuid); + $this->assertTrue($disk->exists("{$fragment}/sprite_0.webp")); + + } + + public function testGenerateSpritesExceedsMaxFrames() + { + Storage::fake('video-thumbs'); + config(['videos.sprites_max_frames' => 25]); + config(['videos.sprites_interval_seconds' => 5]); + $video = VideoTest::create(['filename' => 'test.mp4']); + $job = new ProcessNewVideoStub($video); + $job->duration = 130; + + $job->handle(); + $this->assertEquals(130, $video->fresh()->duration); + $expected_frames = 25; + // additional frames caused by generating regular thumbnails + $additional_frames = $job->frames - $expected_frames; + $this->assertEquals($expected_frames, $job->frames - $additional_frames); + $this->assertEquals([0.0, 5.2, 10.4, 15.6, 20.8, 26.0, 31.2, 36.4, + 41.6, 46.8, 52.0, 57.2, 62.4, 67.6, 72.8, 78.0, 83.2, 88.4, + 93.6, 98.8, 104.0, 109.2, 114.4, 119.6, 124.8], array_slice($job->times, $additional_frames, $job->frames)); + + $disk = Storage::disk('video-thumbs'); + $fragment = fragment_uuid_path($video->uuid); + $this->assertTrue($disk->exists("{$fragment}/sprite_0.webp")); + } + + public function testGenerateSpritesFallsBelowMinFrames() + { + Storage::fake('video-thumbs'); + $video = VideoTest::create(['filename' => 'test.mp4']); + $job = new ProcessNewVideoStub($video); + $job->duration = 20; + + $job->handle(); + $this->assertEquals(20, $video->fresh()->duration); + $expected_frames = 5; + // additional frames caused by generating regular thumbnails + $additional_frames = $job->frames - $expected_frames; + $this->assertEquals($expected_frames, $job->frames - $additional_frames); + $this->assertEquals([0.0, 4.0, 8.0, 12.0, 16.0], array_slice($job->times, $additional_frames, $job->frames)); + $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); - $this->assertTrue($disk->exists("{$fragment}/sprite_10.jpg")); + $this->assertTrue($disk->exists("{$fragment}/sprite_0.webp")); } public function testHandleNotFound() @@ -173,6 +268,7 @@ class ProcessNewVideoStub extends ProcessNewVideo public $duration = 0; public $codec = ''; public $times = []; + public $frames = 0; protected function getCodec($path) { @@ -184,10 +280,14 @@ protected function getVideoDuration($path) return $this->duration; } - protected function generateVideoThumbnail($path, $time, $width, $height, $format) + protected function generateVideoThumbnail($path, $time, $width, $height) { $this->times[] = $time; + $this->frames += 1; - return 'content'; + return VipsImage::black(240, 138) + ->embed(30, 40, 240, 138, ['extend' => Extend::WHITE]) // Extend left & top edges with white color + ->add("#FFFFFF") + ->cast("uchar"); } } From bbea48c803704edb69d8da7e04b8fbdf637a2717 Mon Sep 17 00:00:00 2001 From: gaby kourie Date: Thu, 25 Jan 2024 23:52:46 +0100 Subject: [PATCH 004/100] Fix Vips thumbnails buffer aspect ratio issue. This commit fixes an issue with the aspect ratio of Vips thumbnails buffer by setting the size to 'force'. It also adjusts the corresponding test cases. --- app/Jobs/ProcessNewVideo.php | 26 +++++----- config/videos.php | 14 +++--- tests/php/Jobs/ProcessNewVideoTest.php | 70 ++++++++++++-------------- 3 files changed, 53 insertions(+), 57 deletions(-) diff --git a/app/Jobs/ProcessNewVideo.php b/app/Jobs/ProcessNewVideo.php index 4efa99580..f411d33e5 100644 --- a/app/Jobs/ProcessNewVideo.php +++ b/app/Jobs/ProcessNewVideo.php @@ -240,7 +240,7 @@ protected function generateVideoThumbnail($path, $time, $width, $height) $buffer = $this->ffmpegVideo->frame(TimeCode::fromSeconds($time)) ->save(null, false, true); - return VipsImage::thumbnail_buffer($buffer, $width, ['height' => $height]) + return VipsImage::thumbnail_buffer($buffer, $width, ['height' => $height, 'size' => 'force']) ; } @@ -290,29 +290,29 @@ protected function getThumbnailTimes($duration) protected function generateSprites($path, $duration, $disk, $fragment) { - $maxFrames = config('videos.sprites_max_frames'); - $minFrames = config('videos.sprites_min_frames'); - $framesPerSprite = config('videos.frames_per_sprite'); - $thumbsPerRow = sqrt($framesPerSprite); + $maxThumbnails = config('videos.sprites_max_thumbnails'); + $minThumbnails = config('videos.sprites_min_thumbnails'); + $thumbnailsPerSprite = config('videos.sprites_thumbnails_per_sprite'); + $thumbnailsPerRow = sqrt($thumbnailsPerSprite); $thumbnailWidth = config('videos.sprites_thumbnail_width'); $thumbnailHeight = config('videos.sprites_thumbnail_height'); $spriteFormat = config('videos.sprites_format'); - $intervalSeconds = config('videos.sprites_interval_seconds'); + $defaultThumbnailInterval = config('videos.sprites_thumbnail_interval'); $durationRounded = floor($duration * 10) / 10; - $estimatedFrames = $durationRounded / $intervalSeconds; - // Adjust the frame time based on the number of estimated frames - $intervalSeconds = ($estimatedFrames > $maxFrames) ? $durationRounded / $maxFrames - : (($estimatedFrames < $minFrames) ? $durationRounded / $minFrames : 10); + $estimatedThumbnails = $durationRounded / $defaultThumbnailInterval; + // Adjust the frame time based on the number of estimated thumbnails + $thumbnailInterval = ($estimatedThumbnails > $maxThumbnails) ? $durationRounded / $maxThumbnails + : (($estimatedThumbnails < $minThumbnails) ? $durationRounded / $minThumbnails : $defaultThumbnailInterval); $thumbnails = []; $spritesCounter = 0; - for ($time = 0.0; $time < $durationRounded && $durationRounded > 0; $time += $intervalSeconds) { + for ($time = 0.0; $time < $durationRounded && $durationRounded > 0; $time += $thumbnailInterval) { $time = round($time, 1); // Create a thumbnail from the video at the specified time and add it to the thumbnails array. $thumbnails[] = $this->generateVideoThumbnail($path, $time, $thumbnailWidth, $thumbnailHeight); - if (count($thumbnails) === $framesPerSprite || $time >= abs($durationRounded - $intervalSeconds)) { + if (count($thumbnails) === $thumbnailsPerSprite || $time >= abs($durationRounded - $thumbnailInterval)) { // Join the thumbnails into a NxN sprite - $sprite = VipsImage::arrayjoin($thumbnails, ['across' => $thumbsPerRow]); + $sprite = VipsImage::arrayjoin($thumbnails, ['across' => $thumbnailsPerRow]); // Write the sprite to buffer with quality 75 and stripped metadata $spriteBuffer = $sprite->writeToBuffer(".{$spriteFormat}", [ 'Q' => 75, diff --git a/config/videos.php b/config/videos.php index 4cdcd6967..eea11cd55 100644 --- a/config/videos.php +++ b/config/videos.php @@ -53,20 +53,20 @@ 'track_object_max_jobs_per_user' => env('VIDEOS_TRACK_OBJECT_MAX_JOBS_PER_USER', 10), /* - | Number of max frames to generate for sprites. + | Number of max thumbnails to generate for sprites. */ - 'sprites_max_frames' => 1500, + 'sprites_max_thumbnails' => 1500, /* - | Number of min frames to generate for sprites. + | Number of min thumbnails to generate for sprites. */ - 'sprites_min_frames' => 5, + 'sprites_min_thumbnails' => 5, /* - | Number of frames per sprite. Default 5x5 = 25. + | Number of thumbnails per sprite. Default 5x5 = 25. | The square root of the number must be an integer. */ - 'frames_per_sprite' => 25, + 'sprites_thumbnails_per_sprite' => 25, /* | Dimensions of the thumbnail images to create in sprites. @@ -83,5 +83,5 @@ | Standard time interval at which thumbnails should be sampled when generating sprites. | It will be adjusted dynamically during the process. */ - 'sprites_interval_seconds' => 10, + 'sprites_thumbnail_interval' => 2.5, ]; diff --git a/tests/php/Jobs/ProcessNewVideoTest.php b/tests/php/Jobs/ProcessNewVideoTest.php index c7c76d957..c5dfc2627 100644 --- a/tests/php/Jobs/ProcessNewVideoTest.php +++ b/tests/php/Jobs/ProcessNewVideoTest.php @@ -36,20 +36,18 @@ public function testHandleThumbnails() public function testGenerateSprites() { Storage::fake('video-thumbs'); - config(['videos.frames_per_sprite' => 9]); $video = VideoTest::create(['filename' => 'test.mp4']); $job = new ProcessNewVideoStub($video); $job->duration = 180; $job->handle(); $this->assertEquals(180, $video->fresh()->duration); - $expected_frames = 18; - // additional frames caused by generating regular thumbnails - $additional_frames = $job->frames - $expected_frames; - $this->assertEquals($expected_frames, $job->frames - $additional_frames); - $this->assertEquals([0.0, 10.0, 20.0, 30.0, 40.0, 50.0, - 60.0, 70.0, 80.0, 90.0, 100.0, 110.0, 120.0, 130.0, - 140.0, 150.0, 160.0, 170.0], array_slice($job->times, $additional_frames, $job->frames)); + $expectedThumbnails = 72; + $expectedIntervals = range(0, 177.5, 2.5); + // additional thumbnails caused by generating regular thumbnails + $additionalThumbnails = $job->thumbnails - $expectedThumbnails; + $this->assertEquals($expectedThumbnails, $job->thumbnails - $additionalThumbnails); + $this->assertEquals($expectedIntervals, array_slice($job->times, $additionalThumbnails, $job->thumbnails)); $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); @@ -65,11 +63,11 @@ public function testGenerateSpritesZeroDuration() $job->duration = 0; $job->handle(); - $expected_frames = 0; - // additional frames caused by generating regular thumbnails - $additional_frames = $job->frames - $expected_frames; - $this->assertEquals($expected_frames, $job->frames - $additional_frames); - $this->assertEquals([], array_slice($job->times, $additional_frames, $job->frames)); + $expectedThumbnails = 0; + // additional thumbnails caused by generating regular thumbnails + $additionalThumbnails = $job->thumbnails - $expectedThumbnails; + $this->assertEquals($expectedThumbnails, $job->thumbnails - $additionalThumbnails); + $this->assertEquals([], array_slice($job->times, $additionalThumbnails, $job->thumbnails)); $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); @@ -85,11 +83,11 @@ public function testGenerateSpritesOneSecondDuration() $job->handle(); $this->assertEquals(1, $video->fresh()->duration); - $expected_frames = 5; - // additional frames caused by generating regular thumbnails - $additional_frames = $job->frames - $expected_frames; - $this->assertEquals($expected_frames, $job->frames - $additional_frames); - $this->assertEquals([0.0, 0.2, 0.4, 0.6, 0.8], array_slice($job->times, $additional_frames, $job->frames)); + $expectedThumbnails = 5; + // additional thumbnails caused by generating regular thumbnails + $additionalThumbnails = $job->thumbnails - $expectedThumbnails; + $this->assertEquals($expectedThumbnails, $job->thumbnails - $additionalThumbnails); + $this->assertEquals([0.0, 0.2, 0.4, 0.6, 0.8], array_slice($job->times, $additionalThumbnails, $job->thumbnails)); $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); @@ -97,45 +95,43 @@ public function testGenerateSpritesOneSecondDuration() } - public function testGenerateSpritesExceedsMaxFrames() + public function testGenerateSpritesExceedsMaxThumbnails() { Storage::fake('video-thumbs'); - config(['videos.sprites_max_frames' => 25]); - config(['videos.sprites_interval_seconds' => 5]); + config(['videos.sprites_max_thumbnails' => 50]); $video = VideoTest::create(['filename' => 'test.mp4']); $job = new ProcessNewVideoStub($video); $job->duration = 130; $job->handle(); $this->assertEquals(130, $video->fresh()->duration); - $expected_frames = 25; + $expectedThumbnails = 50; + $expectedIntervals = array_map(fn ($value) => round($value, 2), range(0, 127.4, 2.6)); // additional frames caused by generating regular thumbnails - $additional_frames = $job->frames - $expected_frames; - $this->assertEquals($expected_frames, $job->frames - $additional_frames); - $this->assertEquals([0.0, 5.2, 10.4, 15.6, 20.8, 26.0, 31.2, 36.4, - 41.6, 46.8, 52.0, 57.2, 62.4, 67.6, 72.8, 78.0, 83.2, 88.4, - 93.6, 98.8, 104.0, 109.2, 114.4, 119.6, 124.8], array_slice($job->times, $additional_frames, $job->frames)); + $additionalThumbnails = $job->thumbnails - $expectedThumbnails; + $this->assertEquals($expectedThumbnails, $job->thumbnails - $additionalThumbnails); + $this->assertEquals($expectedIntervals, array_slice($job->times, $additionalThumbnails, $job->thumbnails)); $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); $this->assertTrue($disk->exists("{$fragment}/sprite_0.webp")); + $this->assertTrue($disk->exists("{$fragment}/sprite_1.webp")); } - public function testGenerateSpritesFallsBelowMinFrames() + public function testGenerateSpritesFallsBelowMinThumbnails() { Storage::fake('video-thumbs'); $video = VideoTest::create(['filename' => 'test.mp4']); $job = new ProcessNewVideoStub($video); - $job->duration = 20; + $job->duration = 5; $job->handle(); - $this->assertEquals(20, $video->fresh()->duration); - $expected_frames = 5; + $this->assertEquals(5, $video->fresh()->duration); + $expectedThumbnails = 5; // additional frames caused by generating regular thumbnails - $additional_frames = $job->frames - $expected_frames; - $this->assertEquals($expected_frames, $job->frames - $additional_frames); - $this->assertEquals([0.0, 4.0, 8.0, 12.0, 16.0], array_slice($job->times, $additional_frames, $job->frames)); - + $additionalThumbnails = $job->thumbnails - $expectedThumbnails; + $this->assertEquals($expectedThumbnails, $job->thumbnails - $additionalThumbnails); + $this->assertEquals([0.0, 1.0, 2.0, 3.0, 4.0], array_slice($job->times, $additionalThumbnails, $job->thumbnails)); $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); @@ -268,7 +264,7 @@ class ProcessNewVideoStub extends ProcessNewVideo public $duration = 0; public $codec = ''; public $times = []; - public $frames = 0; + public $thumbnails = 0; protected function getCodec($path) { @@ -283,7 +279,7 @@ protected function getVideoDuration($path) protected function generateVideoThumbnail($path, $time, $width, $height) { $this->times[] = $time; - $this->frames += 1; + $this->thumbnails += 1; return VipsImage::black(240, 138) ->embed(30, 40, 240, 138, ['extend' => Extend::WHITE]) // Extend left & top edges with white color From 1b6c587db6c0252c1f8f29d055ff352075d4b927 Mon Sep 17 00:00:00 2001 From: gaby kourie Date: Sat, 27 Jan 2024 19:36:51 +0100 Subject: [PATCH 005/100] Implement Thumbnail Preview feature on frontend --- .../Views/Videos/VideoController.php | 45 ++++-- .../js/videos/components/scrollStrip.vue | 30 ++++ .../js/videos/components/settingsTab.vue | 12 ++ .../js/videos/components/thumbnailPreview.vue | 141 ++++++++++++++++++ .../js/videos/components/videoProgress.vue | 12 ++ .../js/videos/components/videoTimeline.vue | 5 + resources/assets/js/videos/main.js | 3 +- resources/assets/js/videos/stores/settings.js | 1 + resources/assets/js/videos/videoContainer.vue | 1 + .../sass/videos/components/_thumbnail.scss | 10 ++ resources/assets/sass/videos/main.scss | 1 + resources/views/videos/show.blade.php | 8 + resources/views/videos/show/content.blade.php | 1 + .../videos/show/sidebar-settings.blade.php | 3 + 14 files changed, 262 insertions(+), 11 deletions(-) create mode 100644 resources/assets/js/videos/components/thumbnailPreview.vue create mode 100644 resources/assets/sass/videos/components/_thumbnail.scss diff --git a/app/Http/Controllers/Views/Videos/VideoController.php b/app/Http/Controllers/Views/Videos/VideoController.php index 873fd9d74..bfca29e4e 100644 --- a/app/Http/Controllers/Views/Videos/VideoController.php +++ b/app/Http/Controllers/Views/Videos/VideoController.php @@ -72,15 +72,40 @@ public function show(Request $request, $id) 'too-large' => VIDEO::ERROR_TOO_LARGE, ]); - return view('videos.show', compact( - 'user', - 'video', - 'volume', - 'videos', - 'shapes', - 'labelTrees', - 'annotationSessions', - 'errors' - )); + $fileIds = $volume->orderedFiles()->pluck('uuid', 'id'); + + if ($volume->isImageVolume()) { + $thumbUriTemplate = thumbnail_url(':uuid'); + } else { + $thumbUriTemplate = thumbnail_url(':uuid', config('videos.thumbnail_storage_disk')); + } + + $spritesThumbnailsPerSprite = config('videos.sprites_thumbnails_per_sprite'); + $spritesThumbnailInterval = config('videos.sprites_thumbnail_interval'); + $spritesMaxThumbnails = config('videos.sprites_max_thumbnails'); + $spritesMinThumbnails = config('videos.sprites_min_thumbnails'); + $spritesThumbnailWidth = config('videos.sprites_thumbnail_width'); + $spritesThumbnailHeight = config('videos.sprites_thumbnail_height'); + return view( + 'videos.show', + compact( + 'user', + 'video', + 'volume', + 'videos', + 'shapes', + 'labelTrees', + 'annotationSessions', + 'errors', + 'fileIds', + 'thumbUriTemplate', + 'spritesThumbnailsPerSprite', + 'spritesThumbnailInterval', + 'spritesMaxThumbnails', + 'spritesMinThumbnails', + 'spritesThumbnailWidth', + 'spritesThumbnailHeight' + ) + ); } } diff --git a/resources/assets/js/videos/components/scrollStrip.vue b/resources/assets/js/videos/components/scrollStrip.vue index 4636cf5c2..a6a0fdf9f 100644 --- a/resources/assets/js/videos/components/scrollStrip.vue +++ b/resources/assets/js/videos/components/scrollStrip.vue @@ -10,10 +10,19 @@ :style="scrollerStyle" @mousemove="handleUpdateHoverTime" > +
this.initialElementWidth; }, + showThumbPreview() { + return this.showThumbnailPreview && this.showThumb; + } }, methods: { updateInitialElementWidth() { @@ -210,6 +232,14 @@ export default { this.hasOverflowTop = false; this.hasOverflowBottom = false; }, + handleVideoProgressMousemove(clientX) { + this.showThumb = true; + this.clientMouseX = clientX; + this.scrollstripTop = this.$refs.scroller.getBoundingClientRect().top; + }, + hideThumbnailPreview() { + this.showThumb = false; + }, }, watch: { hoverTime(time) { diff --git a/resources/assets/js/videos/components/settingsTab.vue b/resources/assets/js/videos/components/settingsTab.vue index 3932af765..f5d6c6d38 100644 --- a/resources/assets/js/videos/components/settingsTab.vue +++ b/resources/assets/js/videos/components/settingsTab.vue @@ -15,6 +15,7 @@ export default { 'showLabelTooltip', 'showMousePosition', 'showProgressIndicator', + 'showThumbnailPreview' ], annotationOpacity: 1, showMinimap: true, @@ -23,6 +24,7 @@ export default { showMousePosition: false, playbackRate: 1.0, showProgressIndicator: true, + showThumbnailPreview: true, }; }, methods: { @@ -50,6 +52,12 @@ export default { handleHideProgressIndicator() { this.showProgressIndicator = false; }, + handleShowThumbnailPreview() { + this.showThumbnailPreview = true; + }, + handleHideThumbnailPreview() { + this.showThumbnailPreview = false; + }, }, watch: { annotationOpacity(value) { @@ -86,6 +94,10 @@ export default { this.$emit('update', 'showProgressIndicator', show); Settings.set('showProgressIndicator', show); }, + showThumbnailPreview(show) { + this.$emit('update', 'showThumbnailPreview', show); + Settings.set('showThumbnailPreview', show); + }, }, created() { this.restoreKeys.forEach((key) => { diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue new file mode 100644 index 000000000..121dbc59c --- /dev/null +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -0,0 +1,141 @@ + + + \ No newline at end of file diff --git a/resources/assets/js/videos/components/videoProgress.vue b/resources/assets/js/videos/components/videoProgress.vue index 0906a08ad..ec859d1cc 100644 --- a/resources/assets/js/videos/components/videoProgress.vue +++ b/resources/assets/js/videos/components/videoProgress.vue @@ -2,6 +2,8 @@
diff --git a/resources/assets/js/videos/components/videoTimeline.vue b/resources/assets/js/videos/components/videoTimeline.vue index d0329d8d2..fef6867d2 100644 --- a/resources/assets/js/videos/components/videoTimeline.vue +++ b/resources/assets/js/videos/components/videoTimeline.vue @@ -20,6 +20,7 @@ :duration="duration" :current-time="currentTime" :seeking="seeking" + :showThumbnailPreview="showThumbnailPreview" @seek="emitSeek" @select="emitSelect" @deselect="emitDeselect" @@ -65,6 +66,10 @@ export default { return null; }, }, + showThumbnailPreview: { + type: Boolean, + default: true, + }, }, data() { return { diff --git a/resources/assets/js/videos/main.js b/resources/assets/js/videos/main.js index 414c9ea6b..d415ad484 100644 --- a/resources/assets/js/videos/main.js +++ b/resources/assets/js/videos/main.js @@ -2,7 +2,8 @@ import './filters/videoTime'; import Navbar from './navbar'; import SearchResults from './searchResults'; import VideoContainer from './videoContainer'; - +import ThumbnailPreview from './components/thumbnailPreview'; biigle.$mount('search-results', SearchResults); biigle.$mount('video-annotations-navbar', Navbar); biigle.$mount('video-container', VideoContainer); +biigle.$mount('thumbnail-preview', ThumbnailPreview); \ No newline at end of file diff --git a/resources/assets/js/videos/stores/settings.js b/resources/assets/js/videos/stores/settings.js index 5a88bd2c4..589880ac3 100644 --- a/resources/assets/js/videos/stores/settings.js +++ b/resources/assets/js/videos/stores/settings.js @@ -7,6 +7,7 @@ let defaults = { showLabelTooltip: false, showMousePosition: false, showProgressIndicator: true, + showThumbnailPreview: true, }; export default new Settings({ diff --git a/resources/assets/js/videos/videoContainer.vue b/resources/assets/js/videos/videoContainer.vue index b1c905813..04f17523f 100644 --- a/resources/assets/js/videos/videoContainer.vue +++ b/resources/assets/js/videos/videoContainer.vue @@ -68,6 +68,7 @@ export default { showMousePosition: false, playbackRate: 1.0, showProgressIndicator: true, + showThumbnailPreview: true, }, openTab: '', urlParams: { diff --git a/resources/assets/sass/videos/components/_thumbnail.scss b/resources/assets/sass/videos/components/_thumbnail.scss new file mode 100644 index 000000000..efbc6ef8c --- /dev/null +++ b/resources/assets/sass/videos/components/_thumbnail.scss @@ -0,0 +1,10 @@ +.thumbnail-preview { + position: fixed; + z-index: 1; +} + +.thumbnail-canvas { + @extend %info-box; + padding: 0; + box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.25); +} \ No newline at end of file diff --git a/resources/assets/sass/videos/main.scss b/resources/assets/sass/videos/main.scss index 35aade99b..51c588646 100644 --- a/resources/assets/sass/videos/main.scss +++ b/resources/assets/sass/videos/main.scss @@ -53,3 +53,4 @@ @import 'components/annotationClip'; @import 'components/annotationSegment'; @import 'components/annotationKeyframe'; +@import 'components/thumbnail'; \ No newline at end of file diff --git a/resources/views/videos/show.blade.php b/resources/views/videos/show.blade.php index 09e3164a5..5d666ce6e 100644 --- a/resources/views/videos/show.blade.php +++ b/resources/views/videos/show.blade.php @@ -69,5 +69,13 @@ class="sidebar-container__content" biigle.$declare('videos.videoFilenames', {!! $videos->values() !!}); biigle.$declare('videos.user', {!! $user !!}); biigle.$declare('videos.isAdmin', @can('update', $volume) true @else false @endcan); + biigle.$declare('videos.fileUuids', {!! $fileIds !!}); + biigle.$declare('videos.thumbUri', '{{ $thumbUriTemplate }}'); + biigle.$declare('videos.spritesThumbnailsPerSprite', {!! $spritesThumbnailsPerSprite !!}); + biigle.$declare('videos.spritesThumbnailInterval', {!! $spritesThumbnailInterval !!}); + biigle.$declare('videos.spritesMaxThumbnails', {!! $spritesMaxThumbnails !!}); + biigle.$declare('videos.spritesMinThumbnails', {!! $spritesMinThumbnails !!}); + biigle.$declare('videos.spritesThumbnailWidth', {!! $spritesThumbnailWidth !!}); + biigle.$declare('videos.spritesThumbnailHeight', {!! $spritesThumbnailHeight !!}); @endpush diff --git a/resources/views/videos/show/content.blade.php b/resources/views/videos/show/content.blade.php index b6cf7a816..9f9f15d79 100644 --- a/resources/views/videos/show/content.blade.php +++ b/resources/views/videos/show/content.blade.php @@ -69,6 +69,7 @@ :seeking="seeking" :height-offset="timelineHeightOffset" :pending-annotation="pendingAnnotation" + :show-thumbnail-preview="settings.showThumbnailPreview" v-on:seek="seek" v-on:select="selectAnnotation" v-on:deselect="deselectAnnotation" diff --git a/resources/views/videos/show/sidebar-settings.blade.php b/resources/views/videos/show/sidebar-settings.blade.php index f77eb6e4c..dec80fe3c 100644 --- a/resources/views/videos/show/sidebar-settings.blade.php +++ b/resources/views/videos/show/sidebar-settings.blade.php @@ -35,6 +35,9 @@ Mouse Position
+
+ Thumbnail preview +
From 8d866b25aa51fc71ada74399bf69bb7d0c6a358e Mon Sep 17 00:00:00 2001 From: gaby kourie Date: Thu, 1 Feb 2024 20:58:12 +0100 Subject: [PATCH 006/100] Fix issue with flashing thumbnail when sprite is not found --- .../Views/Videos/VideoController.php | 6 +--- .../js/videos/components/thumbnailPreview.vue | 33 ++++++++++--------- resources/assets/js/videos/main.js | 4 +-- .../sass/videos/components/_thumbnail.scss | 2 ++ .../manual/tutorials/videos/sidebar.blade.php | 5 +++ 5 files changed, 27 insertions(+), 23 deletions(-) diff --git a/app/Http/Controllers/Views/Videos/VideoController.php b/app/Http/Controllers/Views/Videos/VideoController.php index bfca29e4e..7bcbc516a 100644 --- a/app/Http/Controllers/Views/Videos/VideoController.php +++ b/app/Http/Controllers/Views/Videos/VideoController.php @@ -74,11 +74,7 @@ public function show(Request $request, $id) $fileIds = $volume->orderedFiles()->pluck('uuid', 'id'); - if ($volume->isImageVolume()) { - $thumbUriTemplate = thumbnail_url(':uuid'); - } else { - $thumbUriTemplate = thumbnail_url(':uuid', config('videos.thumbnail_storage_disk')); - } + $thumbUriTemplate = thumbnail_url(':uuid', config('videos.thumbnail_storage_disk')); $spritesThumbnailsPerSprite = config('videos.sprites_thumbnails_per_sprite'); $spritesThumbnailInterval = config('videos.sprites_thumbnail_interval'); diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 121dbc59c..3dc2a52b5 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -2,8 +2,10 @@
+ :height="thumbnailHeight" + v-show="!spriteNotFound"> { + this.spriteNotFound = false; this.viewThumbnailPreview(); } - // can't hide the error 404 message on the browser console - // trying to use a http request to ask if the file exists and wrap it with try/catch - // does prevent the GET 404(Not Found) error - // but we get a HEAD 404(Not Found) error instead (maybe server side?) this.sprite.onerror = () => { - if (this.thumbnailPreview.style.display !== 'none') { - this.thumbnailPreview.style.display = 'none'; - } + this.spriteNotFound = true; } } }; diff --git a/resources/assets/js/videos/main.js b/resources/assets/js/videos/main.js index d415ad484..fee4837c8 100644 --- a/resources/assets/js/videos/main.js +++ b/resources/assets/js/videos/main.js @@ -2,8 +2,6 @@ import './filters/videoTime'; import Navbar from './navbar'; import SearchResults from './searchResults'; import VideoContainer from './videoContainer'; -import ThumbnailPreview from './components/thumbnailPreview'; biigle.$mount('search-results', SearchResults); biigle.$mount('video-annotations-navbar', Navbar); -biigle.$mount('video-container', VideoContainer); -biigle.$mount('thumbnail-preview', ThumbnailPreview); \ No newline at end of file +biigle.$mount('video-container', VideoContainer); \ No newline at end of file diff --git a/resources/assets/sass/videos/components/_thumbnail.scss b/resources/assets/sass/videos/components/_thumbnail.scss index efbc6ef8c..22d485b7d 100644 --- a/resources/assets/sass/videos/components/_thumbnail.scss +++ b/resources/assets/sass/videos/components/_thumbnail.scss @@ -1,5 +1,7 @@ .thumbnail-preview { position: fixed; + top: 0; + left: 0; z-index: 1; } diff --git a/resources/views/manual/tutorials/videos/sidebar.blade.php b/resources/views/manual/tutorials/videos/sidebar.blade.php index a2f628dc4..ce74e42f2 100644 --- a/resources/views/manual/tutorials/videos/sidebar.blade.php +++ b/resources/views/manual/tutorials/videos/sidebar.blade.php @@ -71,5 +71,10 @@

The mouse position switch controls the display of an additional map overlay that shows the current position of the cursor on the video in pixels.

+ +

+ The thumbnail switch controls the display of a thumbnail preview that appears when you hover your cursor over the video progress bar. The thumbnail shows a preview of the video frame at the hovered time position. +

+
@endsection From da6f0ea8babbaa8195b6d8c3f6886d7c4267ef4d Mon Sep 17 00:00:00 2001 From: gaby kourie Date: Sun, 4 Feb 2024 19:14:26 +0100 Subject: [PATCH 007/100] Add limit to sprite loading retry attempts --- .../js/videos/components/thumbnailPreview.vue | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 3dc2a52b5..4607ea889 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -49,6 +49,9 @@ export default { thumbProgressBarSpace: 150, sideButtonsWidth: 52, spritesFolderPath: null, + triedUrls: {}, + // retry sprite loading x times + retryAttempts: 2, // start with true to hide flashing black thumbnail spriteNotFound: true, // default values but will be overwritten in created() @@ -72,7 +75,16 @@ export default { methods: { updateSprite() { this.spriteIdx = Math.floor(this.hoverTime / (this.thumbnailInterval * this.thumbnailsPerSprite)); - this.sprite.src = this.spritesFolderPath + "sprite_" + this.spriteIdx + ".webp"; + let SpriteUrl = this.spritesFolderPath + "sprite_" + this.spriteIdx + ".webp"; + + if (!this.triedUrls[SpriteUrl]) { + this.triedUrls[SpriteUrl] = 0 + } + if (this.triedUrls[SpriteUrl] < this.retryAttempts) { + this.sprite.src = SpriteUrl; + } else { + this.spriteNotFound = true; + } }, viewThumbnailPreview() { // calculate the current row and column of the sprite @@ -138,6 +150,9 @@ export default { } this.sprite.onerror = () => { this.spriteNotFound = true; + if (this.sprite.src in this.triedUrls) { + this.triedUrls[this.sprite.src]++; + } } } }; From 123cc8a83216afa680a1a0146dd93838ef7f2b5c Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Wed, 26 Jun 2024 08:57:29 +0200 Subject: [PATCH 008/100] Use ffmpeg and VIPs to create sprites Replace generateVideoThumbnail by ffmpeg call, because it is faster. --- app/Jobs/ProcessNewVideo.php | 57 +++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/app/Jobs/ProcessNewVideo.php b/app/Jobs/ProcessNewVideo.php index f411d33e5..3fbd08b64 100644 --- a/app/Jobs/ProcessNewVideo.php +++ b/app/Jobs/ProcessNewVideo.php @@ -19,6 +19,8 @@ use Log; use Throwable; use VipsImage; +use Illuminate\Support\Arr; +use Symfony\Component\Process\Process; class ProcessNewVideo extends Job implements ShouldQueue { @@ -289,7 +291,6 @@ protected function getThumbnailTimes($duration) */ protected function generateSprites($path, $duration, $disk, $fragment) { - $maxThumbnails = config('videos.sprites_max_thumbnails'); $minThumbnails = config('videos.sprites_min_thumbnails'); $thumbnailsPerSprite = config('videos.sprites_thumbnails_per_sprite'); @@ -303,16 +304,45 @@ protected function generateSprites($path, $duration, $disk, $fragment) // Adjust the frame time based on the number of estimated thumbnails $thumbnailInterval = ($estimatedThumbnails > $maxThumbnails) ? $durationRounded / $maxThumbnails : (($estimatedThumbnails < $minThumbnails) ? $durationRounded / $minThumbnails : $defaultThumbnailInterval); - - $thumbnails = []; $spritesCounter = 0; - for ($time = 0.0; $time < $durationRounded && $durationRounded > 0; $time += $thumbnailInterval) { - $time = round($time, 1); - // Create a thumbnail from the video at the specified time and add it to the thumbnails array. - $thumbnails[] = $this->generateVideoThumbnail($path, $time, $thumbnailWidth, $thumbnailHeight); - if (count($thumbnails) === $thumbnailsPerSprite || $time >= abs($durationRounded - $thumbnailInterval)) { + $frameRate = 1 / $thumbnailInterval; + + $sprite_images_path = "sprite-images/{$fragment}"; + if (!($disk->exists('sprite-images') && $disk->exists($sprite_images_path))) { + $disk->makeDirectory('sprite-images'); + $disk->makeDirectory($sprite_images_path); + } + + $maybeDeleteDir = function () use ($disk, $fragment, $sprite_images_path) { + if ($disk->exists($sprite_images_path)) { + $parentDir = dirname($fragment, 2); + $disk->deleteDirectory("sprite-images/{$parentDir}"); + } + }; + + try { + // Create images from video by using ffmpeg, because it is faster than the generateVideoThumbnail method + $storageAbsolutePath = $disk->path($sprite_images_path); + $process = Process::fromShellCommandline("ffmpeg -i '{$path}' -s {$thumbnailWidth}x{$thumbnailHeight} -vf fps={$frameRate} {$storageAbsolutePath}/frame%03d.png"); + $process->run(); + + $files = $disk->files($sprite_images_path); + + $thumbnails = Arr::map($files, fn ($f) => VipsImage::newFromFile($disk->path('/').$f)); + + // Split array into sprite-chunks + $chunks = []; + $nbrChunks = ceil(count($thumbnails)/$thumbnailsPerSprite); + for ($i = 0; $i < $nbrChunks; $i++) { + // $thumbnails is cut here, so the beginning of the next chunk is always at index 0 + $chunks[] = array_splice($thumbnails, 0, $thumbnailsPerSprite); + } + + $spritesCounter = 0; + foreach ($chunks as $chunk) { // Join the thumbnails into a NxN sprite - $sprite = VipsImage::arrayjoin($thumbnails, ['across' => $thumbnailsPerRow]); + $sprite = VipsImage::arrayjoin($chunk, ['across' => $thumbnailsPerRow]); + // Write the sprite to buffer with quality 75 and stripped metadata $spriteBuffer = $sprite->writeToBuffer(".{$spriteFormat}", [ 'Q' => 75, @@ -320,9 +350,14 @@ protected function generateSprites($path, $duration, $disk, $fragment) ]); $spritePath = "{$fragment}/sprite_{$spritesCounter}.{$spriteFormat}"; $disk->put($spritePath, $spriteBuffer); - $thumbnails = []; $spritesCounter += 1; - } + }; + } catch (Exception $e) { + $maybeDeleteDir(); + throw $e; + } + + $maybeDeleteDir(); } } From 3d7c0d48fb3a6a15f2ded8f85a6f51ecd3b07da5 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Tue, 2 Jul 2024 09:45:04 +0200 Subject: [PATCH 009/100] Change sprite thumbnail size --- config/thumbnails.php | 4 ++-- config/videos.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/thumbnails.php b/config/thumbnails.php index 7a8657e55..b7e089680 100644 --- a/config/thumbnails.php +++ b/config/thumbnails.php @@ -12,8 +12,8 @@ | you are doing, since the views must work with these images. The images are always | scaled proportionally, so this values are kind of a max-width and max-height. */ - 'width' => 180, - 'height' => 135, + 'width' => 360, + 'height' => 270, /* | Thumbnail file format. Depending on your thumbnail service, different formats are diff --git a/config/videos.php b/config/videos.php index eea11cd55..cc5534e27 100644 --- a/config/videos.php +++ b/config/videos.php @@ -71,8 +71,8 @@ /* | Dimensions of the thumbnail images to create in sprites. */ - 'sprites_thumbnail_width' => 240, - 'sprites_thumbnail_height' => 138, + 'sprites_thumbnail_width' => 360, + 'sprites_thumbnail_height' => 270, /* | Sprite file format. From 5352f3b084afc6bcb7ebda8702c158a0ddd8897e Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Thu, 4 Jul 2024 11:31:35 +0200 Subject: [PATCH 010/100] Create thumbnails and sprites by using ffmpeg Images from video are generated once and are adjusted to thumbnail and sprite specifications. --- app/Jobs/ProcessNewVideo.php | 171 ++++++++++++++++++----------------- 1 file changed, 87 insertions(+), 84 deletions(-) diff --git a/app/Jobs/ProcessNewVideo.php b/app/Jobs/ProcessNewVideo.php index 3fbd08b64..84fb90bc7 100644 --- a/app/Jobs/ProcessNewVideo.php +++ b/app/Jobs/ProcessNewVideo.php @@ -6,21 +6,20 @@ use Biigle\Video; use Exception; use FFMpeg\Coordinate\Dimension; -use FFMpeg\Coordinate\TimeCode; use FFMpeg\FFMpeg; use FFMpeg\FFProbe; -use File; use FileCache; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Support\Arr; +use Illuminate\Support\Facades\File; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; use Log; +use Symfony\Component\Process\Process; use Throwable; use VipsImage; -use Illuminate\Support\Arr; -use Symfony\Component\Process\Process; class ProcessNewVideo extends Job implements ShouldQueue { @@ -145,24 +144,39 @@ public function handleFile($file, $path) } $this->video->save(); - $times = $this->getThumbnailTimes($this->video->duration); + $format = config('thumbnails.format'); $disk = Storage::disk(config('videos.thumbnail_storage_disk')); $fragment = fragment_uuid_path($this->video->uuid); - $format = config('thumbnails.format'); $width = config('thumbnails.width'); $height = config('thumbnails.height'); - try { - foreach ($times as $index => $time) { - $buffer = $this->generateVideoThumbnail($path, $time, $width, $height) - ->writeToBuffer(".{$format}", [ - 'Q' => 85, - 'strip' => true, - ]); - $disk->put("{$fragment}/{$index}.{$format}", $buffer); + $tmp = config('videos.tmp_dir'); + $tmpDir = "{$tmp}/{$fragment}"; + if (!File::exists($tmpDir)) { + File::makeDirectory($tmpDir, 0755, true, true); } + + if (!$disk->exists($tmpDir)) { + $disk->makeDirectory($fragment); + } + + // generate images from video + $this->generateImagesfromVideo($path, $this->video->duration, $tmpDir); + + // generate thumbnails + $files = glob($tmpDir."/*.{$format}"); + $this->generateVideoThumbnails($files, $disk->path($fragment.'/'), $format, $width, $height); + // generate sprites - $this->generateSprites($path, $this->video->duration, $disk, $fragment); + $this->generateSprites($disk, $tmpDir, $fragment); + + $parentDir = dirname($fragment, 2); + if ($disk->exists($tmpDir)) { + $disk->deleteDirectory($parentDir); + } + if (File::exists($tmpDir)) { + File::deleteDirectory($tmp."/{$parentDir}"); + } } catch (Exception $e) { // The video seems to be fine if it passed the previous checks. There may be // errors in the actual video data but we can ignore that and skip generating @@ -223,27 +237,36 @@ protected function getVideoDimensions($url) } /** - * Generate a thumbnail from the video at the specified time. + * Generate thumbnails from the video. * * @param string $path Path to the video file. * @param float $time Time for the thumbnail in seconds. - * @param int $width Width of the thumbnail. - * @param int $height Height of the thumbnail. * - * @return string Vips image buffer string. */ - protected function generateVideoThumbnail($path, $time, $width, $height) + protected function generateImagesfromVideo($path, $duration, $destinationPath) { - // Cache the video instance. - if (!isset($this->ffmpegVideo)) { - $this->ffmpegVideo = FFMpeg::create()->open($path); - } + $format = config('thumbnails.format'); + $maxThumbnails = config('videos.sprites_max_thumbnails'); + $minThumbnails = config('videos.sprites_min_thumbnails'); + $defaultThumbnailInterval = config('videos.sprites_thumbnail_interval'); + $durationRounded = floor($duration * 10) / 10; + $estimatedThumbnails = $durationRounded / $defaultThumbnailInterval; + // Adjust the frame time based on the number of estimated thumbnails + $thumbnailInterval = ($estimatedThumbnails > $maxThumbnails) ? $durationRounded / $maxThumbnails + : (($estimatedThumbnails < $minThumbnails) ? $durationRounded / $minThumbnails : $defaultThumbnailInterval); + $frameRate = 1 / $thumbnailInterval; + $process = Process::fromShellCommandline("ffmpeg -i '{$path}' -vf fps={$frameRate} {$destinationPath}/frame%04d.{$format}"); + $process->run(); + return $process->getExitCode(); - $buffer = $this->ffmpegVideo->frame(TimeCode::fromSeconds($time)) - ->save(null, false, true); + } - return VipsImage::thumbnail_buffer($buffer, $width, ['height' => $height, 'size' => 'force']) - ; + protected function generateVideoThumbnails($files, $thumbnailsDir, $format, $width, $height) + { + foreach ($files as $f) { + $newFilename = pathinfo($f, PATHINFO_FILENAME); + Process::fromShellCommandline("ffmpeg -i '{$f}' -s {$width}x{$height} {$thumbnailsDir}{$newFilename}.{$format}")->run(); + } } /** @@ -289,75 +312,55 @@ protected function getThumbnailTimes($duration) * @param string $fragment fragment identifier for organizing the sprites. * */ - protected function generateSprites($path, $duration, $disk, $fragment) + protected function generateSprites($disk, $tmpDir, $fragment) { - $maxThumbnails = config('videos.sprites_max_thumbnails'); - $minThumbnails = config('videos.sprites_min_thumbnails'); $thumbnailsPerSprite = config('videos.sprites_thumbnails_per_sprite'); $thumbnailsPerRow = sqrt($thumbnailsPerSprite); $thumbnailWidth = config('videos.sprites_thumbnail_width'); $thumbnailHeight = config('videos.sprites_thumbnail_height'); $spriteFormat = config('videos.sprites_format'); - $defaultThumbnailInterval = config('videos.sprites_thumbnail_interval'); - $durationRounded = floor($duration * 10) / 10; - $estimatedThumbnails = $durationRounded / $defaultThumbnailInterval; + $thumbFormat = config('thumbnails.format'); + $tmp = config('videos.tmp_dir'); // Adjust the frame time based on the number of estimated thumbnails - $thumbnailInterval = ($estimatedThumbnails > $maxThumbnails) ? $durationRounded / $maxThumbnails - : (($estimatedThumbnails < $minThumbnails) ? $durationRounded / $minThumbnails : $defaultThumbnailInterval); - $spritesCounter = 0; - $frameRate = 1 / $thumbnailInterval; - $sprite_images_path = "sprite-images/{$fragment}"; - if (!($disk->exists('sprite-images') && $disk->exists($sprite_images_path))) { - $disk->makeDirectory('sprite-images'); - $disk->makeDirectory($sprite_images_path); - } - $maybeDeleteDir = function () use ($disk, $fragment, $sprite_images_path) { - if ($disk->exists($sprite_images_path)) { - $parentDir = dirname($fragment, 2); - $disk->deleteDirectory("sprite-images/{$parentDir}"); - } - }; - - try { - // Create images from video by using ffmpeg, because it is faster than the generateVideoThumbnail method - $storageAbsolutePath = $disk->path($sprite_images_path); - $process = Process::fromShellCommandline("ffmpeg -i '{$path}' -s {$thumbnailWidth}x{$thumbnailHeight} -vf fps={$frameRate} {$storageAbsolutePath}/frame%03d.png"); - $process->run(); - - $files = $disk->files($sprite_images_path); + $sprite_images_path = "{$tmp}/sprite-images/{$fragment}"; + if (!File::exists($sprite_images_path)) { + File::makeDirectory($sprite_images_path, 0755, true); + } + $files = File::glob($tmpDir . "/*.{$thumbFormat}"); + foreach ($files as $f) { + $filename = pathinfo($f, PATHINFO_FILENAME); + Process::fromShellCommandline("ffmpeg -i '{$f}' -s {$thumbnailWidth}x{$thumbnailHeight} {$sprite_images_path}/{$filename}.{$spriteFormat}")->run(); + } - $thumbnails = Arr::map($files, fn ($f) => VipsImage::newFromFile($disk->path('/').$f)); + $files = File::glob($sprite_images_path . "/*.{$spriteFormat}"); + $thumbnails = Arr::map($files, fn ($f) => VipsImage::newFromFile($f)); - // Split array into sprite-chunks - $chunks = []; - $nbrChunks = ceil(count($thumbnails)/$thumbnailsPerSprite); - for ($i = 0; $i < $nbrChunks; $i++) { - // $thumbnails is cut here, so the beginning of the next chunk is always at index 0 - $chunks[] = array_splice($thumbnails, 0, $thumbnailsPerSprite); - } + // Split array into sprite-chunks + $chunks = []; + $nbrChunks = ceil(count($thumbnails) / $thumbnailsPerSprite); + for ($i = 0; $i < $nbrChunks; $i++) { + // $thumbnails is cut here, so the beginning of the next chunk is always at index 0 + $chunks[] = array_splice($thumbnails, 0, $thumbnailsPerSprite); + } - $spritesCounter = 0; - foreach ($chunks as $chunk) { - // Join the thumbnails into a NxN sprite - $sprite = VipsImage::arrayjoin($chunk, ['across' => $thumbnailsPerRow]); - - // Write the sprite to buffer with quality 75 and stripped metadata - $spriteBuffer = $sprite->writeToBuffer(".{$spriteFormat}", [ - 'Q' => 75, - 'strip' => true, - ]); - $spritePath = "{$fragment}/sprite_{$spritesCounter}.{$spriteFormat}"; - $disk->put($spritePath, $spriteBuffer); - $spritesCounter += 1; - }; - } catch (Exception $e) { - $maybeDeleteDir(); - throw $e; - + $spritesCounter = 0; + foreach ($chunks as $chunk) { + // Join the thumbnails into a NxN sprite + $sprite = VipsImage::arrayjoin($chunk, ['across' => $thumbnailsPerRow]); + + // Write the sprite to buffer with quality 75 and stripped metadata + $spritePath = "{$fragment}/sprite_{$spritesCounter}.{$spriteFormat}"; + $sprite->writeToFile($disk->path("{$spritePath}"), [ + 'Q' => 75, + 'strip' => true, + ]); + $spritesCounter += 1; + } + if (File::exists($sprite_images_path)) { + File::deleteDirectory("{$tmp}/sprite-images/"); } - $maybeDeleteDir(); } } From 018676e2bba235b77ff7691237492bae576f0042 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Thu, 4 Jul 2024 11:21:08 +0200 Subject: [PATCH 011/100] Add phpdoc --- app/Jobs/ProcessNewVideo.php | 37 ++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/app/Jobs/ProcessNewVideo.php b/app/Jobs/ProcessNewVideo.php index 84fb90bc7..ee9caf318 100644 --- a/app/Jobs/ProcessNewVideo.php +++ b/app/Jobs/ProcessNewVideo.php @@ -165,7 +165,7 @@ public function handleFile($file, $path) // generate thumbnails $files = glob($tmpDir."/*.{$format}"); - $this->generateVideoThumbnails($files, $disk->path($fragment.'/'), $format, $width, $height); + $this->generateVideoThumbnails($files, $disk->path($fragment.'/'), $width, $height); // generate sprites $this->generateSprites($disk, $tmpDir, $fragment); @@ -237,10 +237,12 @@ protected function getVideoDimensions($url) } /** - * Generate thumbnails from the video. + * Extract images from video. * * @param string $path Path to the video file. - * @param float $time Time for the thumbnail in seconds. + * @param float $duration Duration of video in seconds. + * @param $destinationPath Path to where images will be saved. + * @throws Exception if images cannot be extracted from video. * */ protected function generateImagesfromVideo($path, $duration, $destinationPath) @@ -261,8 +263,19 @@ protected function generateImagesfromVideo($path, $duration, $destinationPath) } - protected function generateVideoThumbnails($files, $thumbnailsDir, $format, $width, $height) + /** + * Generate thumbnails from the video images. + * + * @param $files Array of image paths. + * @param $thumbnailsDir Path to directory where thumbnails will be saved. + * @param $width Width of the thumbnail. + * @param $height Height of the thumbnail. + * @throws Exception if image cannot be resized. + * + * **/ + protected function generateVideoThumbnails($files, $thumbnailsDir, $width, $height) { + $format = config('thumbnails.format'); foreach ($files as $f) { $newFilename = pathinfo($f, PATHINFO_FILENAME); Process::fromShellCommandline("ffmpeg -i '{$f}' -s {$width}x{$height} {$thumbnailsDir}{$newFilename}.{$format}")->run(); @@ -302,17 +315,15 @@ protected function getThumbnailTimes($duration) /** * Generate sprites from a video file and save them to a storage disk. * - * This function takes a video file path, its duration, a storage disk instance, - * and a fragment identifier to generate sprites from the video at specified intervals. * Sprites are created as thumbnails of frames and organized into sprite sheets. * - * @param string $path path to the video file. - * @param float $duration duration of the video in seconds. - * @param mixed $disk storage disk instance (e.g., Laravel's Storage). - * @param string $fragment fragment identifier for organizing the sprites. + * @param $disk Storage disk where sprites will be saved. + * @param $thumbnailDir Directory where thumbnails are saved. + * @param $fragment Path where sprite frames will be saved. + * @throws Exception if images cannot be resized and transformed to webp format. * */ - protected function generateSprites($disk, $tmpDir, $fragment) + protected function generateSprites($disk, $thumbnailDir, $fragment) { $thumbnailsPerSprite = config('videos.sprites_thumbnails_per_sprite'); $thumbnailsPerRow = sqrt($thumbnailsPerSprite); @@ -321,14 +332,12 @@ protected function generateSprites($disk, $tmpDir, $fragment) $spriteFormat = config('videos.sprites_format'); $thumbFormat = config('thumbnails.format'); $tmp = config('videos.tmp_dir'); - // Adjust the frame time based on the number of estimated thumbnails - $sprite_images_path = "{$tmp}/sprite-images/{$fragment}"; if (!File::exists($sprite_images_path)) { File::makeDirectory($sprite_images_path, 0755, true); } - $files = File::glob($tmpDir . "/*.{$thumbFormat}"); + $files = File::glob($thumbnailDir . "/*.{$thumbFormat}"); foreach ($files as $f) { $filename = pathinfo($f, PATHINFO_FILENAME); Process::fromShellCommandline("ffmpeg -i '{$f}' -s {$thumbnailWidth}x{$thumbnailHeight} {$sprite_images_path}/{$filename}.{$spriteFormat}")->run(); From a8e7fe126427e94015d64dd6a6444a78cc251709 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Thu, 4 Jul 2024 11:22:24 +0200 Subject: [PATCH 012/100] Throw exception if process doesn't finish successfully --- app/Jobs/ProcessNewVideo.php | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/app/Jobs/ProcessNewVideo.php b/app/Jobs/ProcessNewVideo.php index ee9caf318..cafd4f48b 100644 --- a/app/Jobs/ProcessNewVideo.php +++ b/app/Jobs/ProcessNewVideo.php @@ -153,7 +153,7 @@ public function handleFile($file, $path) $tmp = config('videos.tmp_dir'); $tmpDir = "{$tmp}/{$fragment}"; if (!File::exists($tmpDir)) { - File::makeDirectory($tmpDir, 0755, true, true); + File::makeDirectory($tmpDir, 0755, true); } if (!$disk->exists($tmpDir)) { @@ -257,9 +257,11 @@ protected function generateImagesfromVideo($path, $duration, $destinationPath) $thumbnailInterval = ($estimatedThumbnails > $maxThumbnails) ? $durationRounded / $maxThumbnails : (($estimatedThumbnails < $minThumbnails) ? $durationRounded / $minThumbnails : $defaultThumbnailInterval); $frameRate = 1 / $thumbnailInterval; - $process = Process::fromShellCommandline("ffmpeg -i '{$path}' -vf fps={$frameRate} {$destinationPath}/frame%04d.{$format}"); - $process->run(); - return $process->getExitCode(); + $p = Process::fromShellCommandline("ffmpeg -i '{$path}' -vf fps={$frameRate} {$destinationPath}/frame%04d.{$format}"); + $p->run(); + if ($p->getExitCode() !== 0) { + throw new Exception("Process was terminated with code {$p->getExitCode()}"); + } } @@ -278,7 +280,11 @@ protected function generateVideoThumbnails($files, $thumbnailsDir, $width, $heig $format = config('thumbnails.format'); foreach ($files as $f) { $newFilename = pathinfo($f, PATHINFO_FILENAME); - Process::fromShellCommandline("ffmpeg -i '{$f}' -s {$width}x{$height} {$thumbnailsDir}{$newFilename}.{$format}")->run(); + $p = Process::fromShellCommandline("ffmpeg -i '{$f}' -s {$width}x{$height} {$thumbnailsDir}{$newFilename}.{$format}"); + $p->run(); + if ($p->getExitCode() !== 0) { + throw new Exception("Process was terminated with code {$p->getExitCode()}"); + } } } @@ -340,7 +346,11 @@ protected function generateSprites($disk, $thumbnailDir, $fragment) $files = File::glob($thumbnailDir . "/*.{$thumbFormat}"); foreach ($files as $f) { $filename = pathinfo($f, PATHINFO_FILENAME); - Process::fromShellCommandline("ffmpeg -i '{$f}' -s {$thumbnailWidth}x{$thumbnailHeight} {$sprite_images_path}/{$filename}.{$spriteFormat}")->run(); + $p = Process::fromShellCommandline("ffmpeg -i '{$f}' -s {$thumbnailWidth}x{$thumbnailHeight} {$sprite_images_path}/{$filename}.{$spriteFormat}"); + $p->run(); + if ($p->getExitCode() !== 0) { + throw new Exception("Process was terminated with code {$p->getExitCode()}"); + } } $files = File::glob($sprite_images_path . "/*.{$spriteFormat}"); @@ -353,7 +363,6 @@ protected function generateSprites($disk, $thumbnailDir, $fragment) // $thumbnails is cut here, so the beginning of the next chunk is always at index 0 $chunks[] = array_splice($thumbnails, 0, $thumbnailsPerSprite); } - $spritesCounter = 0; foreach ($chunks as $chunk) { // Join the thumbnails into a NxN sprite From d4d642de0f1681f3011a3e03b2df78b47adda512 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Thu, 4 Jul 2024 12:15:57 +0200 Subject: [PATCH 013/100] Update tests --- tests/php/Jobs/ProcessNewVideoTest.php | 87 ++++++++++---------------- 1 file changed, 33 insertions(+), 54 deletions(-) diff --git a/tests/php/Jobs/ProcessNewVideoTest.php b/tests/php/Jobs/ProcessNewVideoTest.php index c5dfc2627..39aac3c07 100644 --- a/tests/php/Jobs/ProcessNewVideoTest.php +++ b/tests/php/Jobs/ProcessNewVideoTest.php @@ -21,16 +21,16 @@ public function testHandleThumbnails() $video = VideoTest::create(['filename' => 'test.mp4']); $job = new ProcessNewVideoStub($video); $job->duration = 10; - $job->handle(); - $this->assertEquals(10, $video->fresh()->duration); - $this->assertEquals([0.5, 5, 9.5], array_slice($job->times, 0, 3)); $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); - $this->assertTrue($disk->exists("{$fragment}/0.jpg")); - $this->assertTrue($disk->exists("{$fragment}/1.jpg")); - $this->assertTrue($disk->exists("{$fragment}/2.jpg")); + $this->assertCount(5, $disk->files($fragment)); + $this->assertTrue($disk->exists("{$fragment}/0.jpg")); + $this->assertTrue($disk->exists("{$fragment}/1.jpg")); + $this->assertTrue($disk->exists("{$fragment}/2.jpg")); + $this->assertTrue($disk->exists("{$fragment}/3.jpg")); + $this->assertTrue($disk->exists("{$fragment}/sprite_0.webp")); } public function testGenerateSprites() @@ -39,20 +39,14 @@ public function testGenerateSprites() $video = VideoTest::create(['filename' => 'test.mp4']); $job = new ProcessNewVideoStub($video); $job->duration = 180; - $job->handle(); - $this->assertEquals(180, $video->fresh()->duration); - $expectedThumbnails = 72; - $expectedIntervals = range(0, 177.5, 2.5); - // additional thumbnails caused by generating regular thumbnails - $additionalThumbnails = $job->thumbnails - $expectedThumbnails; - $this->assertEquals($expectedThumbnails, $job->thumbnails - $additionalThumbnails); - $this->assertEquals($expectedIntervals, array_slice($job->times, $additionalThumbnails, $job->thumbnails)); $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); + $this->assertCount(75, $disk->files($fragment)); $this->assertTrue($disk->exists("{$fragment}/sprite_0.webp")); $this->assertTrue($disk->exists("{$fragment}/sprite_1.webp")); + $this->assertTrue($disk->exists("{$fragment}/sprite_2.webp")); } public function testGenerateSpritesZeroDuration() @@ -61,16 +55,11 @@ public function testGenerateSpritesZeroDuration() $video = VideoTest::create(['filename' => 'test.mp4']); $job = new ProcessNewVideoStub($video); $job->duration = 0; - $job->handle(); - $expectedThumbnails = 0; - // additional thumbnails caused by generating regular thumbnails - $additionalThumbnails = $job->thumbnails - $expectedThumbnails; - $this->assertEquals($expectedThumbnails, $job->thumbnails - $additionalThumbnails); - $this->assertEquals([], array_slice($job->times, $additionalThumbnails, $job->thumbnails)); $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); + $this->assertCount(0, $disk->files($fragment)); $this->assertFalse($disk->exists("{$fragment}/sprite_0.webp")); } @@ -82,15 +71,11 @@ public function testGenerateSpritesOneSecondDuration() $job->duration = 1; $job->handle(); - $this->assertEquals(1, $video->fresh()->duration); - $expectedThumbnails = 5; - // additional thumbnails caused by generating regular thumbnails - $additionalThumbnails = $job->thumbnails - $expectedThumbnails; - $this->assertEquals($expectedThumbnails, $job->thumbnails - $additionalThumbnails); - $this->assertEquals([0.0, 0.2, 0.4, 0.6, 0.8], array_slice($job->times, $additionalThumbnails, $job->thumbnails)); $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); + $this->assertCount(2, $disk->files($fragment)); + $this->assertTrue($disk->exists("{$fragment}/0.jpg")); $this->assertTrue($disk->exists("{$fragment}/sprite_0.webp")); } @@ -98,43 +83,31 @@ public function testGenerateSpritesOneSecondDuration() public function testGenerateSpritesExceedsMaxThumbnails() { Storage::fake('video-thumbs'); - config(['videos.sprites_max_thumbnails' => 50]); + config(['videos.sprites_max_thumbnails' => 5]); $video = VideoTest::create(['filename' => 'test.mp4']); $job = new ProcessNewVideoStub($video); - $job->duration = 130; - + $job->useFfmpeg = true; $job->handle(); - $this->assertEquals(130, $video->fresh()->duration); - $expectedThumbnails = 50; - $expectedIntervals = array_map(fn ($value) => round($value, 2), range(0, 127.4, 2.6)); - // additional frames caused by generating regular thumbnails - $additionalThumbnails = $job->thumbnails - $expectedThumbnails; - $this->assertEquals($expectedThumbnails, $job->thumbnails - $additionalThumbnails); - $this->assertEquals($expectedIntervals, array_slice($job->times, $additionalThumbnails, $job->thumbnails)); $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); + $this->assertCount(6, $disk->files($fragment)); $this->assertTrue($disk->exists("{$fragment}/sprite_0.webp")); - $this->assertTrue($disk->exists("{$fragment}/sprite_1.webp")); } public function testGenerateSpritesFallsBelowMinThumbnails() { Storage::fake('video-thumbs'); + config(['videos.sprites_min_thumbnails' => 10]); $video = VideoTest::create(['filename' => 'test.mp4']); $job = new ProcessNewVideoStub($video); - $job->duration = 5; + $job->useFfmpeg = true; $job->handle(); - $this->assertEquals(5, $video->fresh()->duration); - $expectedThumbnails = 5; - // additional frames caused by generating regular thumbnails - $additionalThumbnails = $job->thumbnails - $expectedThumbnails; - $this->assertEquals($expectedThumbnails, $job->thumbnails - $additionalThumbnails); - $this->assertEquals([0.0, 1.0, 2.0, 3.0, 4.0], array_slice($job->times, $additionalThumbnails, $job->thumbnails)); $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); + $this->assertCount(11, $disk->files($fragment)); $this->assertTrue($disk->exists("{$fragment}/sprite_0.webp")); } @@ -261,29 +234,35 @@ public function testHandleRemoveErrorOnSuccess() class ProcessNewVideoStub extends ProcessNewVideo { - public $duration = 0; public $codec = ''; - public $times = []; public $thumbnails = 0; + public $duration = 0; + public $useFfmpeg = false; protected function getCodec($path) { return $this->codec ?: parent::getCodec($path); } - protected function getVideoDuration($path) + protected function generateImagesfromVideo($path, $duration, $destinationPath) { - return $this->duration; - } + // Use parent method to test max and min number of thumbnail generation + if($this->useFfmpeg){ + parent::generateImagesfromVideo($path, $duration, $destinationPath); + return; + } - protected function generateVideoThumbnail($path, $time, $width, $height) - { - $this->times[] = $time; - $this->thumbnails += 1; + $defaultThumbnailInterval = config('videos.sprites_thumbnail_interval'); + $durationRounded = floor($this->duration * 10) / 10; + $estimatedThumbnails = $durationRounded / $defaultThumbnailInterval; - return VipsImage::black(240, 138) + for($i=0;$i<$estimatedThumbnails; $i++){ + $img = VipsImage::black(240, 138) ->embed(30, 40, 240, 138, ['extend' => Extend::WHITE]) // Extend left & top edges with white color ->add("#FFFFFF") ->cast("uchar"); + $img->writeToFile($destinationPath."/{$i}.jpg"); + + } } } From 034347200af522c9caa39f2af17032e7d95fe7bd Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Thu, 4 Jul 2024 12:48:18 +0200 Subject: [PATCH 014/100] Update filename of video thumbnails --- resources/assets/js/volumes/volumeContainer.vue | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/resources/assets/js/volumes/volumeContainer.vue b/resources/assets/js/volumes/volumeContainer.vue index 60a4f0b02..96fb4c320 100644 --- a/resources/assets/js/volumes/volumeContainer.vue +++ b/resources/assets/js/volumes/volumeContainer.vue @@ -273,7 +273,9 @@ export default { let thumbnailUrl; if (thumbCount > 1) { thumbnailUrl = Array.from(Array(thumbCount).keys()).map(function (i) { - return thumbUri.replace(':uuid', transformUuid(fileUuids[id]) + '/' + i); + let idx = (i+1).toString(); + let filename = "frame" + "0".repeat(4-idx.length) + idx; + return thumbUri.replace(':uuid', transformUuid(fileUuids[id]) + '/' + filename); }); } else { thumbnailUrl = thumbUri.replace(':uuid', transformUuid(fileUuids[id])); From 5888d015d02cbae24211fce0d22e93206e53680e Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Thu, 4 Jul 2024 13:29:06 +0200 Subject: [PATCH 015/100] Update thumbnaile file names to prevent sorting Change filenames because otherwise they must be sorted. --- app/Jobs/ProcessNewVideo.php | 4 +++- app/Video.php | 3 ++- resources/assets/js/volumes/volumeContainer.vue | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/app/Jobs/ProcessNewVideo.php b/app/Jobs/ProcessNewVideo.php index cafd4f48b..c4aad9385 100644 --- a/app/Jobs/ProcessNewVideo.php +++ b/app/Jobs/ProcessNewVideo.php @@ -257,7 +257,9 @@ protected function generateImagesfromVideo($path, $duration, $destinationPath) $thumbnailInterval = ($estimatedThumbnails > $maxThumbnails) ? $durationRounded / $maxThumbnails : (($estimatedThumbnails < $minThumbnails) ? $durationRounded / $minThumbnails : $defaultThumbnailInterval); $frameRate = 1 / $thumbnailInterval; - $p = Process::fromShellCommandline("ffmpeg -i '{$path}' -vf fps={$frameRate} {$destinationPath}/frame%04d.{$format}"); + + // Leading zeros are important to prevent file sorting afterwards + $p = Process::fromShellCommandline("ffmpeg -i '{$path}' -vf fps={$frameRate} {$destinationPath}/%04d.{$format}"); $p->run(); if ($p->getExitCode() !== 0) { throw new Exception("Process was terminated with code {$p->getExitCode()}"); diff --git a/app/Video.php b/app/Video.php index 4247b6ba5..47f48c3da 100644 --- a/app/Video.php +++ b/app/Video.php @@ -141,8 +141,9 @@ public function getThumbnailUrlAttribute() */ public function getThumbnailsAttribute() { + $zeros = fn($i) => str_repeat("0", 4-strlen(strval($i))); return collect(range(0, config('videos.thumbnail_count') - 1)) - ->map(fn ($i) => "{$this->uuid}/{$i}"); + ->map(fn ($i) => "{$this->uuid}/".$zeros($i)."{$i}"); } /** diff --git a/resources/assets/js/volumes/volumeContainer.vue b/resources/assets/js/volumes/volumeContainer.vue index 96fb4c320..26959c9bd 100644 --- a/resources/assets/js/volumes/volumeContainer.vue +++ b/resources/assets/js/volumes/volumeContainer.vue @@ -273,8 +273,8 @@ export default { let thumbnailUrl; if (thumbCount > 1) { thumbnailUrl = Array.from(Array(thumbCount).keys()).map(function (i) { - let idx = (i+1).toString(); - let filename = "frame" + "0".repeat(4-idx.length) + idx; + let idx = (i + 1).toString(); + let filename = "0".repeat(4 - idx.length) + idx; return thumbUri.replace(':uuid', transformUuid(fileUuids[id]) + '/' + filename); }); } else { From a98109c8c42021c8a05333b7aab0e4dd66d186ba Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Fri, 5 Jul 2024 10:49:17 +0200 Subject: [PATCH 016/100] Add enforcement of aspect ratio --- app/Jobs/ProcessNewVideo.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/Jobs/ProcessNewVideo.php b/app/Jobs/ProcessNewVideo.php index c4aad9385..4db310aaa 100644 --- a/app/Jobs/ProcessNewVideo.php +++ b/app/Jobs/ProcessNewVideo.php @@ -282,7 +282,7 @@ protected function generateVideoThumbnails($files, $thumbnailsDir, $width, $heig $format = config('thumbnails.format'); foreach ($files as $f) { $newFilename = pathinfo($f, PATHINFO_FILENAME); - $p = Process::fromShellCommandline("ffmpeg -i '{$f}' -s {$width}x{$height} {$thumbnailsDir}{$newFilename}.{$format}"); + $p = Process::fromShellCommandline("ffmpeg -i '{$f}' -vf scale={$width}:{$height}:force_original_aspect_ratio=1 {$thumbnailsDir}{$newFilename}.{$format}"); $p->run(); if ($p->getExitCode() !== 0) { throw new Exception("Process was terminated with code {$p->getExitCode()}"); @@ -340,6 +340,9 @@ protected function generateSprites($disk, $thumbnailDir, $fragment) $spriteFormat = config('videos.sprites_format'); $thumbFormat = config('thumbnails.format'); $tmp = config('videos.tmp_dir'); + $aspectRatio = $this->video->width/$this->video->height; + $paddingX = $aspectRatio >= 1 ? 0 : "(ow-iw)/2"; + $paddingY = $aspectRatio >= 1 ? "(oh-ih)/2" : 0; $sprite_images_path = "{$tmp}/sprite-images/{$fragment}"; if (!File::exists($sprite_images_path)) { @@ -348,7 +351,7 @@ protected function generateSprites($disk, $thumbnailDir, $fragment) $files = File::glob($thumbnailDir . "/*.{$thumbFormat}"); foreach ($files as $f) { $filename = pathinfo($f, PATHINFO_FILENAME); - $p = Process::fromShellCommandline("ffmpeg -i '{$f}' -s {$thumbnailWidth}x{$thumbnailHeight} {$sprite_images_path}/{$filename}.{$spriteFormat}"); + $p = Process::fromShellCommandline("ffmpeg -i '{$f}' -vf \"scale={$thumbnailWidth}:{$thumbnailHeight}:force_original_aspect_ratio=1,pad=w={$thumbnailWidth}:h={$thumbnailHeight}:{$paddingX}:{$paddingY}\" {$sprite_images_path}/{$filename}.{$spriteFormat}"); $p->run(); if ($p->getExitCode() !== 0) { throw new Exception("Process was terminated with code {$p->getExitCode()}"); From b6bc0f64427ce618481494d22513fd42e5b4cdfd Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Fri, 5 Jul 2024 10:50:18 +0200 Subject: [PATCH 017/100] Enable displaying thumbnail canvas again --- resources/assets/js/videos/components/thumbnailPreview.vue | 4 ++-- resources/assets/sass/videos/components/_thumbnail.scss | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 4607ea889..bfa463886 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -55,8 +55,8 @@ export default { // start with true to hide flashing black thumbnail spriteNotFound: true, // default values but will be overwritten in created() - thumbnailWidth: 240, - thumbnailHeight: 138, + thumbnailWidth: 360, + thumbnailHeight: 270, thumbnailsPerSprite: 25, thumbnailInterval: 2.5, estimatedThumbnails: 0, diff --git a/resources/assets/sass/videos/components/_thumbnail.scss b/resources/assets/sass/videos/components/_thumbnail.scss index 22d485b7d..8bad245da 100644 --- a/resources/assets/sass/videos/components/_thumbnail.scss +++ b/resources/assets/sass/videos/components/_thumbnail.scss @@ -1,6 +1,6 @@ .thumbnail-preview { position: fixed; - top: 0; + top: -130px; left: 0; z-index: 1; } From f51c9533a3ac690fc36d060ef7e4ef8549b7bab8 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Fri, 5 Jul 2024 11:04:12 +0200 Subject: [PATCH 018/100] Update test --- tests/php/VideoTest.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/php/VideoTest.php b/tests/php/VideoTest.php index 19bb254e1..8c89e11b7 100644 --- a/tests/php/VideoTest.php +++ b/tests/php/VideoTest.php @@ -35,29 +35,29 @@ public function testAnnotations() public function testGetThumbnailAttribute() { config(['videos.thumbnail_count' => 3]); - $this->assertStringContainsString("{$this->model->uuid}/1", $this->model->thumbnail); + $this->assertStringContainsString("{$this->model->uuid}/0001", $this->model->thumbnail); } public function testGetThumbnailUrlAttribute() { config(['videos.thumbnail_count' => 3]); - $this->assertStringContainsString("{$this->model->uuid}/1", $this->model->thumbnailUrl); + $this->assertStringContainsString("{$this->model->uuid}/0001", $this->model->thumbnailUrl); } public function testGetThumbnailsAttribute() { config(['videos.thumbnail_count' => 3]); - $this->assertContains("{$this->model->uuid}/0", $this->model->thumbnails); - $this->assertContains("{$this->model->uuid}/1", $this->model->thumbnails); - $this->assertContains("{$this->model->uuid}/2", $this->model->thumbnails); + $this->assertContains("{$this->model->uuid}/0000", $this->model->thumbnails); + $this->assertContains("{$this->model->uuid}/0001", $this->model->thumbnails); + $this->assertContains("{$this->model->uuid}/0002", $this->model->thumbnails); } public function testGetThumbnailsUrlAttribute() { config(['videos.thumbnail_count' => 3]); - $this->assertStringContainsString("{$this->model->uuid}/0", $this->model->thumbnailsUrl[0]); - $this->assertStringContainsString("{$this->model->uuid}/1", $this->model->thumbnailsUrl[1]); - $this->assertStringContainsString("{$this->model->uuid}/2", $this->model->thumbnailsUrl[2]); + $this->assertStringContainsString("{$this->model->uuid}/0000", $this->model->thumbnailsUrl[0]); + $this->assertStringContainsString("{$this->model->uuid}/0001", $this->model->thumbnailsUrl[1]); + $this->assertStringContainsString("{$this->model->uuid}/0002", $this->model->thumbnailsUrl[2]); } public function testGetErrorAttribute() From c3575518751002250bbef8ceb09ecce58a130041 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Tue, 9 Jul 2024 08:31:22 +0200 Subject: [PATCH 019/100] Add a test to check if temp dirs are deleted --- tests/php/Jobs/ProcessNewVideoTest.php | 27 +++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/php/Jobs/ProcessNewVideoTest.php b/tests/php/Jobs/ProcessNewVideoTest.php index 39aac3c07..e0029dcfa 100644 --- a/tests/php/Jobs/ProcessNewVideoTest.php +++ b/tests/php/Jobs/ProcessNewVideoTest.php @@ -11,6 +11,7 @@ use Storage; use TestCase; use VipsImage; +use Illuminate\Support\Facades\File; class ProcessNewVideoTest extends TestCase { @@ -19,54 +20,67 @@ public function testHandleThumbnails() Storage::fake('video-thumbs'); config(['videos.thumbnail_count' => 3]); $video = VideoTest::create(['filename' => 'test.mp4']); + $tmp = config('videos.tmp_dir'); $job = new ProcessNewVideoStub($video); $job->duration = 10; $job->handle(); $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); + $parentDir = dirname($fragment, 2); $this->assertCount(5, $disk->files($fragment)); $this->assertTrue($disk->exists("{$fragment}/0.jpg")); $this->assertTrue($disk->exists("{$fragment}/1.jpg")); $this->assertTrue($disk->exists("{$fragment}/2.jpg")); $this->assertTrue($disk->exists("{$fragment}/3.jpg")); - $this->assertTrue($disk->exists("{$fragment}/sprite_0.webp")); + $this->assertTrue($disk->exists("{$fragment}/sprite_0.webp")); + $this->assertFalse(File::exists("{$tmp}/{$parentDir}")); + $this->assertFalse(File::exists("{$tmp}/sprite-images")); } public function testGenerateSprites() { Storage::fake('video-thumbs'); $video = VideoTest::create(['filename' => 'test.mp4']); + $tmp = config('videos.tmp_dir'); $job = new ProcessNewVideoStub($video); $job->duration = 180; $job->handle(); $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); + $parentDir = dirname($fragment, 2); $this->assertCount(75, $disk->files($fragment)); $this->assertTrue($disk->exists("{$fragment}/sprite_0.webp")); $this->assertTrue($disk->exists("{$fragment}/sprite_1.webp")); $this->assertTrue($disk->exists("{$fragment}/sprite_2.webp")); + $this->assertFalse(File::exists("{$tmp}/{$parentDir}")); + $this->assertFalse(File::exists("{$tmp}/sprite-images")); } public function testGenerateSpritesZeroDuration() { Storage::fake('video-thumbs'); $video = VideoTest::create(['filename' => 'test.mp4']); + $tmp = config('videos.tmp_dir'); $job = new ProcessNewVideoStub($video); $job->duration = 0; $job->handle(); $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); + $parentDir = dirname($fragment, 2); $this->assertCount(0, $disk->files($fragment)); $this->assertFalse($disk->exists("{$fragment}/sprite_0.webp")); + $this->assertFalse(File::exists("{$tmp}/{$parentDir}")); + $this->assertFalse(File::exists("{$tmp}/sprite-images")); } public function testGenerateSpritesOneSecondDuration() { Storage::fake('video-thumbs'); $video = VideoTest::create(['filename' => 'test.mp4']); + $tmp = config('videos.tmp_dir'); $job = new ProcessNewVideoStub($video); $job->duration = 1; @@ -75,8 +89,11 @@ public function testGenerateSpritesOneSecondDuration() $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); $this->assertCount(2, $disk->files($fragment)); + $parentDir = dirname($fragment, 2); $this->assertTrue($disk->exists("{$fragment}/0.jpg")); $this->assertTrue($disk->exists("{$fragment}/sprite_0.webp")); + $this->assertFalse(File::exists("{$tmp}/{$parentDir}")); + $this->assertFalse(File::exists("{$tmp}/sprite-images")); } @@ -85,14 +102,18 @@ public function testGenerateSpritesExceedsMaxThumbnails() Storage::fake('video-thumbs'); config(['videos.sprites_max_thumbnails' => 5]); $video = VideoTest::create(['filename' => 'test.mp4']); + $tmp = config('videos.tmp_dir'); $job = new ProcessNewVideoStub($video); $job->useFfmpeg = true; $job->handle(); $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); + $parentDir = dirname($fragment, 2); $this->assertCount(6, $disk->files($fragment)); $this->assertTrue($disk->exists("{$fragment}/sprite_0.webp")); + $this->assertFalse(File::exists("{$tmp}/{$parentDir}")); + $this->assertFalse(File::exists("{$tmp}/sprite-images")); } public function testGenerateSpritesFallsBelowMinThumbnails() @@ -100,6 +121,7 @@ public function testGenerateSpritesFallsBelowMinThumbnails() Storage::fake('video-thumbs'); config(['videos.sprites_min_thumbnails' => 10]); $video = VideoTest::create(['filename' => 'test.mp4']); + $tmp = config('videos.tmp_dir'); $job = new ProcessNewVideoStub($video); $job->useFfmpeg = true; @@ -107,8 +129,11 @@ public function testGenerateSpritesFallsBelowMinThumbnails() $disk = Storage::disk('video-thumbs'); $fragment = fragment_uuid_path($video->uuid); + $parentDir = dirname($fragment, 2); $this->assertCount(11, $disk->files($fragment)); $this->assertTrue($disk->exists("{$fragment}/sprite_0.webp")); + $this->assertFalse(File::exists("{$tmp}/{$parentDir}")); + $this->assertFalse(File::exists("{$tmp}/sprite-images")); } public function testHandleNotFound() From 96f4c36d21d9ffd5306a957d25efd1c12d84f8f7 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Mon, 8 Jul 2024 13:39:12 +0200 Subject: [PATCH 020/100] Sample video thumbnails uniformly --- .../Views/Volumes/VolumeController.php | 34 +++++++++++++++---- .../assets/js/volumes/volumeContainer.vue | 12 +++++-- resources/views/volumes/show.blade.php | 1 + 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/app/Http/Controllers/Views/Volumes/VolumeController.php b/app/Http/Controllers/Views/Volumes/VolumeController.php index 17266ecea..cad3866d2 100644 --- a/app/Http/Controllers/Views/Volumes/VolumeController.php +++ b/app/Http/Controllers/Views/Volumes/VolumeController.php @@ -2,17 +2,19 @@ namespace Biigle\Http\Controllers\Views\Volumes; -use Biigle\Http\Controllers\Views\Controller; -use Biigle\LabelTree; -use Biigle\MediaType; -use Biigle\Modules\UserDisks\UserDisk; -use Biigle\Modules\UserStorage\UserStorageServiceProvider; -use Biigle\Project; use Biigle\Role; use Biigle\User; use Biigle\Volume; use Carbon\Carbon; +use Biigle\Project; +use Biigle\LabelTree; +use Biigle\MediaType; +use Illuminate\Support\Arr; use Illuminate\Http\Request; +use Illuminate\Support\Facades\Log; +use Biigle\Modules\UserDisks\UserDisk; +use Biigle\Http\Controllers\Views\Controller; +use Biigle\Modules\UserStorage\UserStorageServiceProvider; class VolumeController extends Controller { @@ -101,8 +103,25 @@ public function index(Request $request, $id) if ($volume->isImageVolume()) { $thumbUriTemplate = thumbnail_url(':uuid'); + $nbrThumbnails = []; } else { $thumbUriTemplate = thumbnail_url(':uuid', config('videos.thumbnail_storage_disk')); + + // Compute number of generated thumbnails for each file + $maxThumbnails = config('videos.sprites_max_thumbnails'); + $minThumbnails = config('videos.sprites_min_thumbnails'); + $defaultThumbnailInterval = config('videos.sprites_thumbnail_interval'); + $nbrThumbnails = $volume->files->mapWithKeys(function ($f) use ($defaultThumbnailInterval, $minThumbnails, $maxThumbnails) { + $duration = floor($f->duration); + $estimatedThumbs = $duration / $defaultThumbnailInterval; + if ($estimatedThumbs < $minThumbnails) { + return [$f->id => $minThumbnails]; + } + if ($estimatedThumbs > $maxThumbnails) { + return [$f->id => $maxThumbnails]; + } + return [$f->id => $estimatedThumbs]; + })->toArray(); } $type = $volume->mediaType->name; @@ -113,7 +132,8 @@ public function index(Request $request, $id) 'projects', 'fileIds', 'thumbUriTemplate', - 'type' + 'type', + 'nbrThumbnails', )); } diff --git a/resources/assets/js/volumes/volumeContainer.vue b/resources/assets/js/volumes/volumeContainer.vue index 26959c9bd..b684cfaf1 100644 --- a/resources/assets/js/volumes/volumeContainer.vue +++ b/resources/assets/js/volumes/volumeContainer.vue @@ -265,6 +265,7 @@ export default { let fileUuids = biigle.$require('volumes.fileUuids'); let thumbUri = biigle.$require('volumes.thumbUri'); let thumbCount = biigle.$require('volumes.thumbCount'); + let nbrSavedThumbs = biigle.$require('volumes.nbrThumbnails'); let annotateUri = biigle.$require('volumes.annotateUri'); let infoUri = biigle.$require('volumes.infoUri'); // Do this here instead of a computed property so the file objects get @@ -272,9 +273,14 @@ export default { this.files = this.fileIds.map(function (id) { let thumbnailUrl; if (thumbCount > 1) { - thumbnailUrl = Array.from(Array(thumbCount).keys()).map(function (i) { - let idx = (i + 1).toString(); - let filename = "0".repeat(4 - idx.length) + idx; + let nbrThumbs = nbrSavedThumbs[id]; + let length = thumbCount < nbrThumbs ? thumbCount : nbrThumbs; + // Sample thumbnails uniformly + let step = thumbCount < nbrThumbs ? Math.floor(nbrThumbs / thumbCount) : 1; + thumbnailUrl = Array.from(Array(length).keys()).map(function (i) { + let fileIdx = (i === 0 ? 1 : i * step + 1).toString(); + // Files start with leading zeros e.g 0001.jpg or 0010.jpg + let filename = "0".repeat(4 - fileIdx.length) + fileIdx; return thumbUri.replace(':uuid', transformUuid(fileUuids[id]) + '/' + filename); }); } else { diff --git a/resources/views/volumes/show.blade.php b/resources/views/volumes/show.blade.php index 467b0612d..f1196ac89 100644 --- a/resources/views/volumes/show.blade.php +++ b/resources/views/volumes/show.blade.php @@ -5,6 +5,7 @@ @push('scripts') \ No newline at end of file + diff --git a/resources/assets/sass/videos/components/_thumbnail.scss b/resources/assets/sass/videos/components/_thumbnail.scss index 22d485b7d..7f9a5b2fc 100644 --- a/resources/assets/sass/videos/components/_thumbnail.scss +++ b/resources/assets/sass/videos/components/_thumbnail.scss @@ -3,10 +3,11 @@ top: 0; left: 0; z-index: 1; + pointer-events: none; } .thumbnail-canvas { @extend %info-box; padding: 0; box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.25); -} \ No newline at end of file +} From ede5309bfde3491f6c02c9bb6902c6e3211ff324 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Fri, 28 Jun 2024 10:08:06 +0200 Subject: [PATCH 046/100] Fix thumbnail preview which didn't change Use video ID prop instead of biigle.require("videoId") because the latter is not updated after switiching videos. --- resources/assets/js/videos/components/scrollStrip.vue | 5 +++++ resources/assets/js/videos/components/thumbnailPreview.vue | 7 +++++-- resources/assets/js/videos/components/videoTimeline.vue | 5 +++++ resources/views/videos/show/content.blade.php | 1 + 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/resources/assets/js/videos/components/scrollStrip.vue b/resources/assets/js/videos/components/scrollStrip.vue index a6a0fdf9f..6e307c494 100644 --- a/resources/assets/js/videos/components/scrollStrip.vue +++ b/resources/assets/js/videos/components/scrollStrip.vue @@ -15,6 +15,7 @@ :hoverTime="hoverTime" :clientMouseX="clientMouseX" :scrollstripTop="scrollstripTop" + :videoId="videoId" v-if="showThumb && showThumbPreview" > Date: Tue, 16 Jul 2024 08:19:58 +0200 Subject: [PATCH 047/100] Fix thumbnail dimensions which were not updated --- .../Views/Videos/VideoController.php | 24 ++++++++++--------- .../js/videos/components/thumbnailPreview.vue | 7 ++++-- resources/views/videos/show.blade.php | 3 +-- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/app/Http/Controllers/Views/Videos/VideoController.php b/app/Http/Controllers/Views/Videos/VideoController.php index b944b7e72..fcb6b5fe4 100644 --- a/app/Http/Controllers/Views/Videos/VideoController.php +++ b/app/Http/Controllers/Views/Videos/VideoController.php @@ -83,15 +83,18 @@ public function show(Request $request, $id) $configWidth = config('thumbnails.width'); $configHeight = config('thumbnails.height'); - // Compute thumbnail's size - if (!$video->error && $video->height !== 0 && $video->width !== 0) { - $ratio = $video->width / $video->height; - $spritesThumbnailWidth = $ratio >= 1 ? $configWidth : ceil($configHeight * $ratio); - $spritesThumbnailHeight = $ratio >= 1 ? ceil($configWidth / $ratio) : $configHeight; - } else { - $spritesThumbnailWidth = $configWidth; - $spritesThumbnailHeight = $configHeight; - } + $videoThumbSizes = $volume->videos->mapWithKeys(function ($video) use ($configWidth, $configHeight) { + // Compute thumbnail's size + if (!$video->error && $video->height !== 0 && $video->width !== 0) { + $ratio = $video->width / $video->height; + $spritesThumbnailWidth = $ratio >= 1 ? $configWidth : ceil($configHeight * $ratio); + $spritesThumbnailHeight = $ratio >= 1 ? ceil($configWidth / $ratio) : $configHeight; + } else { + $spritesThumbnailWidth = $configWidth; + $spritesThumbnailHeight = $configHeight; + } + return [$video->id => ['w' => $spritesThumbnailWidth, 'h' => $spritesThumbnailHeight]]; + }); return view( 'videos.show', @@ -110,8 +113,7 @@ public function show(Request $request, $id) 'spritesThumbnailInterval', 'spritesMaxThumbnails', 'spritesMinThumbnails', - 'spritesThumbnailWidth', - 'spritesThumbnailHeight' + 'videoThumbSizes', ) ); } diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index e13ce809d..e01743fa7 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -61,6 +61,7 @@ export default { thumbnailsPerSprite: 25, thumbnailInterval: 2.5, estimatedThumbnails: 0, + thumbnailSizes: [], }; }, computed: { @@ -138,8 +139,10 @@ export default { created() { this.setSpritesFolderpath(); this.updateThumbnailInterval(); - this.thumbnailWidth = biigle.$require('videos.spritesThumbnailWidth'); - this.thumbnailHeight = biigle.$require('videos.spritesThumbnailHeight'); + this.thumbnailSizes = biigle.$require('videos.thumbnailSizes'); + let thumbSize = this.thumbnailSizes[this.videoId]; + this.thumbnailWidth = thumbSize['w']; + this.thumbnailHeight = thumbSize['h']; this.thumbnailsPerSprite = biigle.$require('videos.spritesThumbnailsPerSprite'); }, mounted() { diff --git a/resources/views/videos/show.blade.php b/resources/views/videos/show.blade.php index 5d666ce6e..6e5e33dcb 100644 --- a/resources/views/videos/show.blade.php +++ b/resources/views/videos/show.blade.php @@ -75,7 +75,6 @@ class="sidebar-container__content" biigle.$declare('videos.spritesThumbnailInterval', {!! $spritesThumbnailInterval !!}); biigle.$declare('videos.spritesMaxThumbnails', {!! $spritesMaxThumbnails !!}); biigle.$declare('videos.spritesMinThumbnails', {!! $spritesMinThumbnails !!}); - biigle.$declare('videos.spritesThumbnailWidth', {!! $spritesThumbnailWidth !!}); - biigle.$declare('videos.spritesThumbnailHeight', {!! $spritesThumbnailHeight !!}); + biigle.$declare('videos.thumbnailSizes', {!! collect($videoThumbSizes) !!}); @endpush From f50e916b5167a2a714e77d85bffdf3cfe618426c Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Tue, 16 Jul 2024 08:49:15 +0200 Subject: [PATCH 048/100] Fix div by zero error for unprocessed videos --- .../Views/Videos/VideoController.php | 2 +- .../Views/Videos/VideoControllerTest.php | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/app/Http/Controllers/Views/Videos/VideoController.php b/app/Http/Controllers/Views/Videos/VideoController.php index fcb6b5fe4..ff58c377d 100644 --- a/app/Http/Controllers/Views/Videos/VideoController.php +++ b/app/Http/Controllers/Views/Videos/VideoController.php @@ -85,7 +85,7 @@ public function show(Request $request, $id) $videoThumbSizes = $volume->videos->mapWithKeys(function ($video) use ($configWidth, $configHeight) { // Compute thumbnail's size - if (!$video->error && $video->height !== 0 && $video->width !== 0) { + if (!$video->error && $video->height && $video->width) { $ratio = $video->width / $video->height; $spritesThumbnailWidth = $ratio >= 1 ? $configWidth : ceil($configHeight * $ratio); $spritesThumbnailHeight = $ratio >= 1 ? ceil($configWidth / $ratio) : $configHeight; diff --git a/tests/php/Http/Controllers/Views/Videos/VideoControllerTest.php b/tests/php/Http/Controllers/Views/Videos/VideoControllerTest.php index 0e77d348e..0aaf8afe8 100644 --- a/tests/php/Http/Controllers/Views/Videos/VideoControllerTest.php +++ b/tests/php/Http/Controllers/Views/Videos/VideoControllerTest.php @@ -11,7 +11,7 @@ class VideoControllerTest extends ApiTestCase public function testShow() { $id = $this->volume(['media_type_id' => MediaType::videoId()])->id; - $video = VideoTest::create(['volume_id' => $id, 'height' => 3, 'width' => 4]); + $video = VideoTest::create(['volume_id' => $id]); $this->beUser(); $this->get('videos/999/annotations')->assertStatus(404); @@ -26,4 +26,19 @@ public function testShowRedirect() $this->beUser(); $this->get('videos/999')->assertRedirect('/videos/999/annotations'); } + + public function testVideoWithoutDimensions() + { + $this->beGuest(); + $id = $this->volume(['media_type_id' => MediaType::videoId()])->id; + + $video = VideoTest::create(['volume_id' => $id, 'height' => NULL, 'width' => NULL]); + $this->get("videos/{$video->id}/annotations")->assertStatus(200); + + $video = VideoTest::create(['volume_id' => $id, 'height' => 0, 'width' => 0]); + $this->get("videos/{$video->id}/annotations")->assertStatus(200); + + $video = VideoTest::create(['volume_id' => $id]); + $this->get("videos/{$video->id}/annotations")->assertStatus(200); + } } From 213b2a49ba0d5b61bcf09251dfdc24b6432b6036 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Tue, 16 Jul 2024 10:15:03 +0200 Subject: [PATCH 049/100] Use Process facade rather than Symfony class directly This can be called with forever() which is crucial here. --- app/Jobs/ProcessNewVideo.php | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/app/Jobs/ProcessNewVideo.php b/app/Jobs/ProcessNewVideo.php index e29f85dea..6baa36e72 100644 --- a/app/Jobs/ProcessNewVideo.php +++ b/app/Jobs/ProcessNewVideo.php @@ -13,10 +13,10 @@ use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Facades\File; +use Illuminate\Support\Facades\Process; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; use Log; -use Symfony\Component\Process\Process; use Throwable; use VipsImage; @@ -245,12 +245,9 @@ protected function extractImagesfromVideo($path, $duration, $destinationPath) $frameRate = 1 / $thumbnailInterval; // Leading zeros are important to prevent file sorting afterwards - $p = Process::fromShellCommandline("ffmpeg -i '{$path}' -vf fps={$frameRate} {$destinationPath}/%04d.{$format}"); - $p->run(); - if ($p->getExitCode() !== 0) { - throw new Exception("Process was terminated with code {$p->getExitCode()}"); - } - + Process::forever() + ->run("ffmpeg -i '{$path}' -vf fps={$frameRate} {$destinationPath}/%04d.{$format}") + ->throw(); } public function generateVideoThumbnails($disk, $fragment, $tmpDir) From 0f4c9b4bdfdd903f402733bafe3319ca85cdff6e Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Wed, 17 Jul 2024 08:05:59 +0200 Subject: [PATCH 050/100] Compute thumbnail size during preview generation --- .../Views/Videos/VideoController.php | 16 ------------ .../js/videos/components/thumbnailPreview.vue | 26 +++++++++++++------ resources/views/videos/show.blade.php | 1 - 3 files changed, 18 insertions(+), 25 deletions(-) diff --git a/app/Http/Controllers/Views/Videos/VideoController.php b/app/Http/Controllers/Views/Videos/VideoController.php index ff58c377d..fa5a2e2bc 100644 --- a/app/Http/Controllers/Views/Videos/VideoController.php +++ b/app/Http/Controllers/Views/Videos/VideoController.php @@ -80,21 +80,6 @@ public function show(Request $request, $id) $spritesThumbnailInterval = config('videos.sprites_thumbnail_interval'); $spritesMaxThumbnails = config('videos.sprites_max_thumbnails'); $spritesMinThumbnails = config('videos.thumbnail_count'); - $configWidth = config('thumbnails.width'); - $configHeight = config('thumbnails.height'); - - $videoThumbSizes = $volume->videos->mapWithKeys(function ($video) use ($configWidth, $configHeight) { - // Compute thumbnail's size - if (!$video->error && $video->height && $video->width) { - $ratio = $video->width / $video->height; - $spritesThumbnailWidth = $ratio >= 1 ? $configWidth : ceil($configHeight * $ratio); - $spritesThumbnailHeight = $ratio >= 1 ? ceil($configWidth / $ratio) : $configHeight; - } else { - $spritesThumbnailWidth = $configWidth; - $spritesThumbnailHeight = $configHeight; - } - return [$video->id => ['w' => $spritesThumbnailWidth, 'h' => $spritesThumbnailHeight]]; - }); return view( 'videos.show', @@ -113,7 +98,6 @@ public function show(Request $request, $id) 'spritesThumbnailInterval', 'spritesMaxThumbnails', 'spritesMinThumbnails', - 'videoThumbSizes', ) ); } diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index e01743fa7..0ba14d74c 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -61,7 +61,6 @@ export default { thumbnailsPerSprite: 25, thumbnailInterval: 2.5, estimatedThumbnails: 0, - thumbnailSizes: [], }; }, computed: { @@ -75,6 +74,13 @@ export default { transform: `translate(${left}px, -100%)`, top: `${top}px`, }; + }, + spriteGridInfo() { + let nbrThumbnailsOnSprite = this.estimatedThumbnails - this.spriteIdx * this.thumbnailsPerSprite; + nbrThumbnailsOnSprite = nbrThumbnailsOnSprite > this.thumbnailsPerSprite ? this.thumbnailsPerSprite : nbrThumbnailsOnSprite; + let nbrCols = Math.sqrt(this.thumbnailsPerSprite); + let nbrRows = Math.ceil(nbrThumbnailsOnSprite / nbrCols); + return [nbrCols, nbrRows]; } }, methods: { @@ -130,6 +136,16 @@ export default { let fileUuid = fileUuids[this.videoId]; this.spritesFolderPath = thumbUri.replace(':uuid', transformUuid(fileUuid) + '/').replace('.jpg', ''); }, + initDimensions() { + let nbrCols = this.spriteGridInfo[0]; + let nbrRows = this.spriteGridInfo[1]; + this.thumbnailWidth = this.sprite.width / nbrCols; + this.thumbnailHeight = this.sprite.height / nbrRows; + this.canvasWidth = Math.ceil(this.thumbnailWidth / 2); + this.canvasHeight = Math.ceil(this.thumbnailHeight / 2); + this.thumbnailCanvas.width = this.canvasWidth; + this.thumbnailCanvas.height = this.canvasHeight; + } }, watch: { hoverTime() { @@ -140,21 +156,15 @@ export default { this.setSpritesFolderpath(); this.updateThumbnailInterval(); this.thumbnailSizes = biigle.$require('videos.thumbnailSizes'); - let thumbSize = this.thumbnailSizes[this.videoId]; - this.thumbnailWidth = thumbSize['w']; - this.thumbnailHeight = thumbSize['h']; this.thumbnailsPerSprite = biigle.$require('videos.spritesThumbnailsPerSprite'); }, mounted() { this.thumbnailPreview = this.$refs.thumbnailPreview; this.thumbnailCanvas = this.$refs.thumbnailCanvas; - this.canvasWidth = Math.ceil(this.thumbnailWidth / 2); - this.canvasHeight = Math.ceil(this.thumbnailHeight / 2); - this.thumbnailCanvas.width = this.canvasWidth; - this.thumbnailCanvas.height = this.canvasHeight; this.updateSprite(); this.sprite.onload = () => { this.spriteNotFound = false; + this.initDimensions(); this.viewThumbnailPreview(); } this.sprite.onerror = () => { diff --git a/resources/views/videos/show.blade.php b/resources/views/videos/show.blade.php index 6e5e33dcb..1883fa1f5 100644 --- a/resources/views/videos/show.blade.php +++ b/resources/views/videos/show.blade.php @@ -75,6 +75,5 @@ class="sidebar-container__content" biigle.$declare('videos.spritesThumbnailInterval', {!! $spritesThumbnailInterval !!}); biigle.$declare('videos.spritesMaxThumbnails', {!! $spritesMaxThumbnails !!}); biigle.$declare('videos.spritesMinThumbnails', {!! $spritesMinThumbnails !!}); - biigle.$declare('videos.thumbnailSizes', {!! collect($videoThumbSizes) !!}); @endpush From 414dcc430ea238dce9856bc8b7e9eb6524dd9ef7 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Wed, 17 Jul 2024 08:10:44 +0200 Subject: [PATCH 051/100] Revert changes in test class from f50e916b --- .../Views/Videos/VideoControllerTest.php | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/php/Http/Controllers/Views/Videos/VideoControllerTest.php b/tests/php/Http/Controllers/Views/Videos/VideoControllerTest.php index 0aaf8afe8..7ff4e9619 100644 --- a/tests/php/Http/Controllers/Views/Videos/VideoControllerTest.php +++ b/tests/php/Http/Controllers/Views/Videos/VideoControllerTest.php @@ -26,19 +26,4 @@ public function testShowRedirect() $this->beUser(); $this->get('videos/999')->assertRedirect('/videos/999/annotations'); } - - public function testVideoWithoutDimensions() - { - $this->beGuest(); - $id = $this->volume(['media_type_id' => MediaType::videoId()])->id; - - $video = VideoTest::create(['volume_id' => $id, 'height' => NULL, 'width' => NULL]); - $this->get("videos/{$video->id}/annotations")->assertStatus(200); - - $video = VideoTest::create(['volume_id' => $id, 'height' => 0, 'width' => 0]); - $this->get("videos/{$video->id}/annotations")->assertStatus(200); - - $video = VideoTest::create(['volume_id' => $id]); - $this->get("videos/{$video->id}/annotations")->assertStatus(200); - } } From a3177aa36a2b812c5bd68fdec096d270b4424ceb Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Wed, 17 Jul 2024 10:21:47 +0200 Subject: [PATCH 052/100] Move hover time to thumbnail preview --- .../js/videos/components/currentTime.vue | 16 ---- .../js/videos/components/thumbnailPreview.vue | 74 +++++++++++++++++-- 2 files changed, 69 insertions(+), 21 deletions(-) diff --git a/resources/assets/js/videos/components/currentTime.vue b/resources/assets/js/videos/components/currentTime.vue index bb8095ca7..157b31345 100644 --- a/resources/assets/js/videos/components/currentTime.vue +++ b/resources/assets/js/videos/components/currentTime.vue @@ -8,11 +8,6 @@ - @@ -26,10 +21,6 @@ export default { type: Number, required: true, }, - hoverTime: { - type: Number, - default: 0, - }, seeking: { type: Boolean, default: false, @@ -42,18 +33,11 @@ export default { currentTimeText() { return Vue.filter('videoTime')(this.currentTime); }, - hoverTimeText() { - return Vue.filter('videoTime')(this.hoverTime); - }, classObject() { return { 'current-time--seeking': this.seeking, - 'current-time--hover': this.showHoverTime, }; }, - showHoverTime() { - return this.hoverTime !== 0; - }, }, }; diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 0ba14d74c..99e2ba9d6 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -6,8 +6,14 @@ v-show="!spriteNotFound"> + ref="thumbnailCanvas" + v-show="!spriteNotFound"> + + + @@ -61,13 +67,18 @@ export default { thumbnailsPerSprite: 25, thumbnailInterval: 2.5, estimatedThumbnails: 0, + fontSize: 14.5, + hovertimeCanvas: null, + hoverTimeBarHeight: 20, + hoverTimeBarWidth: 120, }; }, computed: { thumbnailStyle() { + let width = this.spriteNotFound ? this.hoverTimeBarWidth : this.canvasWidth; let left = Math.min( - this.clientMouseX - this.canvasWidth / 2, - window.innerWidth - this.canvasWidth - this.sideButtonsWidth + this.clientMouseX - width / 2, + window.innerWidth - width - this.sideButtonsWidth ); let top = this.scrollstripTop - this.thumbProgressBarSpace; return { @@ -81,6 +92,22 @@ export default { let nbrCols = Math.sqrt(this.thumbnailsPerSprite); let nbrRows = Math.ceil(nbrThumbnailsOnSprite / nbrCols); return [nbrCols, nbrRows]; + }, + hoverTimeText() { + return Vue.filter('videoTime')(this.hoverTime); + }, + hoverTimeStyle() { + return { 'font': `bold ${this.fontSize}px Sans-Serif`, 'color': '#cccccc', 'bgColor': '#222' }; + }, + xtext() { + return this.canvasWidth / 2; + }, + ytext() { + // compute font size in pixel + return this.canvasHeight + this.fontSizeInPx + (this.hoverTimeBarHeight - this.fontSizeInPx) / 2; + }, + fontSizeInPx() { + return this.fontSize * (72 / 96); } }, methods: { @@ -113,6 +140,15 @@ export default { // draw the current thumbnail to the canvas let context = this.thumbnailCanvas.getContext('2d'); context.drawImage(this.sprite, sourceX, sourceY, this.thumbnailWidth, this.thumbnailHeight, 0, 0, this.thumbnailCanvas.width, this.thumbnailCanvas.height); + + // draw the hover time bar + context.clearRect(0, this.canvasHeight, this.canvasWidth, this.hoverTimeBarHeight); + context.fillStyle = this.hoverTimeStyle['bgColor']; + context.fillRect(0, this.canvasHeight, this.canvasWidth, this.hoverTimeBarHeight); + context.font = this.hoverTimeStyle['font']; + context.fillStyle = this.hoverTimeStyle['color'] + context.textAlign = 'center'; + context.fillText(this.hoverTimeText, this.xtext, this.ytext); }, updateThumbnailInterval() { let maxThumbnails = biigle.$require('videos.spritesMaxThumbnails'); @@ -136,6 +172,18 @@ export default { let fileUuid = fileUuids[this.videoId]; this.spritesFolderPath = thumbUri.replace(':uuid', transformUuid(fileUuid) + '/').replace('.jpg', ''); }, + viewHoverTimeBar() { + // draw the hover time bar + let ctx = this.hovertimeCanvas.getContext('2d'); + ctx.clearRect(0, 0, this.hoverTimeBarWidth, this.hoverTimeBarHeight); + ctx.fillStyle = this.hoverTimeStyle['bgColor']; + ctx.fillRect(0, 0, this.hoverTimeBarWidth, this.hoverTimeBarHeight); + ctx.font = this.hoverTimeStyle['font']; + ctx.fillStyle = this.hoverTimeStyle['color'] + ctx.textAlign = 'center'; + let ytext = this.hoverTimeBarHeight - (this.hoverTimeBarHeight - this.fontSizeInPx)/2 + ctx.fillText(this.hoverTimeText, this.hoverTimeBarWidth/2, ytext); + }, initDimensions() { let nbrCols = this.spriteGridInfo[0]; let nbrRows = this.spriteGridInfo[1]; @@ -143,13 +191,24 @@ export default { this.thumbnailHeight = this.sprite.height / nbrRows; this.canvasWidth = Math.ceil(this.thumbnailWidth / 2); this.canvasHeight = Math.ceil(this.thumbnailHeight / 2); + + // If thumbnail is too narrow, enlarge it to 120px so that the hover time fits + if (this.canvasWidth < this.hoverTimeBarWidth) { + let ratio = this.canvasHeight / this.canvasWidth; + this.canvasWidth = this.hoverTimeBarWidth + this.canvasHeight = this.canvasWidth * ratio; + } + this.thumbnailCanvas.width = this.canvasWidth; - this.thumbnailCanvas.height = this.canvasHeight; + this.thumbnailCanvas.height = this.canvasHeight + this.hoverTimeBarHeight; } }, watch: { hoverTime() { this.updateSprite(); + if (this.spriteNotFound) { + this.viewHoverTimeBar(); + } }, }, created() { @@ -161,6 +220,11 @@ export default { mounted() { this.thumbnailPreview = this.$refs.thumbnailPreview; this.thumbnailCanvas = this.$refs.thumbnailCanvas; + + this.hovertimeCanvas = this.$refs.hovertimeCanvas; + this.hovertimeCanvas.width = this.hoverTimeBarWidth; + this.hovertimeCanvas.height = this.hoverTimeBarHeight; + this.updateSprite(); this.sprite.onload = () => { this.spriteNotFound = false; From ea2d0955b00acd6268d346f6d40d6cd330b5e089 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Wed, 17 Jul 2024 09:51:58 +0200 Subject: [PATCH 053/100] Prevent displaying canvas if error exists --- .../assets/js/videos/components/scrollStrip.vue | 7 ++++++- .../js/videos/components/thumbnailPreview.vue | 13 +++++++++++-- .../assets/js/videos/components/videoTimeline.vue | 7 ++++++- resources/views/videos/show/content.blade.php | 1 + 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/resources/assets/js/videos/components/scrollStrip.vue b/resources/assets/js/videos/components/scrollStrip.vue index 6e307c494..0c4a681ad 100644 --- a/resources/assets/js/videos/components/scrollStrip.vue +++ b/resources/assets/js/videos/components/scrollStrip.vue @@ -16,6 +16,7 @@ :clientMouseX="clientMouseX" :scrollstripTop="scrollstripTop" :videoId="videoId" + :has-error="hasError" v-if="showThumb && showThumbPreview" > + v-show="!hasError" + > Date: Wed, 17 Jul 2024 10:34:57 +0200 Subject: [PATCH 054/100] Prevent flashing canvas --- resources/assets/js/videos/components/thumbnailPreview.vue | 2 ++ 1 file changed, 2 insertions(+) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 2fbadd36f..b240fb7f8 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -6,11 +6,13 @@ v-show="!hasError" > From d39f7b568a6fe3f90922b81d03463d9f82581aa9 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Thu, 18 Jul 2024 08:45:59 +0200 Subject: [PATCH 055/100] Remove unused css class --- resources/assets/js/videos/components/currentTime.vue | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/resources/assets/js/videos/components/currentTime.vue b/resources/assets/js/videos/components/currentTime.vue index 157b31345..830c36711 100644 --- a/resources/assets/js/videos/components/currentTime.vue +++ b/resources/assets/js/videos/components/currentTime.vue @@ -1,8 +1,6 @@ @@ -143,14 +142,7 @@ export default { let context = this.thumbnailCanvas.getContext('2d'); context.drawImage(this.sprite, sourceX, sourceY, this.thumbnailWidth, this.thumbnailHeight, 0, 0, this.thumbnailCanvas.width, this.thumbnailCanvas.height); - // draw the hover time bar - context.clearRect(0, this.canvasHeight, this.canvasWidth, this.hoverTimeBarHeight); - context.fillStyle = this.hoverTimeStyle['bgColor']; - context.fillRect(0, this.canvasHeight, this.canvasWidth, this.hoverTimeBarHeight); - context.font = this.hoverTimeStyle['font']; - context.fillStyle = this.hoverTimeStyle['color'] - context.textAlign = 'center'; - context.fillText(this.hoverTimeText, this.xtext, this.ytext); + this.viewHoverTimeBar(); }, updateThumbnailInterval() { let maxThumbnails = biigle.$require('videos.spritesMaxThumbnails'); @@ -202,7 +194,12 @@ export default { } this.thumbnailCanvas.width = this.canvasWidth; - this.thumbnailCanvas.height = this.canvasHeight + this.hoverTimeBarHeight; + this.thumbnailCanvas.height = this.canvasHeight; + + // Update hover time canvas width if thumbnail canvas width is larger + this.hoverTimeBarWidth = this.canvasWidth > this.hoverTimeBarWidth ? this.canvasWidth : this.hoverTimeBarWidth; + this.hovertimeCanvas.width = this.hoverTimeBarWidth; + this.hovertimeCanvas.height = this.hoverTimeBarHeight; } }, watch: { @@ -222,7 +219,6 @@ export default { mounted() { this.thumbnailPreview = this.$refs.thumbnailPreview; this.thumbnailCanvas = this.$refs.thumbnailCanvas; - this.hovertimeCanvas = this.$refs.hovertimeCanvas; this.hovertimeCanvas.width = this.hoverTimeBarWidth; this.hovertimeCanvas.height = this.hoverTimeBarHeight; diff --git a/resources/assets/sass/videos/components/_thumbnail.scss b/resources/assets/sass/videos/components/_thumbnail.scss index 7f9a5b2fc..b82b475d6 100644 --- a/resources/assets/sass/videos/components/_thumbnail.scss +++ b/resources/assets/sass/videos/components/_thumbnail.scss @@ -4,10 +4,13 @@ left: 0; z-index: 1; pointer-events: none; + display: flex; + flex-direction: column; } .thumbnail-canvas { @extend %info-box; padding: 0; box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.25); -} + display: block; +} \ No newline at end of file From f29f83e1470eb8ab60a30e7de83afd5db0babfbb Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Thu, 18 Jul 2024 10:19:02 +0200 Subject: [PATCH 058/100] Set canvas' background color in css class --- resources/assets/js/videos/components/thumbnailPreview.vue | 2 -- resources/assets/sass/videos/components/_thumbnail.scss | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 1901b5ce2..320a0a546 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -170,8 +170,6 @@ export default { // draw the hover time bar let ctx = this.hovertimeCanvas.getContext('2d'); ctx.clearRect(0, 0, this.hoverTimeBarWidth, this.hoverTimeBarHeight); - ctx.fillStyle = this.hoverTimeStyle['bgColor']; - ctx.fillRect(0, 0, this.hoverTimeBarWidth, this.hoverTimeBarHeight); ctx.font = this.hoverTimeStyle['font']; ctx.fillStyle = this.hoverTimeStyle['color'] ctx.textAlign = 'center'; diff --git a/resources/assets/sass/videos/components/_thumbnail.scss b/resources/assets/sass/videos/components/_thumbnail.scss index b82b475d6..425ba4c5e 100644 --- a/resources/assets/sass/videos/components/_thumbnail.scss +++ b/resources/assets/sass/videos/components/_thumbnail.scss @@ -13,4 +13,6 @@ padding: 0; box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.25); display: block; + background-color: #222; + opacity: 1; } \ No newline at end of file From cd59a8591a3dc759e2f1228ddf1b9537959f1b6a Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Thu, 18 Jul 2024 10:22:19 +0200 Subject: [PATCH 059/100] Make computed prop a constant --- resources/assets/js/videos/components/thumbnailPreview.vue | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 320a0a546..1de87a44d 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -69,6 +69,7 @@ export default { thumbnailInterval: 2.5, estimatedThumbnails: 0, fontSize: 14.5, + fontSizeInPx: 11, hovertimeCanvas: null, hoverTimeBarHeight: 20, hoverTimeBarWidth: 120, @@ -107,9 +108,6 @@ export default { // compute font size in pixel return this.canvasHeight + this.fontSizeInPx + (this.hoverTimeBarHeight - this.fontSizeInPx) / 2; }, - fontSizeInPx() { - return this.fontSize * (72 / 96); - } }, methods: { updateSprite() { From eb1eb170db213ebbc4c9f4fd1e00ca3882bd54f2 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Fri, 19 Jul 2024 08:53:08 +0200 Subject: [PATCH 060/100] Fix empty thumbnail for vertical small mouse movements --- .../js/videos/components/thumbnailPreview.vue | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 1de87a44d..0d3c1dabc 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -211,15 +211,7 @@ export default { this.updateThumbnailInterval(); this.thumbnailSizes = biigle.$require('videos.thumbnailSizes'); this.thumbnailsPerSprite = biigle.$require('videos.spritesThumbnailsPerSprite'); - }, - mounted() { - this.thumbnailPreview = this.$refs.thumbnailPreview; - this.thumbnailCanvas = this.$refs.thumbnailCanvas; - this.hovertimeCanvas = this.$refs.hovertimeCanvas; - this.hovertimeCanvas.width = this.hoverTimeBarWidth; - this.hovertimeCanvas.height = this.hoverTimeBarHeight; - this.updateSprite(); this.sprite.onload = () => { this.spriteNotFound = false; this.initDimensions(); @@ -230,7 +222,17 @@ export default { if (this.sprite.src in this.triedUrls) { this.triedUrls[this.sprite.src]++; } + this.viewHoverTimeBar(); } + }, + mounted() { + this.thumbnailPreview = this.$refs.thumbnailPreview; + this.thumbnailCanvas = this.$refs.thumbnailCanvas; + this.hovertimeCanvas = this.$refs.hovertimeCanvas; + this.hovertimeCanvas.width = this.hoverTimeBarWidth; + this.hovertimeCanvas.height = this.hoverTimeBarHeight; + + this.updateSprite(); } }; From fe87a902fba1ff7d46186538c641a1258b45ab46 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Fri, 19 Jul 2024 08:58:44 +0200 Subject: [PATCH 061/100] Smaller changes --- .../assets/js/videos/components/thumbnailPreview.vue | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 0d3c1dabc..c0a603d47 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -99,13 +99,12 @@ export default { return Vue.filter('videoTime')(this.hoverTime); }, hoverTimeStyle() { - return { 'font': `bold ${this.fontSize}px Sans-Serif`, 'color': '#cccccc', 'bgColor': '#222' }; + return { 'font': `bold ${this.fontSize}px Sans-Serif`, 'color': '#cccccc' }; }, xtext() { return this.canvasWidth / 2; }, ytext() { - // compute font size in pixel return this.canvasHeight + this.fontSizeInPx + (this.hoverTimeBarHeight - this.fontSizeInPx) / 2; }, }, @@ -140,6 +139,7 @@ export default { let context = this.thumbnailCanvas.getContext('2d'); context.drawImage(this.sprite, sourceX, sourceY, this.thumbnailWidth, this.thumbnailHeight, 0, 0, this.thumbnailCanvas.width, this.thumbnailCanvas.height); + // Call viewHoverTimeBar here to prevent flickering hover time bar this.viewHoverTimeBar(); }, updateThumbnailInterval() { @@ -171,8 +171,8 @@ export default { ctx.font = this.hoverTimeStyle['font']; ctx.fillStyle = this.hoverTimeStyle['color'] ctx.textAlign = 'center'; - let ytext = this.hoverTimeBarHeight - (this.hoverTimeBarHeight - this.fontSizeInPx)/2 - ctx.fillText(this.hoverTimeText, this.hoverTimeBarWidth/2, ytext); + let ytext = this.hoverTimeBarHeight - (this.hoverTimeBarHeight - this.fontSizeInPx) / 2 + ctx.fillText(this.hoverTimeText, this.hoverTimeBarWidth / 2, ytext); }, initDimensions() { let nbrCols = this.spriteGridInfo[0]; From d47ba5c078ad5d9109d34ee9717930ac9d332298 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Fri, 19 Jul 2024 10:47:30 +0200 Subject: [PATCH 062/100] Update sprite only if sprite index was changed --- .../js/videos/components/thumbnailPreview.vue | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index c0a603d47..dc938f0df 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -110,7 +110,6 @@ export default { }, methods: { updateSprite() { - this.spriteIdx = Math.floor(this.hoverTime / (this.thumbnailInterval * this.thumbnailsPerSprite)); let SpriteUrl = this.spritesFolderPath + "sprite_" + this.spriteIdx + ".webp"; if (!this.triedUrls[SpriteUrl]) { @@ -196,13 +195,20 @@ export default { this.hoverTimeBarWidth = this.canvasWidth > this.hoverTimeBarWidth ? this.canvasWidth : this.hoverTimeBarWidth; this.hovertimeCanvas.width = this.hoverTimeBarWidth; this.hovertimeCanvas.height = this.hoverTimeBarHeight; - } + }, }, watch: { hoverTime() { - this.updateSprite(); + let spriteIdx = Math.floor(this.hoverTime / (this.thumbnailInterval * this.thumbnailsPerSprite)); + if (this.spriteIdx !== spriteIdx){ + this.spriteIdx = spriteIdx; + this.updateSprite(); + } + if (this.spriteNotFound) { this.viewHoverTimeBar(); + } else { + this.viewThumbnailPreview(); } }, }, @@ -215,6 +221,7 @@ export default { this.sprite.onload = () => { this.spriteNotFound = false; this.initDimensions(); + // Call viewThumbnailPreview here again to prevent glitching thumbnails this.viewThumbnailPreview(); } this.sprite.onerror = () => { From 93d6143d19e60cf3449797b16899fdf3b4d1124b Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Fri, 19 Jul 2024 10:50:53 +0200 Subject: [PATCH 063/100] Tweak style of video thumbnail preview --- .../assets/js/videos/components/scrollStrip.vue | 6 +++--- .../js/videos/components/thumbnailPreview.vue | 4 ++-- .../sass/videos/components/_thumbnail.scss | 17 +++++++++-------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/resources/assets/js/videos/components/scrollStrip.vue b/resources/assets/js/videos/components/scrollStrip.vue index abd705ff3..d586ae12f 100644 --- a/resources/assets/js/videos/components/scrollStrip.vue +++ b/resources/assets/js/videos/components/scrollStrip.vue @@ -16,7 +16,7 @@ :clientMouseX="clientMouseX" :scrollstripTop="scrollstripTop" :videoId="videoId" - v-if="showThumb && showThumbPreview && !hasError" + v-if="showThumbPreview" > this.initialElementWidth; }, showThumbPreview() { - return this.showThumbnailPreview && this.showThumb; - } + return this.showThumbnailPreview && this.showThumb && !this.hasError; + }, }, methods: { updateInitialElementWidth() { diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index c0a603d47..5404b9418 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -6,7 +6,7 @@ > @@ -52,7 +52,7 @@ export default { thumbnailCanvas: null, sprite: new Image(), spriteIdx: 0, - thumbProgressBarSpace: 5, + thumbProgressBarSpace: 10, sideButtonsWidth: 52, spritesFolderPath: null, triedUrls: {}, diff --git a/resources/assets/sass/videos/components/_thumbnail.scss b/resources/assets/sass/videos/components/_thumbnail.scss index 425ba4c5e..a5ae3fab0 100644 --- a/resources/assets/sass/videos/components/_thumbnail.scss +++ b/resources/assets/sass/videos/components/_thumbnail.scss @@ -6,13 +6,14 @@ pointer-events: none; display: flex; flex-direction: column; -} - -.thumbnail-canvas { - @extend %info-box; - padding: 0; box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.25); - display: block; background-color: #222; - opacity: 1; -} \ No newline at end of file + border-radius: 4px; + border: 1px solid #353535; + + .thumbnail-canvas { + display: block; + border-radius: 4px; + } +} + From b4029b9ea18af979fcef835651751b7aac1d5a12 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Fri, 19 Jul 2024 10:55:27 +0200 Subject: [PATCH 064/100] Add preload thumbnail function --- .../js/videos/components/thumbnailPreview.vue | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index dc938f0df..e1f1f269c 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -73,6 +73,8 @@ export default { hovertimeCanvas: null, hoverTimeBarHeight: 20, hoverTimeBarWidth: 120, + preloadedSprites: [], + lastSpriteIdx: 0, }; }, computed: { @@ -109,6 +111,26 @@ export default { }, }, methods: { + preloadPreviousSprite() { + let prevIdx = this.spriteIdx - 1; + if (this.spriteIdx === 0 || prevIdx in this.preloadedSprites) { + return; + } + let prevSpriteUrl = this.spritesFolderPath + "sprite_" + prevIdx + ".webp"; + let prevImg = new Image(); + prevImg.src = prevSpriteUrl; + this.preloadedSprites[prevIdx] = prevImg; + }, + preloadNextSprite() { + let nextIdx = this.spriteIdx + 1; + if (this.spriteIdx === this.lastSpriteIdx || nextIdx in this.preloadedSprites) { + return; + } + let nextSpriteUrl = this.spritesFolderPath + "sprite_" + nextIdx + ".webp"; + let nextImg = new Image(); + nextImg.src = nextSpriteUrl; + this.preloadedSprites[nextIdx] = nextImg; + }, updateSprite() { let SpriteUrl = this.spritesFolderPath + "sprite_" + this.spriteIdx + ".webp"; @@ -116,10 +138,21 @@ export default { this.triedUrls[SpriteUrl] = 0 } if (this.triedUrls[SpriteUrl] < this.retryAttempts) { - this.sprite.src = SpriteUrl; + if (this.spriteIdx in this.preloadedSprites) { + let onloadFunc = this.sprite.onload; + this.sprite = this.preloadedSprites[this.spriteIdx]; + onloadFunc.call(this.sprite, new Event('load')); + this.sprite.onload = onloadFunc; + } else { + this.sprite.src = SpriteUrl; + this.preloadedSprites[this.spriteIdx] = this.sprite; + } } else { this.spriteNotFound = true; } + + this.preloadPreviousSprite(); + this.preloadNextSprite(); }, viewThumbnailPreview() { // calculate the current row and column of the sprite @@ -147,6 +180,7 @@ export default { let defaultThumbnailInterval = biigle.$require('videos.spritesThumbnailInterval'); this.durationRounded = Math.floor(this.duration * 10) / 10; this.estimatedThumbnails = Math.floor(this.durationRounded / defaultThumbnailInterval); + this.lastSpriteIdx = Math.floor(this.estimatedThumbnails / this.thumbnailsPerSprite); if (this.estimatedThumbnails > maxThumbnails) { this.estimatedThumbnails = maxThumbnails; this.thumbnailInterval = this.durationRounded / maxThumbnails; From dd98943e0d255375a78bc25ac6c09225c0143903 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Fri, 19 Jul 2024 10:57:37 +0200 Subject: [PATCH 065/100] Remove unused css classes --- .../assets/sass/videos/components/_currentTime.scss | 9 --------- 1 file changed, 9 deletions(-) diff --git a/resources/assets/sass/videos/components/_currentTime.scss b/resources/assets/sass/videos/components/_currentTime.scss index 2da268f5e..e16261d19 100644 --- a/resources/assets/sass/videos/components/_currentTime.scss +++ b/resources/assets/sass/videos/components/_currentTime.scss @@ -9,12 +9,3 @@ color: $text-muted; } } - -.current-time--seeking { - color: $text-muted; -} - -.current-time--hover { - padding: 8px; - font-size: $font-size-small; -} From eb14f6c5378828c4193d191afd284858e6347f48 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Fri, 19 Jul 2024 10:59:39 +0200 Subject: [PATCH 066/100] Remove unused methods --- resources/assets/js/videos/components/thumbnailPreview.vue | 6 ------ 1 file changed, 6 deletions(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 5404b9418..e3cd8fdbe 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -101,12 +101,6 @@ export default { hoverTimeStyle() { return { 'font': `bold ${this.fontSize}px Sans-Serif`, 'color': '#cccccc' }; }, - xtext() { - return this.canvasWidth / 2; - }, - ytext() { - return this.canvasHeight + this.fontSizeInPx + (this.hoverTimeBarHeight - this.fontSizeInPx) / 2; - }, }, methods: { updateSprite() { From a2063e5e3856ff7fd72cdbeb7313121ecb9f02f3 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Fri, 19 Jul 2024 11:00:38 +0200 Subject: [PATCH 067/100] Make constant a computed prop again --- resources/assets/js/videos/components/thumbnailPreview.vue | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index e3cd8fdbe..c04e47ee1 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -69,7 +69,6 @@ export default { thumbnailInterval: 2.5, estimatedThumbnails: 0, fontSize: 14.5, - fontSizeInPx: 11, hovertimeCanvas: null, hoverTimeBarHeight: 20, hoverTimeBarWidth: 120, @@ -101,6 +100,9 @@ export default { hoverTimeStyle() { return { 'font': `bold ${this.fontSize}px Sans-Serif`, 'color': '#cccccc' }; }, + fontSizeInPx() { + return this.fontSize * (72 / 96); + } }, methods: { updateSprite() { From 387603e7b160b3600f7cc1f344672f7f718d2905 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Mon, 22 Jul 2024 09:33:34 +0200 Subject: [PATCH 068/100] Add sprite url generation method --- .../assets/js/videos/components/thumbnailPreview.vue | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 695703db6..41ee048a3 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -112,7 +112,7 @@ export default { if (this.spriteIdx === 0 || prevIdx in this.preloadedSprites) { return; } - let prevSpriteUrl = this.spritesFolderPath + "sprite_" + prevIdx + ".webp"; + let prevSpriteUrl = this.getSpriteUrl(prevIdx); let prevImg = new Image(); prevImg.src = prevSpriteUrl; this.preloadedSprites[prevIdx] = prevImg; @@ -122,13 +122,13 @@ export default { if (this.spriteIdx === this.lastSpriteIdx || nextIdx in this.preloadedSprites) { return; } - let nextSpriteUrl = this.spritesFolderPath + "sprite_" + nextIdx + ".webp"; + let nextSpriteUrl = this.getSpriteUrl(nextIdx); let nextImg = new Image(); nextImg.src = nextSpriteUrl; this.preloadedSprites[nextIdx] = nextImg; }, updateSprite() { - let SpriteUrl = this.spritesFolderPath + "sprite_" + this.spriteIdx + ".webp"; + let SpriteUrl = this.getSpriteUrl(this.spriteIdx); if (!this.triedUrls[SpriteUrl]) { this.triedUrls[SpriteUrl] = 0 @@ -146,10 +146,12 @@ export default { } else { this.spriteNotFound = true; } - this.preloadPreviousSprite(); this.preloadNextSprite(); }, + getSpriteUrl(idx) { + return this.spritesFolderPath + "sprite_" + idx + ".webp"; + }, viewThumbnailPreview() { // calculate the current row and column of the sprite let thumbnailIndex = Math.floor(this.hoverTime / this.thumbnailInterval) % this.thumbnailsPerSprite; From 3f70ce3ce3577e214e7ddc6038cdf945240fa89d Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Mon, 22 Jul 2024 09:34:27 +0200 Subject: [PATCH 069/100] Fix bug with wrong sprite index after initialization --- resources/assets/js/videos/components/scrollStrip.vue | 2 +- resources/assets/js/videos/components/thumbnailPreview.vue | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/resources/assets/js/videos/components/scrollStrip.vue b/resources/assets/js/videos/components/scrollStrip.vue index d586ae12f..deace949d 100644 --- a/resources/assets/js/videos/components/scrollStrip.vue +++ b/resources/assets/js/videos/components/scrollStrip.vue @@ -166,7 +166,7 @@ export default { return this.elementWidth + this.scrollLeft > this.initialElementWidth; }, showThumbPreview() { - return this.showThumbnailPreview && this.showThumb && !this.hasError; + return this.showThumbnailPreview && this.showThumb && !this.hasError && this.hoverTime !== 0; }, }, methods: { diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 41ee048a3..06efc9739 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -128,6 +128,7 @@ export default { this.preloadedSprites[nextIdx] = nextImg; }, updateSprite() { + this.spriteIdx = Math.floor(this.hoverTime / (this.thumbnailInterval * this.thumbnailsPerSprite)); let SpriteUrl = this.getSpriteUrl(this.spriteIdx); if (!this.triedUrls[SpriteUrl]) { From de57cb39f254bae03e151ae5b09b3132c49cd6e8 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Mon, 22 Jul 2024 10:27:25 +0200 Subject: [PATCH 070/100] Save only previous, current and next sprite. --- resources/assets/js/videos/components/thumbnailPreview.vue | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 06efc9739..8a2556502 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -72,7 +72,7 @@ export default { hovertimeCanvas: null, hoverTimeBarHeight: 20, hoverTimeBarWidth: 120, - preloadedSprites: [], + preloadedSprites: {}, lastSpriteIdx: 0, }; }, @@ -127,6 +127,10 @@ export default { nextImg.src = nextSpriteUrl; this.preloadedSprites[nextIdx] = nextImg; }, + removeOldSprites() { + delete this.preloadedSprites[this.spriteIdx - 2]; + delete this.preloadedSprites[this.spriteIdx + 2]; + }, updateSprite() { this.spriteIdx = Math.floor(this.hoverTime / (this.thumbnailInterval * this.thumbnailsPerSprite)); let SpriteUrl = this.getSpriteUrl(this.spriteIdx); @@ -149,6 +153,7 @@ export default { } this.preloadPreviousSprite(); this.preloadNextSprite(); + this.removeOldSprites(); }, getSpriteUrl(idx) { return this.spritesFolderPath + "sprite_" + idx + ".webp"; From 2b117175473343fe2c11a52952b3143919a530a2 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Mon, 22 Jul 2024 15:48:16 +0200 Subject: [PATCH 071/100] Update resources/assets/js/videos/components/thumbnailPreview.vue --- resources/assets/js/videos/components/thumbnailPreview.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index c04e47ee1..d1c1f607e 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -101,7 +101,7 @@ export default { return { 'font': `bold ${this.fontSize}px Sans-Serif`, 'color': '#cccccc' }; }, fontSizeInPx() { - return this.fontSize * (72 / 96); + return this.fontSize * 0.75; } }, methods: { From 685b8ce250b2f75e806588128e01f47a727f25ac Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Wed, 24 Jul 2024 08:57:31 +0200 Subject: [PATCH 072/100] Fix error for missing sprites --- .../js/videos/components/thumbnailPreview.vue | 41 +++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index a62434ba5..38b1096c9 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -109,23 +109,50 @@ export default { methods: { preloadPreviousSprite() { let prevIdx = this.spriteIdx - 1; - if (this.spriteIdx === 0 || prevIdx in this.preloadedSprites) { + let prevSpriteUrl = this.getSpriteUrl(prevIdx); + + if (this.spriteIdx === 0 + || prevIdx in this.preloadedSprites + || this.triedUrls[prevSpriteUrl] >= this.retryAttempts) { return; } - let prevSpriteUrl = this.getSpriteUrl(prevIdx); + + if (!this.triedUrls[prevSpriteUrl]) { + this.triedUrls[prevSpriteUrl] = 0 + } let prevImg = new Image(); + prevImg.onload = () => { + this.preloadedSprites[prevIdx] = prevImg; + } + prevImg.onerror = () => { + if (prevSpriteUrl in this.triedUrls) { + this.triedUrls[prevSpriteUrl]++; + } + } prevImg.src = prevSpriteUrl; - this.preloadedSprites[prevIdx] = prevImg; }, preloadNextSprite() { let nextIdx = this.spriteIdx + 1; - if (this.spriteIdx === this.lastSpriteIdx || nextIdx in this.preloadedSprites) { + let nextSpriteUrl = this.getSpriteUrl(nextIdx); + + if (this.spriteIdx === this.lastSpriteIdx + || nextIdx in this.preloadedSprites + || this.triedUrls[nextSpriteUrl] >= this.retryAttempts) { return; } - let nextSpriteUrl = this.getSpriteUrl(nextIdx); + if (!this.triedUrls[nextSpriteUrl]) { + this.triedUrls[nextSpriteUrl] = 0 + } let nextImg = new Image(); + nextImg.onload = () => { + this.preloadedSprites[nextIdx] = nextImg; + } + nextImg.onerror = () => { + if (nextSpriteUrl in this.triedUrls) { + this.triedUrls[nextSpriteUrl]++; + } + } nextImg.src = nextSpriteUrl; - this.preloadedSprites[nextIdx] = nextImg; }, removeOldSprites() { delete this.preloadedSprites[this.spriteIdx - 2]; @@ -146,7 +173,6 @@ export default { this.sprite.onload = onloadFunc; } else { this.sprite.src = SpriteUrl; - this.preloadedSprites[this.spriteIdx] = this.sprite; } } else { this.spriteNotFound = true; @@ -258,6 +284,7 @@ export default { this.sprite.onload = () => { this.spriteNotFound = false; + this.preloadedSprites[this.spriteIdx] = this.sprite; this.initDimensions(); // Call viewThumbnailPreview here again to prevent glitching thumbnails this.viewThumbnailPreview(); From 8343b6419b757029fece2dcd62ea228f4b54527c Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Wed, 24 Jul 2024 10:04:34 +0200 Subject: [PATCH 073/100] Fix missing error function --- resources/assets/js/videos/components/thumbnailPreview.vue | 2 ++ 1 file changed, 2 insertions(+) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 38b1096c9..7c694158c 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -168,9 +168,11 @@ export default { if (this.triedUrls[SpriteUrl] < this.retryAttempts) { if (this.spriteIdx in this.preloadedSprites) { let onloadFunc = this.sprite.onload; + let onErrFunc = this.sprite.onerror; this.sprite = this.preloadedSprites[this.spriteIdx]; onloadFunc.call(this.sprite, new Event('load')); this.sprite.onload = onloadFunc; + this.sprite.onerror = onErrFunc; } else { this.sprite.src = SpriteUrl; } From 4c11902deeefa6e7b69410d26a00baf0e58fa8d4 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Mon, 29 Jul 2024 08:57:20 +0200 Subject: [PATCH 074/100] Fix broken image error for missing sprites --- .../js/videos/components/thumbnailPreview.vue | 40 +++++++++---------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 7c694158c..466c667d4 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -104,7 +104,7 @@ export default { }, fontSizeInPx() { return this.fontSize * 0.75; - } + }, }, methods: { preloadPreviousSprite() { @@ -122,7 +122,7 @@ export default { } let prevImg = new Image(); prevImg.onload = () => { - this.preloadedSprites[prevIdx] = prevImg; + this.preloadedSprites[prevSpriteUrl] = prevImg; } prevImg.onerror = () => { if (prevSpriteUrl in this.triedUrls) { @@ -145,7 +145,7 @@ export default { } let nextImg = new Image(); nextImg.onload = () => { - this.preloadedSprites[nextIdx] = nextImg; + this.preloadedSprites[nextSpriteUrl] = nextImg; } nextImg.onerror = () => { if (nextSpriteUrl in this.triedUrls) { @@ -155,26 +155,25 @@ export default { nextImg.src = nextSpriteUrl; }, removeOldSprites() { - delete this.preloadedSprites[this.spriteIdx - 2]; - delete this.preloadedSprites[this.spriteIdx + 2]; + delete this.preloadedSprites[this.getSpriteUrl(this.spriteIdx - 2)]; + delete this.preloadedSprites[this.getSpriteUrl(this.spriteIdx + 2)]; }, updateSprite() { - this.spriteIdx = Math.floor(this.hoverTime / (this.thumbnailInterval * this.thumbnailsPerSprite)); - let SpriteUrl = this.getSpriteUrl(this.spriteIdx); + let spriteUrl = this.getSpriteUrl(this.spriteIdx); - if (!this.triedUrls[SpriteUrl]) { - this.triedUrls[SpriteUrl] = 0 + if (!this.triedUrls[spriteUrl]) { + this.triedUrls[spriteUrl] = 0 } - if (this.triedUrls[SpriteUrl] < this.retryAttempts) { - if (this.spriteIdx in this.preloadedSprites) { + if (this.triedUrls[spriteUrl] < this.retryAttempts) { + if (spriteUrl in this.preloadedSprites && this.triedUrls[spriteUrl] === 0) { let onloadFunc = this.sprite.onload; - let onErrFunc = this.sprite.onerror; - this.sprite = this.preloadedSprites[this.spriteIdx]; + let onerrFunc = this.sprite.onerror; + this.sprite = this.preloadedSprites[spriteUrl]; onloadFunc.call(this.sprite, new Event('load')); this.sprite.onload = onloadFunc; - this.sprite.onerror = onErrFunc; + this.sprite.onerror = onerrFunc; } else { - this.sprite.src = SpriteUrl; + this.sprite.src = spriteUrl; } } else { this.spriteNotFound = true; @@ -265,11 +264,7 @@ export default { }, watch: { hoverTime() { - let spriteIdx = Math.floor(this.hoverTime / (this.thumbnailInterval * this.thumbnailsPerSprite)); - if (this.spriteIdx !== spriteIdx){ - this.spriteIdx = spriteIdx; - this.updateSprite(); - } + this.spriteIdx = Math.floor(this.hoverTime / (this.thumbnailInterval * this.thumbnailsPerSprite)); if (this.spriteNotFound) { this.viewHoverTimeBar(); @@ -277,6 +272,9 @@ export default { this.viewThumbnailPreview(); } }, + spriteIdx() { + this.updateSprite(); + } }, created() { this.setSpritesFolderpath(); @@ -286,7 +284,7 @@ export default { this.sprite.onload = () => { this.spriteNotFound = false; - this.preloadedSprites[this.spriteIdx] = this.sprite; + this.preloadedSprites[this.sprite.src] = this.sprite; this.initDimensions(); // Call viewThumbnailPreview here again to prevent glitching thumbnails this.viewThumbnailPreview(); From 1e5043c789123e0e08fdb3965f6c3366d02ad55b Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Mon, 29 Jul 2024 10:25:37 +0200 Subject: [PATCH 075/100] Fix problem with retained old sprites --- .../js/videos/components/thumbnailPreview.vue | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 466c667d4..11aacef91 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -155,8 +155,17 @@ export default { nextImg.src = nextSpriteUrl; }, removeOldSprites() { - delete this.preloadedSprites[this.getSpriteUrl(this.spriteIdx - 2)]; - delete this.preloadedSprites[this.getSpriteUrl(this.spriteIdx + 2)]; + let preloadedSprites = {} + for (let i = this.spriteIdx - 1; i < this.spriteIdx + 2; i++) { + if (i !== 0 || i !== this.lastSpriteIdx) { + let url = this.getSpriteUrl(i); + let img = this.preloadedSprites[url]; + if (img) { + preloadedSprites[url] = img; + } + } + } + this.preloadedSprites = preloadedSprites; }, updateSprite() { let spriteUrl = this.getSpriteUrl(this.spriteIdx); From cbe7785eb1c4c8d29299035c7a44571e48facef3 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Mon, 29 Jul 2024 10:35:01 +0200 Subject: [PATCH 076/100] Fix wrong sprite index after initialization --- resources/assets/js/videos/components/thumbnailPreview.vue | 1 + 1 file changed, 1 insertion(+) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 11aacef91..08f825bb8 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -313,6 +313,7 @@ export default { this.hovertimeCanvas.width = this.hoverTimeBarWidth; this.hovertimeCanvas.height = this.hoverTimeBarHeight; + this.spriteIdx = Math.floor(this.hoverTime / (this.thumbnailInterval * this.thumbnailsPerSprite)); this.updateSprite(); } }; From 4b38ae83506e38007d883c79c27d61bf031c5aa1 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Tue, 27 Aug 2024 08:31:32 +0200 Subject: [PATCH 077/100] Fix wrong sprite index When hovering fast it can happen that the sprite index and sprite url do not belong to the same sprite any more. Fix it by setting sprite index in update method instead of watcher. --- .../assets/js/videos/components/thumbnailPreview.vue | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 08f825bb8..440db1f97 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -168,6 +168,7 @@ export default { this.preloadedSprites = preloadedSprites; }, updateSprite() { + this.spriteIdx = Math.floor(this.hoverTime / (this.thumbnailInterval * this.thumbnailsPerSprite)); let spriteUrl = this.getSpriteUrl(this.spriteIdx); if (!this.triedUrls[spriteUrl]) { @@ -273,17 +274,16 @@ export default { }, watch: { hoverTime() { - this.spriteIdx = Math.floor(this.hoverTime / (this.thumbnailInterval * this.thumbnailsPerSprite)); - + let spriteIdx = Math.floor(this.hoverTime / (this.thumbnailInterval * this.thumbnailsPerSprite)); + if(this.spriteIdx != spriteIdx) { + this.updateSprite(); + } if (this.spriteNotFound) { this.viewHoverTimeBar(); } else { this.viewThumbnailPreview(); } }, - spriteIdx() { - this.updateSprite(); - } }, created() { this.setSpritesFolderpath(); From 736bd3495ad854282070daf6d32ab7ee317555d4 Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Wed, 28 Aug 2024 10:15:58 +0200 Subject: [PATCH 078/100] Fix broken image error when hovering fast --- .../js/videos/components/thumbnailPreview.vue | 63 +++++++++---------- 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 440db1f97..6ca023243 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -1,19 +1,8 @@ @@ -121,9 +110,8 @@ export default { this.triedUrls[prevSpriteUrl] = 0 } let prevImg = new Image(); - prevImg.onload = () => { - this.preloadedSprites[prevSpriteUrl] = prevImg; - } + this.preloadedSprites[prevSpriteUrl] = prevImg; + prevImg.onerror = () => { if (prevSpriteUrl in this.triedUrls) { this.triedUrls[prevSpriteUrl]++; @@ -144,9 +132,8 @@ export default { this.triedUrls[nextSpriteUrl] = 0 } let nextImg = new Image(); - nextImg.onload = () => { - this.preloadedSprites[nextSpriteUrl] = nextImg; - } + this.preloadedSprites[nextSpriteUrl] = nextImg; + nextImg.onerror = () => { if (nextSpriteUrl in this.triedUrls) { this.triedUrls[nextSpriteUrl]++; @@ -174,20 +161,22 @@ export default { if (!this.triedUrls[spriteUrl]) { this.triedUrls[spriteUrl] = 0 } - if (this.triedUrls[spriteUrl] < this.retryAttempts) { - if (spriteUrl in this.preloadedSprites && this.triedUrls[spriteUrl] === 0) { - let onloadFunc = this.sprite.onload; - let onerrFunc = this.sprite.onerror; - this.sprite = this.preloadedSprites[spriteUrl]; - onloadFunc.call(this.sprite, new Event('load')); - this.sprite.onload = onloadFunc; - this.sprite.onerror = onerrFunc; - } else { - this.sprite.src = spriteUrl; - } + + let preloadedSprite = this.preloadedSprites[spriteUrl]; + + if (preloadedSprite && this.finishedLoading(preloadedSprite) && this.triedUrls[spriteUrl] < this.retryAttempts) { + let onloadFunc = this.sprite.onload; + let onerrFunc = this.sprite.onerror; + this.sprite = preloadedSprite; + onloadFunc.call(this.sprite, new Event('load')); + this.sprite.onload = onloadFunc; + this.sprite.onerror = onerrFunc; + } else if (!preloadedSprite && this.triedUrls[spriteUrl] < this.retryAttempts) { + this.sprite.src = spriteUrl; } else { this.spriteNotFound = true; } + this.preloadPreviousSprite(); this.preloadNextSprite(); this.removeOldSprites(); @@ -211,7 +200,7 @@ export default { // draw the current thumbnail to the canvas let context = this.thumbnailCanvas.getContext('2d'); context.drawImage(this.sprite, sourceX, sourceY, this.thumbnailWidth, this.thumbnailHeight, 0, 0, this.thumbnailCanvas.width, this.thumbnailCanvas.height); - + // Call viewHoverTimeBar here to prevent flickering hover time bar this.viewHoverTimeBar(); }, @@ -271,11 +260,17 @@ export default { this.hovertimeCanvas.width = this.hoverTimeBarWidth; this.hovertimeCanvas.height = this.hoverTimeBarHeight; }, + finishedLoading(sprite) { + if (!sprite) { + return false; + } + return sprite.complete && sprite.naturalWidth && sprite.naturalWidth !== 0 + } }, watch: { hoverTime() { let spriteIdx = Math.floor(this.hoverTime / (this.thumbnailInterval * this.thumbnailsPerSprite)); - if(this.spriteIdx != spriteIdx) { + if (this.spriteIdx != spriteIdx) { this.updateSprite(); } if (this.spriteNotFound) { @@ -295,8 +290,6 @@ export default { this.spriteNotFound = false; this.preloadedSprites[this.sprite.src] = this.sprite; this.initDimensions(); - // Call viewThumbnailPreview here again to prevent glitching thumbnails - this.viewThumbnailPreview(); } this.sprite.onerror = () => { this.spriteNotFound = true; From 9442c3a229be58bb923fc2b8fe67546975780a74 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Wed, 4 Sep 2024 16:39:00 +0200 Subject: [PATCH 079/100] Fix display of video thumbnail on error --- resources/assets/js/videos/components/thumbnailPreview.vue | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 6ca023243..92e15340f 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -1,8 +1,8 @@ @@ -63,6 +63,7 @@ export default { hoverTimeBarWidth: 120, preloadedSprites: {}, lastSpriteIdx: 0, + hasAnySprite: false, }; }, computed: { @@ -185,6 +186,7 @@ export default { return this.spritesFolderPath + "sprite_" + idx + ".webp"; }, viewThumbnailPreview() { + this.hasAnySprite = true; // calculate the current row and column of the sprite let thumbnailIndex = Math.floor(this.hoverTime / this.thumbnailInterval) % this.thumbnailsPerSprite; if (this.hoverTime >= this.durationRounded) { From a3bf7d6bdcbac8156ad48bcd77ccf84c2f8585f7 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Wed, 4 Sep 2024 16:47:42 +0200 Subject: [PATCH 080/100] Update video thumbnail only when required --- .../js/videos/components/thumbnailPreview.vue | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 92e15340f..11bae2ea6 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -41,6 +41,7 @@ export default { thumbnailCanvas: null, sprite: new Image(), spriteIdx: 0, + thumbnailIndex: -1, thumbProgressBarSpace: 10, sideButtonsWidth: 52, spritesFolderPath: null, @@ -192,6 +193,13 @@ export default { if (this.hoverTime >= this.durationRounded) { thumbnailIndex = thumbnailIndex === 0 ? this.thumbnailsPerSprite - 1 : this.estimatedThumbnails - 1; } + + // Skip redrawing the same thumbnail than before. + if (this.thumbnailIndex === thumbnailIndex) { + return; + } + this.thumbnailIndex = thumbnailIndex; + let thumbnailRow = Math.floor(thumbnailIndex / Math.sqrt(this.thumbnailsPerSprite)); let thumbnailColumn = thumbnailIndex % Math.sqrt(this.thumbnailsPerSprite); @@ -202,9 +210,6 @@ export default { // draw the current thumbnail to the canvas let context = this.thumbnailCanvas.getContext('2d'); context.drawImage(this.sprite, sourceX, sourceY, this.thumbnailWidth, this.thumbnailHeight, 0, 0, this.thumbnailCanvas.width, this.thumbnailCanvas.height); - - // Call viewHoverTimeBar here to prevent flickering hover time bar - this.viewHoverTimeBar(); }, updateThumbnailInterval() { let maxThumbnails = biigle.$require('videos.spritesMaxThumbnails'); @@ -275,11 +280,10 @@ export default { if (this.spriteIdx != spriteIdx) { this.updateSprite(); } - if (this.spriteNotFound) { - this.viewHoverTimeBar(); - } else { + if (!this.spriteNotFound) { this.viewThumbnailPreview(); } + this.viewHoverTimeBar(); }, }, created() { From 8b1e7ae02882707cb1518faead4421c4cc55a7d1 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Wed, 4 Sep 2024 16:47:56 +0200 Subject: [PATCH 081/100] Fix initial display of video thumbnail --- resources/assets/js/videos/components/thumbnailPreview.vue | 3 +++ 1 file changed, 3 insertions(+) diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 11bae2ea6..54b27326f 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -296,6 +296,8 @@ export default { this.spriteNotFound = false; this.preloadedSprites[this.sprite.src] = this.sprite; this.initDimensions(); + this.viewThumbnailPreview(); + this.viewHoverTimeBar(); } this.sprite.onerror = () => { this.spriteNotFound = true; @@ -314,6 +316,7 @@ export default { this.spriteIdx = Math.floor(this.hoverTime / (this.thumbnailInterval * this.thumbnailsPerSprite)); this.updateSprite(); + this.viewHoverTimeBar(); } }; From f7f92a68f57bdba7c4a17f46d919cc4d846d42b5 Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Wed, 4 Sep 2024 16:51:12 +0200 Subject: [PATCH 082/100] Fix linter errors --- app/Jobs/ProcessNewVideo.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Jobs/ProcessNewVideo.php b/app/Jobs/ProcessNewVideo.php index a812e2ce0..7e04e9d33 100644 --- a/app/Jobs/ProcessNewVideo.php +++ b/app/Jobs/ProcessNewVideo.php @@ -171,7 +171,7 @@ public function handleFile($file, $path) // displayed. Log::warning("Could not generate thumbnails for new video {$this->video->id}: {$e->getMessage()}"); } finally { - if (File::exists($tmpDir)) { + if (isset($tmpDir) && File::exists($tmpDir)) { File::deleteDirectory($tmpDir); } } From a54b261e2b6a243de70cccdb32eedb5e6ca5d6ea Mon Sep 17 00:00:00 2001 From: Martin Zurowietz Date: Wed, 4 Sep 2024 17:05:33 +0200 Subject: [PATCH 083/100] Fix use statement in test --- tests/php/Jobs/ProcessNewVideoTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/php/Jobs/ProcessNewVideoTest.php b/tests/php/Jobs/ProcessNewVideoTest.php index 0dc082483..7b7827ab5 100644 --- a/tests/php/Jobs/ProcessNewVideoTest.php +++ b/tests/php/Jobs/ProcessNewVideoTest.php @@ -9,9 +9,9 @@ use FileCache; use Illuminate\Support\Facades\File; use Jcupitt\Vips\Extend; +use Jcupitt\Vips\Image as VipsImage; use Storage; use TestCase; -use VipsImage; class ProcessNewVideoTest extends TestCase { From 08e2be766cdd76dbf7cedccc0b66336b86950cab Mon Sep 17 00:00:00 2001 From: Leane Schlundt Date: Mon, 23 Sep 2024 08:11:38 +0200 Subject: [PATCH 084/100] Show thumbnails if setting is enabled --- .../js/videos/components/scrollStrip.vue | 13 +++++---- .../js/videos/components/thumbnailPreview.vue | 28 +++++++++++++------ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/resources/assets/js/videos/components/scrollStrip.vue b/resources/assets/js/videos/components/scrollStrip.vue index deace949d..22cc21b62 100644 --- a/resources/assets/js/videos/components/scrollStrip.vue +++ b/resources/assets/js/videos/components/scrollStrip.vue @@ -16,7 +16,8 @@ :clientMouseX="clientMouseX" :scrollstripTop="scrollstripTop" :videoId="videoId" - v-if="showThumbPreview" + :showThumbnails="showThumbnailPreview" + v-if="canShowThumbPreview" > this.initialElementWidth; }, - showThumbPreview() { - return this.showThumbnailPreview && this.showThumb && !this.hasError && this.hoverTime !== 0; + canShowThumbPreview() { + return this.canShowThumb && !this.hasError && this.hoverTime !== 0 && this.duration !== 0; }, }, methods: { @@ -242,12 +243,12 @@ export default { this.hasOverflowBottom = false; }, handleVideoProgressMousemove(clientX) { - this.showThumb = true; + this.canShowThumb = true; this.clientMouseX = clientX; this.scrollstripTop = this.$refs.scroller.getBoundingClientRect().top; }, hideThumbnailPreview() { - this.showThumb = false; + this.canShowThumb = false; }, }, watch: { diff --git a/resources/assets/js/videos/components/thumbnailPreview.vue b/resources/assets/js/videos/components/thumbnailPreview.vue index 54b27326f..4fd89d3e9 100644 --- a/resources/assets/js/videos/components/thumbnailPreview.vue +++ b/resources/assets/js/videos/components/thumbnailPreview.vue @@ -1,6 +1,6 @@