From 66c4e7dc5e1efe914b43bbf6a02d42daceadbce9 Mon Sep 17 00:00:00 2001 From: Nico Hoffmann Date: Sat, 1 Jun 2024 12:40:01 +0200 Subject: [PATCH] Upload: clean up code --- panel/src/helpers/upload.js | 80 ++++++++++++++++++------------------- panel/src/panel/upload.js | 4 +- src/Api/Upload.php | 24 +++++++---- src/Panel/Panel.php | 2 +- tests/Api/UploadTest.php | 8 ++-- 5 files changed, 63 insertions(+), 55 deletions(-) diff --git a/panel/src/helpers/upload.js b/panel/src/helpers/upload.js index 0c890c89bc..dabf702011 100644 --- a/panel/src/helpers/upload.js +++ b/panel/src/helpers/upload.js @@ -1,45 +1,5 @@ import { random } from "./string.js"; -/** - * Uploads a file in chunks - * @param {File} file - file to upload - * @param {Object} params - upload options (see `upload` method for details) - * @param {number} size - chunk size in bytes - */ -export async function chunked(file, params, size = 5242880) { - const parts = Math.ceil(file.size / size); - const id = random(4).toLowerCase(); - let response; - - for (let i = 0; i < parts; i++) { - const start = i * size; - const end = Math.min(start + size, file.size); - const chunk = parts > 1 ? file.slice(start, end, file.type) : file; - - // when more than one part, add flag to - // recognize chunked upload and its last chunk - if (parts > 1) { - params.headers = { - ...params.headers, - "Upload-Length": file.size, - "Upload-Offset": start, - "Upload-Id": id - }; - } - - response = await upload(chunk, { - ...params, - // calculate the total progress based on chunk progress - progress: (xhr, chunk, percent) => { - const total = (i + percent / 100) / parts; - params.progress(xhr, file, Math.round(total * 100)); - } - }); - } - - return response; -} - /** * Uploads a file using XMLHttpRequest. * @@ -140,4 +100,44 @@ export async function upload(file, params) { }); } +/** + * Uploads a file in chunks + * @param {File} file - file to upload + * @param {Object} params - upload options (see `upload` method for details) + * @param {number} size - chunk size in bytes (default: 5 MB) + */ +export async function uploadAsChunks(file, params, size = 5242880) { + const parts = Math.ceil(file.size / size); + const id = random(4).toLowerCase(); + let response; + + for (let i = 0; i < parts; i++) { + const start = i * size; + const end = Math.min(start + size, file.size); + const chunk = parts > 1 ? file.slice(start, end, file.type) : file; + + // when more than one part, add flag to + // recognize chunked upload and its last chunk + if (parts > 1) { + params.headers = { + ...params.headers, + "Upload-Length": file.size, + "Upload-Offset": start, + "Upload-Id": id + }; + } + + response = await upload(chunk, { + ...params, + // calculate the total progress based on chunk progress + progress: (xhr, chunk, percent) => { + const total = (i + percent / 100) / parts; + params.progress(xhr, file, Math.round(total * 100)); + } + }); + } + + return response; +} + export default upload; diff --git a/panel/src/panel/upload.js b/panel/src/panel/upload.js index 39673343eb..b40b0c84f1 100644 --- a/panel/src/panel/upload.js +++ b/panel/src/panel/upload.js @@ -2,7 +2,7 @@ import { uuid } from "@/helpers/string"; import State from "./state.js"; import listeners from "./listeners.js"; import queue from "@/helpers/queue.js"; -import { chunked as upload } from "@/helpers/upload.js"; +import { uploadAsChunks } from "@/helpers/upload.js"; import { extension, name, niceSize } from "@/helpers/file.js"; export const defaults = () => { @@ -312,7 +312,7 @@ export default (panel) => { }, async upload(file, attributes) { try { - const response = await upload( + const response = await uploadAsChunks( file.src, { attributes: attributes, diff --git a/src/Api/Upload.php b/src/Api/Upload.php index 63c869736e..84dc55cff8 100644 --- a/src/Api/Upload.php +++ b/src/Api/Upload.php @@ -48,7 +48,9 @@ public static function chunkId(string $id): string $id = Str::slug($id, '', 'a-z0-9'); if (strlen($id) < 3) { - throw new InvalidArgumentException('Chunk ID must at least be 3 characters long'); + throw new InvalidArgumentException( + 'Chunk ID must at least be 3 characters long' + ); } return $id; @@ -76,9 +78,9 @@ public static function chunkSize(): int /** * Clean up tmp directory of stale files */ - public static function cleanTmpDirectory(): void + public static function cleanTmpDir(): void { - foreach (Dir::files($dir = static::tmp(), [], true) as $file) { + foreach (Dir::files($dir = static::tmpDir(), [], true) as $file) { // remove any file that hasn't been altered // in the last 24 hours if (F::modified($file) < time() - 86400) { @@ -212,7 +214,7 @@ public function processChunk( string $filename ): string|null { // ensure the tmp upload directory exists - Dir::make($dir = static::tmp()); + Dir::make($dir = static::tmpDir()); // create path for file in tmp upload directory; // prefix with id while file isn't completely uploaded yet @@ -318,7 +320,7 @@ public function source(string $source, string $filename): string * temporarily storing (incomplete) uploads * @codeCoverageIgnore */ - protected static function tmp(): string + protected static function tmpDir(): string { return App::instance()->root('cache') . '/.uploads'; } @@ -363,7 +365,9 @@ protected static function validateChunk( // sent chunk is expected to be the first part, // but tmp file already exists if (F::exists($tmp) === true) { - throw new DuplicateException('A tmp file upload with the same filename and upload id already exists: ' . $filename); + throw new DuplicateException( + 'A tmp file upload with the same filename and upload id already exists: ' . $filename + ); } // validate file (extension, name) for first chunk; @@ -378,12 +382,16 @@ protected static function validateChunk( // validate subsequent chunks: // no tmp in place if (F::exists($tmp) === false) { - throw new NotFoundException('Chunk offset ' . $offset . ' for non-existing tmp file: ' . $filename); + throw new NotFoundException( + 'Chunk offset ' . $offset . ' for non-existing tmp file: ' . $filename + ); } // sent chunk's offset is not the continuation of the tmp file if ($offset !== F::size($tmp)) { - throw new InvalidArgumentException('Chunk offset ' . $offset . ' does not match the existing tmp upload file size of ' . F::size($tmp)); + throw new InvalidArgumentException( + 'Chunk offset ' . $offset . ' does not match the existing tmp upload file size of ' . F::size($tmp) + ); } } diff --git a/src/Panel/Panel.php b/src/Panel/Panel.php index 23aea3ac02..7b0e525642 100644 --- a/src/Panel/Panel.php +++ b/src/Panel/Panel.php @@ -150,7 +150,7 @@ protected static function garbage(): void // run garbage collection with a chance of 10%; if (mt_rand(1, 10000) <= 0.1 * 10000) { // clean up leftover upload chunks - Upload::cleanTmpDirectory(); + Upload::cleanTmpDir(); } } diff --git a/tests/Api/UploadTest.php b/tests/Api/UploadTest.php index 483fedd285..73c97d5789 100644 --- a/tests/Api/UploadTest.php +++ b/tests/Api/UploadTest.php @@ -107,9 +107,9 @@ public function testChunkSize() } /** - * @covers ::cleanTmpDirectory + * @covers ::cleanTmpDir */ - public function testCleanTmpDirectory() + public function testCleanTmpDir() { $dir = static::TMP . '/site/cache/.uploads'; F::write($a = $dir . '/abcd-a.md', ''); @@ -120,7 +120,7 @@ public function testCleanTmpDirectory() $this->assertFileExists($a); $this->assertFileExists($b); - Upload::cleanTmpDirectory(); + Upload::cleanTmpDir(); $this->assertDirectoryExists($dir); $this->assertFileExists($a); @@ -128,7 +128,7 @@ public function testCleanTmpDirectory() touch($a, time() - 86400 - 1); - Upload::cleanTmpDirectory(); + Upload::cleanTmpDir(); $this->assertDirectoryDoesNotExist($dir); $this->assertFileDoesNotExist($a);