Skip to content

Commit

Permalink
Chunked file uploads
Browse files Browse the repository at this point in the history
  • Loading branch information
distantnative committed May 26, 2024
1 parent 2ea5404 commit 4f3a616
Show file tree
Hide file tree
Showing 12 changed files with 791 additions and 19 deletions.
21 changes: 15 additions & 6 deletions config/api/routes/files.php
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
<?php

use Kirby\Api\Upload;

// routing pattern to match all models with files
$filePattern = '(account/|pages/[^/]+/|site/|users/[^/]+/|)files/(:any)';
$filePattern = '(account/|pages/[^/]+/|site/|users/[^/]+/|)files/(:any)';
$parentPattern = '(account|pages/[^/]+|site|users/[^/]+)/files';

/**
Expand Down Expand Up @@ -47,17 +49,24 @@
// move_uploaded_file() not working with unit test
// @codeCoverageIgnoreStart
return $this->upload(function ($source, $filename) use ($path) {
$props = [
// check if upload is sent in chunks and handle them
$source = Upload::chunk($this, $source, $filename);

// complete files return an absolute path;
// if file is not yet complete, end here
if ($source === null) {
return;
}

// move the source file to the content folder
return $this->parent($path)->createFile([
'content' => [
'sort' => $this->requestBody('sort')
],
'source' => $source,
'template' => $this->requestBody('template'),
'filename' => $filename
];

// move the source file from the temp dir
return $this->parent($path)->createFile($props, true);
], true);
});
// @codeCoverageIgnoreEnd
}
Expand Down
48 changes: 46 additions & 2 deletions panel/src/helpers/upload.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,45 @@
import { random } from "./string.js";

/**
* Uploads a file in chunks
* @param {File} file - file to upload
* @param {Object} params - upload options (see `upload` method for details)
* @param {number} size - chunk size in bytes
*/
export async function chunked(file, params, size = 5242880) {
const parts = Math.ceil(file.size / size);
const id = random(4).toLowerCase();
let response;

for (let i = 0; i < parts; i++) {
const start = i * size;
const end = Math.min(start + size, file.size);
const chunk = parts > 1 ? file.slice(start, end, file.type) : file;

// when more than one part, add flag to
// recognize chunked upload and its last chunk
if (parts > 1) {
params.headers = {
...params.headers,
"Upload-Length": file.size,
"Upload-Offset": start,
"Upload-Id": id
};
}

response = await upload(chunk, {
...params,
// calculate the total progress based on chunk progress
progress: (xhr, chunk, percent) => {
const total = (i + percent / 100) / parts;
params.progress(xhr, file, Math.round(total * 100));
}
});
}

return response;
}

/**
* Uploads a file using XMLHttpRequest.
*
Expand All @@ -13,7 +55,7 @@
* @param {Function} params.success - callback when upload succeeded
* @param {Function} params.error - callback when upload failed
*/
export default async (file, params) => {
export async function upload(file, params) {
return new Promise((resolve, reject) => {
const defaults = {
url: "/",
Expand Down Expand Up @@ -96,4 +138,6 @@ export default async (file, params) => {

xhr.send(data);
});
};
}

export default upload;
24 changes: 14 additions & 10 deletions panel/src/panel/upload.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { uuid } from "@/helpers/string";
import State from "./state.js";
import listeners from "./listeners.js";
import queue from "@/helpers/queue.js";
import upload from "@/helpers/upload.js";
import { chunked as upload } from "@/helpers/upload.js";
import { extension, name, niceSize } from "@/helpers/file.js";

export const defaults = () => {
Expand Down Expand Up @@ -312,15 +312,19 @@ export default (panel) => {
},
async upload(file, attributes) {
try {
const response = await upload(file.src, {
attributes: attributes,
headers: { "x-csrf": panel.system.csrf },
filename: file.name + "." + file.extension,
url: this.url,
progress: (xhr, src, progress) => {
file.progress = progress;
}
});
const response = await upload(
file.src,
{
attributes: attributes,
headers: { "x-csrf": panel.system.csrf },
filename: file.name + "." + file.extension,
url: this.url,
progress: (xhr, src, progress) => {
file.progress = progress;
}
},
panel.config.upload
);

file.completed = true;
file.model = response.data;
Expand Down
2 changes: 2 additions & 0 deletions src/Api/Api.php
Original file line number Diff line number Diff line change
Expand Up @@ -605,6 +605,8 @@ protected function setRequestMethod(
* Added debug parameter for testing purposes as we did in the Email class
*
* @throws \Exception If request has no files or there was an error with the upload
*
* @todo Move most of the logic to `Api\Upload` class
*/
public function upload(
Closure $callback,
Expand Down
214 changes: 214 additions & 0 deletions src/Api/Upload.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,214 @@
<?php

namespace Kirby\Api;

use Kirby\Cms\App;
use Kirby\Cms\File;
use Kirby\Cms\FileRules;
use Kirby\Cms\Page;
use Kirby\Exception\DuplicateException;
use Kirby\Exception\Exception;
use Kirby\Exception\InvalidArgumentException;
use Kirby\Exception\NotFoundException;
use Kirby\Filesystem\Dir;
use Kirby\Filesystem\F;
use Kirby\Toolkit\Str;

/**
* The Upload class supports file uploads in the
* context of the API
*
* @package Kirby Api
* @author Nico Hoffmann <[email protected]>
* @link https://getkirby.com
* @copyright Bastian Allgeier
* @license https://getkirby.com/license
* @since 4.3.0
* @internal
*/
class Upload
{
/**
* Handle chunked uploads by merging all chunks
* in the tmp directory and only returning the new
* $source path to the tmp file once complete
*
* @throws \Kirby\Exception\DuplicateException Duplicate first chunk (same filename and id)
* @throws \Kirby\Exception\Exception Chunk offset does not match existing tmp file
* @throws \Kirby\Exception\NotFoundException Subsequent chunk has no existing tmp file
*/
public static function chunk(
Api $api,
string $source,
string $filename
): string|null {
// if the file is uploaded in chunks…
if ($total = (int)$api->requestHeaders('Upload-Length')) {
// ensure the tmp upload directory exists
Dir::make($dir = static::tmp());

// create path for file in tmp upload directory;
// prefix with id while file isn't completely uploaded yet
$id = static::chunkId($api->requestHeaders('Upload-Id'));
$filename = basename($filename);
$tmpRoot = $dir . '/' . $id . '-' . $filename;

// validate various aspects of the request
// to ensure the chunk isn't trying to do malicious actions
static::validateChunk(
source: $source,
tmp: $tmpRoot,
total: $total,
offset: $api->requestHeaders('Upload-Offset'),
template: $api->requestBody('template'),
);

// stream chunk content and append it to partial file
stream_copy_to_stream(
fopen($source, 'r'),
fopen($tmpRoot, 'a')
);

// clear file stat cache so the following call to `F::size`
// really returns the updated file size
clearstatcache();

// if file isn't complete yet, return early
if (F::size($tmpRoot) < $total) {
return null;
}

// remove id from partial filename now the file is complete,
// so we can pass the path from the tmp upload directory
// as new source path for the file back to the API upload method
rename(
$tmpRoot,
$newRoot = $dir . '/' . $filename
);

return $newRoot;
}

return $source;
}

/**
* Ensures a clean chunk ID by stripping forbidden characters
*/
public static function chunkId(string $id): string
{
return Str::slug($id, '', 'a-z0-9');
}

/**
* Returns the ideal size for a file chunk
*/
public static function chunkSize(): int
{
$max = [
Str::toBytes(ini_get('upload_max_filesize')),
Str::toBytes(ini_get('post_max_size'))
];

// consider cloudflare proxy limit, if detected
if (isset($_SERVER['HTTP_CF_CONNECTING_IP']) === true) {
$max[] = Str::toBytes('100M');
}

// to be sure, only use 95% of the max possible upload size
return (int)floor(min($max) * 0.95);
}

/**
* Clean up tmp directory of stale files
*/
public static function clean(): void
{
foreach (Dir::files($dir = static::tmp(), [], true) as $file) {
// remove any file that hasn't been altered
// in the last 24 hours
if (F::modified($file) < time() - 86400) {
F::remove($file);
}
}

// remove tmp directory if completely empty
if (Dir::isEmpty($dir) === true) {
Dir::remove($dir);
}
}

/**
* Returns root of directory used for
* temporarily storing (incomplete) uploads
* @codeCoverageIgnore
*/
protected static function tmp(): string
{
return App::instance()->root('cache') . '/.uploads';
}

/**
* Ensures the sent chunk is valid
*
* @throws \Kirby\Exception\DuplicateException Duplicate first chunk (same filename and id)
* @throws \Kirby\Exception\Exception Chunk offset does not match existing tmp file
* @throws \Kirby\Exception\InvalidArgumentException The maximum file size for this blueprint was exceeded
* @throws \Kirby\Exception\NotFoundException Subsequent chunk has no existing tmp file
*/
protected static function validateChunk(
string $source,
string $tmp,
int $total,
int $offset,
string|null $template = null
): void {
$file = new File([
'parent' => new Page(['slug' => 'tmp']),
'filename' => $filename = basename($tmp),
'template' => $template
]);

// if the blueprint `maxsize` option is set,
// ensure that the total size communicated in the header
// as well as the current tmp size after adding this chunk
// does not exceed the max limit
if (
($max = $file->blueprint()->accept()['maxsize'] ?? null) &&
(
$total > $max ||
(F::size($source) + F::size($tmp)) > $max
)
) {
throw new InvalidArgumentException(['key' => 'file.maxsize']);
}

// validate the first chunk
if ($offset === 0) {
// sent chunk is expected to be the first part,
// but tmp file already exists
if (F::exists($tmp) === true) {
throw new DuplicateException('A tmp file upload with the same filename and upload id already exists: ' . $filename);
}

// validate file (extension, name) for first chunk;
// will also be validate again by `$model->createFile()`
// when completely uploaded
FileRules::validFile($file, false);

// first chunk is valid
return;
}

// validate subsequent chunks:
// no tmp in place
if (F::exists($tmp) === false) {
throw new NotFoundException('Chunk offset ' . $offset . ' for non-existing tmp file: ' . $filename);
}

// sent chunk's offset is not the continuation of the tmp file
if ($offset !== F::size($tmp)) {
throw new Exception('Chunk offset ' . $offset . ' does not match the existing tmp upload file size of ' . F::size($tmp));
}
}
}
2 changes: 1 addition & 1 deletion src/Cms/FileRules.php
Original file line number Diff line number Diff line change
Expand Up @@ -315,7 +315,7 @@ public static function validFilename(File $file, string $filename): bool
public static function validMime(File $file, string $mime = null): bool
{
// make it easier to compare the mime
$mime = strtolower($mime);
$mime = strtolower($mime ?? '');

if (empty($mime)) {
throw new InvalidArgumentException([
Expand Down
Loading

0 comments on commit 4f3a616

Please sign in to comment.