From 90935871ba884d69a6c3a8e417d114c641ca0928 Mon Sep 17 00:00:00 2001 From: Pixo Date: Tue, 21 Nov 2023 20:25:56 +0100 Subject: [PATCH] [_]:WIP test multipart upload --- .../Services/Network/Encrypt.swift | 2 + .../Services/Network/NetworkFacade.swift | 34 ++++++++++++--- .../Services/Network/Upload.swift | 3 +- .../Services/Network/UploadMultipart.swift | 43 +++++++++++-------- .../Types/NetworkTypes.swift | 8 +++- Sources/InternxtSwiftCore/Utils/Errors.swift | 1 + 6 files changed, 66 insertions(+), 25 deletions(-) diff --git a/Sources/InternxtSwiftCore/Services/Network/Encrypt.swift b/Sources/InternxtSwiftCore/Services/Network/Encrypt.swift index cfff222..df88de0 100644 --- a/Sources/InternxtSwiftCore/Services/Network/Encrypt.swift +++ b/Sources/InternxtSwiftCore/Services/Network/Encrypt.swift @@ -81,6 +81,8 @@ public struct Encrypt { } + + public func encrypt(string: String, password: String, salt: [UInt8], iv: Data, rounds: Int = 2145) throws -> Data{ diff --git a/Sources/InternxtSwiftCore/Services/Network/NetworkFacade.swift b/Sources/InternxtSwiftCore/Services/Network/NetworkFacade.swift index 5b679bc..5c098a1 100644 --- a/Sources/InternxtSwiftCore/Services/Network/NetworkFacade.swift +++ b/Sources/InternxtSwiftCore/Services/Network/NetworkFacade.swift @@ -6,6 +6,7 @@ // import Foundation +import CryptoKit let MULTIPART_MIN_SIZE = 100 * 1024 * 1024; let MULTIPART_CHUNK_SIZE = 100 * 1024 * 1024; @@ -99,6 +100,8 @@ public struct NetworkFacade { return try await upload.start(index: index, bucketId: bucketId, mnemonic: mnemonic, encryptedFileURL: encryptedOutput, progressHandler: progressHandler) } + + private func runMultipartUpload( input: InputStream, fileSize: Int, @@ -109,6 +112,9 @@ public struct NetworkFacade { progressHandler: @escaping ProgressHandler, debug: Bool = false ) async throws -> FinishUploadResponse { + var hasher = SHA256.init() + + let parts = 3 @@ -128,13 +134,13 @@ public struct NetworkFacade { let hash = encrypt.getFileContentHash(stream: InputStream(data: encryptedChunk)) let uploadUrl = uploadUrls[partIndex] - try await uploadMultipart.uploadPart(encryptedChunk: encryptedChunk, uploadUrl: uploadUrl, partIndex: partIndex){progress in + let etag = try await uploadMultipart.uploadPart(encryptedChunk: encryptedChunk, uploadUrl: uploadUrl, partIndex: partIndex){progress in //print("UPLOAD PROGRESS FOR PART \(partIndex)", progress) } + let uploadedPartConfig = UploadedPartConfig( - hash: hash, - uuid: startUploadResult.uuid + etag: etag, partNumber: partIndex + 1 ) uploadedPartsConfigs.append(uploadedPartConfig) @@ -148,6 +154,7 @@ public struct NetworkFacade { key: fileKey, iv: iv ){encryptedChunk in + hasher.update(data: encryptedChunk) // If something fails here, the error is propagated // and the stream reading is stopped try await processEncryptedChunk(encryptedChunk: encryptedChunk, partIndex: partIndex) @@ -155,8 +162,25 @@ public struct NetworkFacade { partIndex += 1 } - - let finishUpload = try await uploadMultipart.finishUpload(bucketId: bucketId, uploadedParts: uploadedPartsConfigs, index: Data(index), debug: debug) + + let fileSHA256digest = hasher.finalize() + + var sha256Hash = [UInt8]() + fileSHA256digest.withUnsafeBytes {bytes in + sha256Hash.append(contentsOf: bytes) + } + + let fileHash = HMAC().ripemd160(message: Data(sha256Hash)) + + + let finishUpload = try await uploadMultipart.finishUpload( + bucketId: bucketId, + fileHash: fileHash.toHexString(), + uploadUuid: startUploadResult.uuid, + uploadedParts: uploadedPartsConfigs, + index: Data(index), + debug: debug + ) print("Chunk number \(partIndex) uploaded") return finishUpload } diff --git a/Sources/InternxtSwiftCore/Services/Network/Upload.swift b/Sources/InternxtSwiftCore/Services/Network/Upload.swift index bc0e8fc..8fec684 100644 --- a/Sources/InternxtSwiftCore/Services/Network/Upload.swift +++ b/Sources/InternxtSwiftCore/Services/Network/Upload.swift @@ -91,7 +91,8 @@ public class Upload: NSObject { var shards: Array = Array() shards.append(ShardUploadPayload( hash: cryptoUtils.bytesToHexString(Array(fileHash)), - uuid: uploadResult.uuid + uuid: uploadResult.uuid, + parts: nil )) let finishUploadResult = try await networkAPI.finishUpload(bucketId: bucketId, payload: FinishUploadPayload( index: cryptoUtils.bytesToHexString(index), diff --git a/Sources/InternxtSwiftCore/Services/Network/UploadMultipart.swift b/Sources/InternxtSwiftCore/Services/Network/UploadMultipart.swift index 989c41e..af3d758 100644 --- a/Sources/InternxtSwiftCore/Services/Network/UploadMultipart.swift +++ b/Sources/InternxtSwiftCore/Services/Network/UploadMultipart.swift @@ -14,8 +14,8 @@ enum UploadMultipartError: Error { } public struct UploadedPartConfig { - let hash: Data - let uuid: String + let etag: String + let partNumber: Int } @available(macOS 10.15, *) @@ -69,27 +69,32 @@ public class UploadMultipart: NSObject { return startUploadResult } - func uploadPart(encryptedChunk: Data, uploadUrl: String, partIndex: Int, progressHandler: @escaping ProgressHandler) async throws -> Void { + func uploadPart(encryptedChunk: Data, uploadUrl: String, partIndex: Int, progressHandler: @escaping ProgressHandler) async throws -> String { // Upload the chunk to the given URL - let successUpload = try await self.uploadEncryptedChunk(encryptedChunk: encryptedChunk, uploadUrl: uploadUrl, progressHandler: progressHandler) - - if successUpload == false { - throw UploadError.UploadNotSuccessful - } + let uploadEtag = try await self.uploadEncryptedChunk(encryptedChunk: encryptedChunk, uploadUrl: uploadUrl, progressHandler: progressHandler) + + return uploadEtag } - func finishUpload(bucketId: String, uploadedParts: [UploadedPartConfig], index: Data, debug: Bool = false) async throws -> FinishUploadResponse { + func finishUpload(bucketId: String, fileHash: String, uploadUuid: String, uploadedParts: [UploadedPartConfig], index: Data, debug: Bool = false) async throws -> FinishUploadResponse { + + var shards: Array = Array() - uploadedParts.forEach{uploadedPart in - shards.append(ShardUploadPayload( - hash: uploadedPart.hash.toHexString(), - uuid: uploadedPart.uuid - )) - } + let shardPayload = ShardUploadPayload( + hash: fileHash, + uuid: uploadUuid, + parts: uploadedParts.map{ uploadedPart in + return ShardPartPayload( + ETag: uploadedPart.etag, + PartNumber: uploadedPart.partNumber + ) + } + ) + shards.append(shardPayload) let payload = try FinishUploadPayload( @@ -97,7 +102,6 @@ public class UploadMultipart: NSObject { shards: shards ) - print("UPLOADS", payload) let finishUploadResult = try await networkAPI.finishUpload( bucketId: bucketId, payload: payload, @@ -109,7 +113,7 @@ public class UploadMultipart: NSObject { - private func uploadEncryptedChunk(encryptedChunk: Data, uploadUrl: String, progressHandler: ProgressHandler?) async throws -> Bool { + private func uploadEncryptedChunk(encryptedChunk: Data, uploadUrl: String, progressHandler: ProgressHandler?) async throws -> String { return try await withCheckedThrowingContinuation { (continuation) in var request = URLRequest( url: URL(string: uploadUrl)!, @@ -128,7 +132,10 @@ public class UploadMultipart: NSObject { if response?.statusCode != 200 { return continuation.resume(with: .failure(UploadError.UploadNotSuccessful)) } else { - return continuation.resume(with: .success(true)) + guard let etagValue = response?.value(forHTTPHeaderField: "Etag") else { + return continuation.resume(with: .failure(UploadError.MissingEtag)) + } + return continuation.resume(with: .success(etagValue)) } } diff --git a/Sources/InternxtSwiftCore/Types/NetworkTypes.swift b/Sources/InternxtSwiftCore/Types/NetworkTypes.swift index 8589a13..a1b644e 100644 --- a/Sources/InternxtSwiftCore/Types/NetworkTypes.swift +++ b/Sources/InternxtSwiftCore/Types/NetworkTypes.swift @@ -27,14 +27,20 @@ public struct StartUploadResponse: Decodable { public let uploads: Array } +public struct ShardPartPayload: Codable { + public let ETag: String + public let PartNumber: Int +} public struct ShardUploadPayload: Codable { public let hash: String public let uuid: String + public let parts: [ShardPartPayload]? - init(hash: String, uuid: String) { + init(hash: String, uuid: String, parts: [ShardPartPayload]? = nil) { self.hash = hash self.uuid = uuid + self.parts = parts } } diff --git a/Sources/InternxtSwiftCore/Utils/Errors.swift b/Sources/InternxtSwiftCore/Utils/Errors.swift index 65e15d9..6a78720 100644 --- a/Sources/InternxtSwiftCore/Utils/Errors.swift +++ b/Sources/InternxtSwiftCore/Utils/Errors.swift @@ -45,6 +45,7 @@ enum UploadError: Error { case MissingUploadUrl case UploadNotSuccessful case UploadedSizeNotMatching + case MissingEtag }