feat: uploading blocks of a file by index (#270)

* feat: implemented uploading data by block index

* feat: tests and docs for uploading by block index

README.md

@@ -224,6 +224,22 @@ await fdp.file.uploadData('my-new-pod', '/my-dir/myfile.txt', 'Hello world!', {
 })
 ```
 
+Data can also be uploaded block by block, even without an FDP account. Each block will be secured by a Swarm node. Later, with an FDP account, the data can be finalized in the form of a file.
+```js
+const data = '123'
+const blockSize = 1 // recommended value is 1000000 bytes
+const blocksCount = 3
+const blocks = []
+for (let i = 0; i < blocksCount; i++) {
+  const dataBlock = getDataBlock(data, blockSize, i)
+  // fdp instance with or without logged in user
+  blocks.push(await fdp.file.uploadDataBlock(dataBlock, i))
+}
+
+// fdp instance with logged in user
+const fileMeta = await fdp.file.uploadData(pod, fullPath, blocks)
+```
+
 Deleting a file from a pod
 
 ```js

src/file/file.ts

@@ -12,9 +12,9 @@ import {
   uploadBytes,
 } from './utils'
 import { writeFeedData } from '../feed/api'
-import { downloadData, uploadData } from './handler'
+import { downloadData, uploadData, uploadDataBlock } from './handler'
 import { getFileMetadataRawBytes, rawFileMetadataToFileMetadata } from './adapter'
-import { DataDownloadOptions, DataUploadOptions, FileReceiveOptions, FileShareInfo } from './types'
+import { DataDownloadOptions, DataUploadOptions, ExternalDataBlock, FileReceiveOptions, FileShareInfo } from './types'
 import { addEntryToDirectory, DEFAULT_UPLOAD_OPTIONS, removeEntryFromDirectory } from '../content-items/handler'
 import { Reference } from '@ethersphere/bee-js'
 import { getRawMetadata } from '../content-items/utils'
@@ -53,13 +53,13 @@ export class File {
    *
    * @param podName pod where file is stored
    * @param fullPath full path of the file
-   * @param data file content
+   * @param data file content or ExternalDataBlock[] indexed in ascending order
    * @param options upload options
    */
   async uploadData(
     podName: string,
     fullPath: string,
-    data: Uint8Array | string,
+    data: Uint8Array | string | ExternalDataBlock[],
     options?: DataUploadOptions,
   ): Promise<FileMetadata> {
     options = { ...DEFAULT_UPLOAD_OPTIONS, ...options }
@@ -153,4 +153,17 @@ export class File {
 
     return meta
   }
+
+  /**
+   * Uploads a data block without constructing a file metadata
+   *
+   * @param block block data
+   * @param blockIndex block index
+   */
+  async uploadDataBlock(block: Uint8Array, blockIndex: number): Promise<ExternalDataBlock> {
+    return {
+      ...(await uploadDataBlock(this.accountData.connection, block)),
+      index: blockIndex,
+    }
+  }
 }

src/file/handler.ts

@@ -3,11 +3,14 @@ import { Bee, Data, BeeRequestOptions } from '@ethersphere/bee-js'
 import { EthAddress } from '@ethersphere/bee-js/dist/types/utils/eth'
 import {
   assertFullPathWithName,
+  assertSequenceOfExternalDataBlocksCorrect,
   calcUploadBlockPercentage,
   DEFAULT_FILE_PERMISSIONS,
   downloadBlocksManifest,
-  extractPathInfo,
+  externalDataBlocksToBlocks,
+  extractPathInfo, getDataBlock,
   getFileMode,
+  isExternalDataBlocks,
   updateDownloadProgress,
   updateUploadProgress,
   uploadBytes,
@@ -17,7 +20,15 @@ import { blocksToManifest, getFileMetadataRawBytes, rawFileMetadataToFileMetadat
 import { assertRawFileMetadata } from '../directory/utils'
 import { getCreationPathInfo, getRawMetadata } from '../content-items/utils'
 import { PodPasswordBytes } from '../utils/encryption'
-import { Blocks, DataDownloadOptions, DataUploadOptions, DownloadProgressType, UploadProgressType } from './types'
+import {
+  Block,
+  Blocks,
+  DataDownloadOptions,
+  DataUploadOptions,
+  DownloadProgressType,
+  ExternalDataBlock,
+  UploadProgressType,
+} from './types'
 import { assertPodName, getExtendedPodsListByAccountData, META_VERSION } from '../pod/utils'
 import { getUnixTimestamp } from '../utils/time'
 import { addEntryToDirectory, DEFAULT_UPLOAD_OPTIONS } from '../content-items/handler'
@@ -26,6 +37,7 @@ import { AccountData } from '../account/account-data'
 import { prepareEthAddress } from '../utils/wallet'
 import { assertWallet } from '../utils/type'
 import { getNextEpoch } from '../feed/lookup/utils'
+import { Connection } from '../connection/connection'
 
 /**
  * File prefix
@@ -134,7 +146,7 @@ export function generateBlockName(blockNumber: number): string {
 export async function uploadData(
   podName: string,
   fullPath: string,
-  data: Uint8Array | string,
+  data: Uint8Array | string | ExternalDataBlock[],
   accountData: AccountData,
   options: DataUploadOptions,
 ): Promise<FileMetadata> {
@@ -144,8 +156,6 @@ export async function uploadData(
 
   const blockSize = options.blockSize ?? Number(DEFAULT_UPLOAD_OPTIONS!.blockSize)
   const contentType = options.contentType ?? String(DEFAULT_UPLOAD_OPTIONS!.contentType)
-
-  data = typeof data === 'string' ? stringToBytes(data) : data
   const connection = accountData.connection
   updateUploadProgress(options, UploadProgressType.GetPodInfo)
   const { podWallet, pod } = await getExtendedPodsListByAccountData(accountData, podName)
@@ -159,23 +169,28 @@ export async function uploadData(
   )
   const pathInfo = extractPathInfo(fullPath)
   const now = getUnixTimestamp()
-  const totalBlocks = Math.ceil(data.length / blockSize)
+
   const blocks: Blocks = { blocks: [] }
-  for (let i = 0; i < totalBlocks; i++) {
-    const blockData = {
-      totalBlocks,
-      currentBlockId: i,
-      percentage: calcUploadBlockPercentage(i, totalBlocks),
+  let fileSize = data.length
+
+  if (isExternalDataBlocks(data)) {
+    assertSequenceOfExternalDataBlocksCorrect(data)
+    blocks.blocks = externalDataBlocksToBlocks(data)
+    fileSize = data.reduce((acc, block) => acc + block.size, 0)
+  } else {
+    data = typeof data === 'string' ? stringToBytes(data) : data
+    const totalBlocks = Math.ceil(data.length / blockSize)
+    for (let i = 0; i < totalBlocks; i++) {
+      const blockData = {
+        totalBlocks,
+        currentBlockId: i,
+        percentage: calcUploadBlockPercentage(i, totalBlocks),
+      }
+      updateUploadProgress(options, UploadProgressType.UploadBlockStart, blockData)
+      const currentBlock = getDataBlock(data, blockSize, i)
+      blocks.blocks.push(await uploadDataBlock(connection, currentBlock))
+      updateUploadProgress(options, UploadProgressType.UploadBlockEnd, blockData)
     }
-    updateUploadProgress(options, UploadProgressType.UploadBlockStart, blockData)
-    const currentBlock = data.slice(i * blockSize, (i + 1) * blockSize)
-    const result = await uploadBytes(connection, currentBlock)
-    blocks.blocks.push({
-      size: currentBlock.length,
-      compressedSize: currentBlock.length,
-      reference: result.reference,
-    })
-    updateUploadProgress(options, UploadProgressType.UploadBlockEnd, blockData)
   }
 
   updateUploadProgress(options, UploadProgressType.UploadBlocksMeta)
@@ -185,7 +200,7 @@ export async function uploadData(
     version: META_VERSION,
     filePath: pathInfo.path,
     fileName: pathInfo.filename,
-    fileSize: data.length,
+    fileSize,
     blockSize,
     contentType,
     compression: '',
@@ -212,3 +227,16 @@ export async function uploadData(
 
   return meta
 }
+
+/**
+ * Upload data block
+ * @param connection connection
+ * @param block block to upload
+ */
+export async function uploadDataBlock(connection: Connection, block: Uint8Array): Promise<Block> {
+  return {
+    size: block.length,
+    compressedSize: block.length,
+    reference: (await uploadBytes(connection, block)).reference,
+  }
+}

src/file/types.ts

@@ -173,6 +173,16 @@ export interface Block {
   reference: Reference
 }
 
+/**
+ * FDP file block format for external usage
+ */
+export interface ExternalDataBlock extends Block {
+  /**
+   * Block index
+   */
+  index: number
+}
+
 /**
  * File share information
  */

src/file/utils.ts

@@ -11,15 +11,19 @@ import {
   UploadProgressType,
   DataDownloadOptions,
   DownloadProgressType,
+  ExternalDataBlock,
+  Block,
 } from './types'
 import { rawBlocksToBlocks } from './adapter'
 import CryptoJS from 'crypto-js'
-import { assertArray, assertString, isNumber, isObject, isString } from '../utils/type'
+import { assertArray, assertNumber, assertString, isNumber, isObject, isString } from '../utils/type'
 import { FileMetadata, RawFileMetadata } from '../pod/types'
 import { EncryptedReference } from '../utils/hex'
 import { isRawFileMetadata, splitPath } from '../directory/utils'
 import { getUnixTimestamp } from '../utils/time'
 import { jsonParse } from '../utils/json'
+import { assertReference } from '../utils/string'
+import { stringToBytes } from '../utils/bytes'
 
 /**
  * Default file permission in octal format
@@ -298,3 +302,110 @@ export function calcUploadBlockPercentage(blockId: number, totalBlocks: number):
 
   return Math.round(((blockId + 1) / totalBlocks) * 100)
 }
+
+/**
+ * Asserts that a given value is an ExternalDataBlock
+ * @param value The value to assert
+ */
+export function assertExternalDataBlock(value: unknown): asserts value is ExternalDataBlock {
+  if (typeof value !== 'object' || value === null) {
+    throw new Error('Expected an object for ExternalDataBlock')
+  }
+
+  const block = value as ExternalDataBlock
+
+  assertNumber(block.size, 'Expected "size" to be a number')
+  assertNumber(block.compressedSize, 'Expected "compressedSize" to be a number')
+  assertReference(block.reference)
+  assertNumber(block.index, 'Expected "index" to be a number')
+}
+
+/**
+ * Checks if the given value is an ExternalDataBlock
+ * @param value The value to check
+ */
+export function isExternalDataBlock(value: unknown): boolean {
+  try {
+    assertExternalDataBlock(value)
+
+    return true
+  } catch (e) {
+    return false
+  }
+}
+
+/**
+ * Asserts that a given value is an array of ExternalDataBlock
+ * @param value The value to assert
+ */
+export function assertExternalDataBlocks(value: unknown): asserts value is ExternalDataBlock[] {
+  if (!Array.isArray(value)) {
+    throw new Error('Expected an array for ExternalDataBlocks')
+  }
+
+  for (const block of value) {
+    assertExternalDataBlock(block)
+  }
+}
+
+/**
+ * Checks if the given value is an array of ExternalDataBlock
+ * @param value The value to check
+ */
+export function isExternalDataBlocks(value: unknown): value is ExternalDataBlock[] {
+  try {
+    assertExternalDataBlocks(value)
+
+    return true
+  } catch (e) {
+    return false
+  }
+}
+
+/**
+ * Converts ExternalDataBlock[] to Block[]
+ * @param externalDataBlocks The ExternalDataBlock[] to convert
+ */
+export function externalDataBlocksToBlocks(externalDataBlocks: ExternalDataBlock[]): Block[] {
+  return externalDataBlocks.map(block => ({
+    size: block.size,
+    compressedSize: block.compressedSize,
+    reference: block.reference,
+  }))
+}
+
+/**
+ * Checks if the sequence of ExternalDataBlocks is correctly indexed and sorted.
+ * @param externalDataBlocks The array of ExternalDataBlocks to check.
+ */
+export function isSequenceOfExternalDataBlocksCorrect(externalDataBlocks: ExternalDataBlock[]): boolean {
+  for (let i = 0; i < externalDataBlocks.length; i++) {
+    if (externalDataBlocks[i].index !== i) {
+      return false
+    }
+  }
+
+  return true
+}
+
+/**
+ * Asserts that the sequence of ExternalDataBlocks is correctly indexed.
+ * @param externalDataBlocks The array of ExternalDataBlocks to assert.
+ */
+export function assertSequenceOfExternalDataBlocksCorrect(externalDataBlocks: ExternalDataBlock[]): void {
+  if (!isSequenceOfExternalDataBlocksCorrect(externalDataBlocks)) {
+    throw new Error('The sequence of `ExternalDataBlock` is not correctly indexed.')
+  }
+}
+
+/**
+ * Gets data block by index from data
+ * @param data Data
+ * @param blockSize Size of block
+ * @param blockIndex Index of block
+ */
+export function getDataBlock(data: string | Uint8Array, blockSize: number, blockIndex: number): Uint8Array {
+  data = typeof data === 'string' ? stringToBytes(data) : data
+
+  return data.slice(blockIndex * blockSize, (blockIndex + 1) * blockSize)
+}

src/types.ts

@@ -11,7 +11,18 @@ export {
   DownloadProgressInfo,
   DataUploadOptions,
   DataDownloadOptions,
+  Block,
+  ExternalDataBlock,
 } from './file/types'
+export {
+  calcUploadBlockPercentage,
+  assertExternalDataBlock,
+  isExternalDataBlock,
+  assertExternalDataBlocks,
+  isExternalDataBlocks,
+  externalDataBlocksToBlocks,
+  getDataBlock,
+} from './file/utils'
 
 /**
  * Fair Data Protocol options

src/utils/string.ts

@@ -1,4 +1,5 @@
 import CryptoJS from 'crypto-js'
+import { ENCRYPTED_REFERENCE_HEX_LENGTH, Reference, REFERENCE_HEX_LENGTH, Utils } from '@ethersphere/bee-js'
 
 /**
  * Replace all occurrences of a string with another string
@@ -17,3 +18,15 @@ export function replaceAll(data: string, search: string, replacement: string): s
 export function generateRandomBase64String(length = 10): string {
   return CryptoJS.lib.WordArray.random(length).toString(CryptoJS.enc.Base64).substring(0, length)
 }
+
+/**
+ * Asserts that the given value is a Reference
+ * @param value value to assert
+ */
+export function assertReference(value: unknown): asserts value is Reference {
+  try {
+    Utils.assertHexString(value, REFERENCE_HEX_LENGTH)
+  } catch (e) {
+    Utils.assertHexString(value, ENCRYPTED_REFERENCE_HEX_LENGTH)
+  }
+}

src/utils/type.ts

@@ -8,9 +8,9 @@ export const ETH_ADDR_HEX_LENGTH = 40
 /**
  * Asserts that the given value is a number
  */
-export function assertNumber(value: unknown): asserts value is number {
+export function assertNumber(value: unknown, customMessage?: string): asserts value is number {
   if (!isNumber(value)) {
-    throw new Error('Expected a number')
+    throw new Error(customMessage ? customMessage : 'Expected a number')
   }
 }