diff --git a/src/bee.ts b/src/bee.ts index 9e847d6d..53ac7517 100644 --- a/src/bee.ts +++ b/src/bee.ts @@ -40,6 +40,7 @@ import type { Tag, Topic, UploadOptions, + UploadRedundancyOptions, UploadResultWithCid, } from './types' import { @@ -143,7 +144,7 @@ export class Bee { async uploadData( postageBatchId: string | BatchId, data: string | Uint8Array, - options?: UploadOptions, + options?: UploadOptions & UploadRedundancyOptions, requestOptions?: BeeRequestOptions, ): Promise { assertBatchId(postageBatchId) @@ -264,7 +265,7 @@ export class Bee { postageBatchId: string | BatchId, data: string | Uint8Array | Readable | File, name?: string, - options?: FileUploadOptions, + options?: FileUploadOptions & UploadRedundancyOptions, requestOptions?: BeeRequestOptions, ): Promise { assertBatchId(postageBatchId) @@ -377,7 +378,7 @@ export class Bee { async uploadFiles( postageBatchId: string | BatchId, fileList: FileList | File[], - options?: CollectionUploadOptions, + options?: CollectionUploadOptions & UploadRedundancyOptions, requestOptions?: BeeRequestOptions, ): Promise { assertBatchId(postageBatchId) @@ -405,7 +406,7 @@ export class Bee { async uploadCollection( postageBatchId: string | BatchId, collection: Collection, - options?: CollectionUploadOptions, + options?: CollectionUploadOptions & UploadRedundancyOptions, ): Promise { assertBatchId(postageBatchId) assertCollection(collection) @@ -437,7 +438,7 @@ export class Bee { async uploadFilesFromDirectory( postageBatchId: string | BatchId, dir: string, - options?: CollectionUploadOptions, + options?: CollectionUploadOptions & UploadRedundancyOptions, requestOptions?: BeeRequestOptions, ): Promise { assertBatchId(postageBatchId) diff --git a/src/index.ts b/src/index.ts index db098cf0..239660d0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,7 @@ export * from './utils/error' export * as Utils from './utils/expose' export { Bee, BeeDebug } -// for requrie-like imports +// for require-like imports declare global { interface Window { // binded as 'BeeJs' via Webpack diff --git a/src/modules/bytes.ts b/src/modules/bytes.ts index 3c210788..50ef0b40 100644 --- a/src/modules/bytes.ts +++ b/src/modules/bytes.ts @@ -1,7 +1,16 @@ -import type { BatchId, BeeRequestOptions, Data, Reference, ReferenceOrEns, UploadOptions } from '../types' +import type { + BatchId, + BeeRequestOptions, + Data, + DownloadRedundancyOptions, + Reference, + ReferenceOrEns, + UploadOptions, + UploadRedundancyOptions, +} from '../types' import { UploadResult } from '../types' import { wrapBytesWithHelpers } from '../utils/bytes' -import { extractUploadHeaders } from '../utils/headers' +import { extractDownloadHeaders, extractRedundantUploadHeaders } from '../utils/headers' import { http } from '../utils/http' import { makeTagUid } from '../utils/type' @@ -19,7 +28,7 @@ export async function upload( requestOptions: BeeRequestOptions, data: string | Uint8Array, postageBatchId: BatchId, - options?: UploadOptions, + options?: UploadOptions & UploadRedundancyOptions, ): Promise { const response = await http<{ reference: Reference }>(requestOptions, { url: endpoint, @@ -28,7 +37,7 @@ export async function upload( data, headers: { 'content-type': 'application/octet-stream', - ...extractUploadHeaders(postageBatchId, options), + ...extractRedundantUploadHeaders(postageBatchId, options), }, }) @@ -44,10 +53,15 @@ export async function upload( * @param ky * @param hash Bee content reference */ -export async function download(requestOptions: BeeRequestOptions, hash: ReferenceOrEns): Promise { +export async function download( + requestOptions: BeeRequestOptions, + hash: ReferenceOrEns, + options?: DownloadRedundancyOptions, +): Promise { const response = await http(requestOptions, { responseType: 'arraybuffer', url: `${endpoint}/${hash}`, + headers: extractDownloadHeaders(options), }) return wrapBytesWithHelpers(new Uint8Array(response.data)) @@ -62,10 +76,12 @@ export async function download(requestOptions: BeeRequestOptions, hash: Referenc export async function downloadReadable( requestOptions: BeeRequestOptions, hash: ReferenceOrEns, + options?: DownloadRedundancyOptions, ): Promise> { const response = await http>(requestOptions, { responseType: 'stream', url: `${endpoint}/${hash}`, + headers: extractDownloadHeaders(options), }) return response.data diff --git a/src/modules/bzz.ts b/src/modules/bzz.ts index 87102cac..2cccffaa 100644 --- a/src/modules/bzz.ts +++ b/src/modules/bzz.ts @@ -4,17 +4,19 @@ import { Collection, CollectionUploadOptions, Data, + DownloadRedundancyOptions, FileData, FileUploadOptions, Readable, Reference, ReferenceOrEns, UploadHeaders, + UploadRedundancyOptions, UploadResult, } from '../types' import { wrapBytesWithHelpers } from '../utils/bytes' import { assertCollection } from '../utils/collection' -import { extractUploadHeaders, readFileHeaders } from '../utils/headers' +import { extractDownloadHeaders, extractRedundantUploadHeaders, readFileHeaders } from '../utils/headers' import { http } from '../utils/http' import { isReadable } from '../utils/stream' import { makeTar } from '../utils/tar' @@ -27,8 +29,11 @@ interface FileUploadHeaders extends UploadHeaders { 'content-type'?: string } -function extractFileUploadHeaders(postageBatchId: BatchId, options?: FileUploadOptions): FileUploadHeaders { - const headers: FileUploadHeaders = extractUploadHeaders(postageBatchId, options) +function extractFileUploadHeaders( + postageBatchId: BatchId, + options?: FileUploadOptions & UploadRedundancyOptions, +): FileUploadHeaders { + const headers: FileUploadHeaders = extractRedundantUploadHeaders(postageBatchId, options) if (options?.size) headers['content-length'] = String(options.size) @@ -51,10 +56,12 @@ export async function uploadFile( data: string | Uint8Array | Readable | ArrayBuffer, postageBatchId: BatchId, name?: string, - options?: FileUploadOptions, + options?: FileUploadOptions & UploadRedundancyOptions, ): Promise { if (isReadable(data) && !options?.contentType) { - if (!options) options = {} + if (!options) { + options = {} + } options.contentType = 'application/octet-stream' } @@ -86,11 +93,13 @@ export async function downloadFile( requestOptions: BeeRequestOptions, hash: ReferenceOrEns, path = '', + options?: DownloadRedundancyOptions, ): Promise> { const response = await http(requestOptions, { method: 'GET', responseType: 'arraybuffer', url: `${bzzEndpoint}/${hash}/${path}`, + headers: extractDownloadHeaders(options), }) const file = { ...readFileHeaders(response.headers as Record), @@ -111,11 +120,13 @@ export async function downloadFileReadable( requestOptions: BeeRequestOptions, hash: ReferenceOrEns, path = '', + options?: DownloadRedundancyOptions, ): Promise>> { const response = await http>(requestOptions, { method: 'GET', responseType: 'stream', url: `${bzzEndpoint}/${hash}/${path}`, + headers: extractDownloadHeaders(options), }) const file = { ...readFileHeaders(response.headers as Record), @@ -136,13 +147,17 @@ interface CollectionUploadHeaders extends UploadHeaders { function extractCollectionUploadHeaders( postageBatchId: BatchId, - options?: CollectionUploadOptions, -): CollectionUploadHeaders { - const headers: CollectionUploadHeaders = extractUploadHeaders(postageBatchId, options) + options?: CollectionUploadOptions & UploadRedundancyOptions, +): CollectionUploadHeaders & UploadRedundancyOptions { + const headers: CollectionUploadHeaders = extractRedundantUploadHeaders(postageBatchId, options) - if (options?.indexDocument) headers['swarm-index-document'] = options.indexDocument + if (options?.indexDocument) { + headers['swarm-index-document'] = options.indexDocument + } - if (options?.errorDocument) headers['swarm-error-document'] = options.errorDocument + if (options?.errorDocument) { + headers['swarm-error-document'] = options.errorDocument + } return headers } @@ -158,7 +173,7 @@ export async function uploadCollection( requestOptions: BeeRequestOptions, collection: Collection, postageBatchId: BatchId, - options?: CollectionUploadOptions, + options?: CollectionUploadOptions & UploadRedundancyOptions, ): Promise { assertCollection(collection) const tarData = makeTar(collection) diff --git a/src/types/index.ts b/src/types/index.ts index a6c8ae58..ecb04c1b 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -175,6 +175,54 @@ export interface UploadOptions { deferred?: boolean } +/** + * Add redundancy to the data being uploaded so that downloaders can download it with better UX. + * 0 value is default and does not add any redundancy to the file. + */ +export enum RedundancyLevel { + OFF = 0, + MEDIUM = 1, + STRONG = 2, + INSANE = 3, + PARANOID = 4, +} + +export interface UploadRedundancyOptions { + redundancyLevel?: RedundancyLevel +} + +/** + * Specify the retrieve strategy on redundant data. + * The possible values are NONE, DATA, PROX and RACE. + * Strategy NONE means no prefetching takes place. + * Strategy DATA means only data chunks are prefetched. + * Strategy PROX means only chunks that are close to the node are prefetched. + * Strategy RACE means all chunks are prefetched: n data chunks and k parity chunks. The first n chunks to arrive are used to reconstruct the file. + * Multiple strategies can be used in a fallback cascade if the swarm redundancy fallback mode is set to true. + * The default strategy is NONE, DATA, falling back to PROX, falling back to RACE + */ +export enum RedundancyStrategy { + NONE = 0, + DATA = 1, + PROX = 2, + RACE = 3, +} + +export interface DownloadRedundancyOptions { + /** + * Specify the retrieve strategy on redundant data. + */ + redundancyStrategy?: RedundancyStrategy + /** + * Specify if the retrieve strategies (chunk prefetching on redundant data) are used in a fallback cascade. The default is true. + */ + fallback?: boolean + /** + * Specify the timeout for chunk retrieval. The default is 30 seconds. + */ + timeoutMs?: number +} + export interface FileUploadOptions extends UploadOptions { /** * Specifies Content-Length for the given data. It is required when uploading with Readable. diff --git a/src/utils/expose.ts b/src/utils/expose.ts index 1fe16cb8..3d6b65cd 100644 --- a/src/utils/expose.ts +++ b/src/utils/expose.ts @@ -53,8 +53,10 @@ export { getDepthForCapacity, getStampCostInBzz, getStampCostInPlur, - getStampMaximumCapacityBytes, getStampEffectiveBytes, + getStampMaximumCapacityBytes, getStampTtlSeconds, getStampUsage, } from './stamps' + +export { approximateOverheadForRedundancyLevel, getRedundancyStat, getRedundancyStats } from './redundancy' diff --git a/src/utils/headers.ts b/src/utils/headers.ts index 8b7b8963..63b78549 100644 --- a/src/utils/headers.ts +++ b/src/utils/headers.ts @@ -1,4 +1,4 @@ -import { BatchId, FileHeaders, UploadOptions } from '../types' +import { BatchId, DownloadRedundancyOptions, FileHeaders, UploadOptions, UploadRedundancyOptions } from '../types' import { BeeError } from './error' /** @@ -53,13 +53,52 @@ export function extractUploadHeaders(postageBatchId: BatchId, options?: UploadOp 'swarm-postage-batch-id': postageBatchId, } - if (options?.pin) headers['swarm-pin'] = String(options.pin) + if (options?.pin) { + headers['swarm-pin'] = String(options.pin) + } + + if (options?.encrypt) { + headers['swarm-encrypt'] = String(options.encrypt) + } + + if (options?.tag) { + headers['swarm-tag'] = String(options.tag) + } + + if (typeof options?.deferred === 'boolean') { + headers['swarm-deferred-upload'] = options.deferred.toString() + } + + return headers +} - if (options?.encrypt) headers['swarm-encrypt'] = String(options.encrypt) +export function extractRedundantUploadHeaders( + postageBatchId: BatchId, + options?: UploadOptions & UploadRedundancyOptions, +): Record { + const headers = extractUploadHeaders(postageBatchId, options) - if (options?.tag) headers['swarm-tag'] = String(options.tag) + if (options?.redundancyLevel) { + headers['swarm-redundancy-level'] = String(options.redundancyLevel) + } + + return headers +} + +export function extractDownloadHeaders(options?: DownloadRedundancyOptions): Record { + const headers: Record = {} - if (typeof options?.deferred === 'boolean') headers['swarm-deferred-upload'] = options.deferred.toString() + if (options?.redundancyStrategy) { + headers['swarm-redundancy-strategy'] = String(options.redundancyStrategy) + } + + if (options?.fallback === false) { + headers['swarm-redundancy-fallback-mode'] = 'false' + } + + if (options?.timeoutMs !== undefined) { + headers['swarm-chunk-retrieval-timeout'] = String(options.timeoutMs) + } return headers } diff --git a/src/utils/redundancy.ts b/src/utils/redundancy.ts new file mode 100644 index 00000000..3ae07a00 --- /dev/null +++ b/src/utils/redundancy.ts @@ -0,0 +1,143 @@ +import { RedundancyLevel } from '..' + +const mediumTable = [ + [94, 68, 46, 28, 14, 5, 1], + [9, 8, 7, 6, 5, 4, 3], +] +const encMediumTable = [ + [47, 34, 23, 14, 7, 2], + [9, 8, 7, 6, 5, 4], +] +const strongTable = [ + [104, 95, 86, 77, 69, 61, 53, 46, 39, 32, 26, 20, 15, 10, 6, 3, 1], + [21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5], +] +const encStrongTable = [ + [52, 47, 43, 38, 34, 30, 26, 23, 19, 16, 13, 10, 7, 5, 3, 1], + [21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6], +] +const insaneTable = [ + [92, 87, 82, 77, 73, 68, 63, 59, 54, 50, 45, 41, 37, 33, 29, 26, 22, 19, 16, 13, 10, 8, 5, 3, 2, 1], + [31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6], +] +const encInsaneTable = [ + [46, 43, 41, 38, 36, 34, 31, 29, 27, 25, 22, 20, 18, 16, 14, 13, 11, 9, 8, 6, 5, 4, 2, 1], + [31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 7], +] +const paranoidTable = [ + [ + 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, + 8, 7, 6, 5, 4, 3, 2, 1, + ], + [ + 90, 88, 87, 85, 84, 82, 81, 79, 77, 76, 74, 72, 71, 69, 67, 66, 64, 62, 60, 59, 57, 55, 53, 51, 49, 48, 46, 44, 41, + 39, 37, 35, 32, 30, 27, 24, 20, + ], +] +const encParanoidTable = [ + [18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1], + [88, 85, 82, 79, 76, 72, 69, 66, 62, 59, 55, 51, 48, 44, 39, 35, 30, 24], +] + +const tables = { + [RedundancyLevel.MEDIUM]: [mediumTable, encMediumTable], + [RedundancyLevel.STRONG]: [strongTable, encStrongTable], + [RedundancyLevel.INSANE]: [insaneTable, encInsaneTable], + [RedundancyLevel.PARANOID]: [paranoidTable, encParanoidTable], +} + +/** + * Returns an approximate multiplier for the overhead of a given redundancy level. + * Redundancy level is a tradeoff between storage overhead and fault tolerance. + * Use this number to estimate the amount of chunks that will be stored for a given + * redundancy level. + */ +export function approximateOverheadForRedundancyLevel(chunks: number, level: RedundancyLevel, encrypted: boolean) { + const tableType = + level === RedundancyLevel.MEDIUM + ? tables[RedundancyLevel.MEDIUM] + : level === RedundancyLevel.STRONG + ? tables[RedundancyLevel.STRONG] + : level === RedundancyLevel.INSANE + ? tables[RedundancyLevel.INSANE] + : tables[RedundancyLevel.PARANOID] + const table = encrypted ? tableType[1] : tableType[0] + const [supportedChunks, parities] = table + for (let i = 0; i < supportedChunks.length; i++) { + if (chunks >= supportedChunks[i]) { + return parities[i] / supportedChunks[i] + } + } + return parities[parities.length - 1] / supportedChunks[supportedChunks.length - 1] +} + +interface RedundancyStats { + label: string + value: RedundancyLevel + errorTolerance: number +} + +const medium = { + label: 'medium', + value: RedundancyLevel.MEDIUM, + errorTolerance: 0.01, +} +const strong = { + label: 'strong', + value: RedundancyLevel.STRONG, + errorTolerance: 0.05, +} +const insane = { + label: 'insane', + value: RedundancyLevel.INSANE, + errorTolerance: 0.1, +} +const paranoid = { + label: 'paranoid', + value: RedundancyLevel.PARANOID, + errorTolerance: 0.5, +} + +export function getRedundancyStats(): { + medium: RedundancyStats + strong: RedundancyStats + insane: RedundancyStats + paranoid: RedundancyStats +} { + return { + medium, + strong, + insane, + paranoid, + } +} + +export function getRedundancyStat(level?: string | RedundancyLevel): RedundancyStats { + if (typeof level === 'string') { + switch (level.toLowerCase()) { + case 'medium': + return medium + case 'strong': + return strong + case 'insane': + return insane + case 'paranoid': + return paranoid + default: + throw new Error(`Unknown redundancy level '${level}'`) + } + } + + switch (level) { + case RedundancyLevel.MEDIUM: + return medium + case RedundancyLevel.STRONG: + return strong + case RedundancyLevel.INSANE: + return insane + case RedundancyLevel.PARANOID: + return paranoid + default: + throw new Error(`Unknown redundancy level '${level}'`) + } +}