added stream upload method

__mocks__/aws-sdk.js

@@ -22,6 +22,15 @@ module.exports =  {
     // putObject
     this.putObject = () => ({
       promise: () => Promise.resolve({ ETag: '"bafybeicaubxlzbr4sgc3tfwakfn7ganskxlgxmx25pdrcsojchgs3xpfqq"' }),
+      on: (event, cb) => {
+        if( event === 'complete') {
+          cb({
+            data: {
+              ETag: '"bafybeicaubxlzbr4sgc3tfwakfn7ganskxlgxmx25pdrcsojchgs3xpfqq"',
+            }
+          })
+        }
+      } 
     });
 
     //getObject

__mocks__/s3-streaming-upload.js

@@ -0,0 +1,12 @@
+class Uploader {
+  constructor() {}
+  send(cb) {
+    cb(undefined, {
+      ETag: '"bafybeicaubxlzbr4sgc3tfwakfn7ganskxlgxmx25pdrcsojchgs3xpfqq"',
+    });
+  }
+};
+
+module.exports =  {
+  Uploader,
+}

__tests__/streamUpload.js

@@ -0,0 +1,26 @@
+// make these available during the tests
+global.TextDecoder = require('util').TextDecoder
+global.TextEncoder = require('util').TextEncoder
+
+const streamUpload = require('../methods/streamUpload');
+
+it('uploads a file', async () => {
+  const stream = Buffer.from('Hello world'); 
+
+  const file = await streamUpload({
+    apiKey: '123',
+    apiSecret: 'abc',
+    stream,
+    key: 'my-file',
+  });
+
+  const expectedResult = {
+    'bucket': 'bucket-1',
+    'hash': 'bafybeicaubxlzbr4sgc3tfwakfn7ganskxlgxmx25pdrcsojchgs3xpfqq',
+    'hashV0': 'QmSgvgwxZGaBLqkGyWemEDqikCqU52XxsYLKtdy3vGZ8uq',
+    'key': 'my-file',
+    'publicUrl': 'https://storageapi.fleek.co/bucket-1/my-file'
+  };
+
+  expect(file).toEqual(expectedResult);
+});

index.d.ts

@@ -1,3 +1,21 @@
+export interface streamUploadInput {
+  apiKey: string,
+  apiSecret: string,
+  data: any,
+  key: string,
+  bucket?: string,
+}
+
+export interface streamUploadOutput {
+  hash: string,
+  hashV0: string,
+  key: string,
+  bucket: string,
+  publicUrl: string,
+}
+
+export type streamUploadType = (input: streamUploadInput) => Promise<streamUploadOutput>;
+
 export interface uploadInput {
   apiKey: string,
   apiSecret: string,
@@ -17,6 +35,7 @@ export interface uploadOutput {
 
 export type uploadType = (input: uploadInput) => Promise<uploadOutput>;
 
+
 export interface getFileFromHashInput {
   hash: string,
   getFileFromHashOptions?: string[],
@@ -95,6 +114,7 @@ export interface deleteFileOutput {
 
 declare module "@fleekhq/fleek-storage-js" {
   export const upload: uploadType;
+  export const streamUpload: streamUploadType;
   export const get: (input: getInput) => Promise<getOutput>;
   export const getFileFromHash: (input: getFileFromHashInput) => Promise<any>;
   export const listBuckets: (input: listBucketsInput) => Promise<listBucketsOutput[]>;

index.js

@@ -4,6 +4,7 @@ const getFileFromHash = require('./methods/getFileFromHash');
 const listBuckets = require('./methods/listBuckets');
 const listFiles = require('./methods/listFiles');
 const deleteFile = require('./methods/deleteFile');
+const streamUpload = require('./methods/streamUpload');
 
 //
 // ***************************************************** 
@@ -19,6 +20,8 @@ const deleteFile = require('./methods/deleteFile');
 // *****************************************************
 // deleteFile: Deletes a file from a bucket
 // *****************************************************
+// streamUpload: Uploads from a stream
+// *****************************************************
 //
 
 const fleekJs = {
@@ -28,6 +31,7 @@ const fleekJs = {
   listBuckets,
   listFiles,
   deleteFile,
+  streamUpload,
 };
 
 module.exports = fleekJs;

methods/streamUpload.js

@@ -0,0 +1,55 @@
+const initS3 = require('../utils/init-s3');
+const getBucket = require('../utils/get-bucket');
+const streamUploadFile = require('../utils/stream-upload');
+const getPublicUrl = require('../utils/get-public-url');
+
+const streamUpload = async ({
+  apiKey,
+  apiSecret,
+  stream,
+  key,
+  bucket,
+}) => {
+  try {
+    const s3 = initS3(
+      apiKey,
+      apiSecret,
+    );
+
+    if (!stream) {
+      throw 'No file to upload';
+    }
+
+    if (!key) {
+      throw 'No file key was specified'
+    }
+
+    let bucketName = bucket;
+
+    if (!bucketName) {
+      bucketName = await getBucket(s3);
+    }
+
+    const params = {
+      bucket: bucketName,
+      objectName: key,
+      stream,
+    };
+
+    const result = await streamUploadFile(s3, params);
+
+    const returnData = {
+      key,
+      bucket: bucketName,
+      publicUrl: getPublicUrl(bucketName, key),
+      hash: result.hash,
+      hashV0: result.hashV0,
+    };
+
+    return returnData;
+  } catch(error) {
+    throw error;
+  }
+};
+
+module.exports = streamUpload;

package.json

@@ -23,6 +23,7 @@
   "dependencies": {
     "aws-sdk": "^2.989.0",
     "axios": "^0.21.1",
-    "multiformats": "^9.4.7"
+    "multiformats": "^9.4.7",
+    "s3-streaming-upload": "^0.3.4"
   }
 }

utils/stream-upload.js

@@ -0,0 +1,38 @@
+const { CID } = require('multiformats');
+const Uploader = require('s3-streaming-upload').Uploader;
+
+const streamUploadFile = (s3, params) => (new Promise((resolve, reject) => {
+  const request = new Uploader({
+    service: s3,
+    objectParams: {
+      ACL: 'public-read',
+    },
+    ...params,
+  });
+
+  request.send((err, data) => {
+    if (err) {
+      reject(err);
+    }
+    const { ETag } = data;
+
+    const hash = ETag.replace(/^"|"$/g, '');
+
+    const cidObj = CID.parse(hash);
+
+    let cidv0;
+
+    const cidv1 = cidObj.toV1().toString();
+
+    try {
+      cidv0 = cidObj.toV0().toString();
+    } catch (e) {
+      // fallback when cbor is used
+      cidv0 = cidv1;
+    }
+
+    resolve ({ hash: cidv1, hashV0: cidv0 });
+  });
+}));
+
+module.exports = streamUploadFile;

yarn.lock

@@ -703,7 +703,7 @@ atob@^2.1.2:
   resolved "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz"
   integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==
 
-aws-sdk@^2.989.0:
+aws-sdk@^2.814.0, aws-sdk@^2.989.0:
   version "2.989.0"
   resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.989.0.tgz#ed3cce6b94856b469784bc3312a0b64438b9fe67"
   integrity sha512-sMjvqeF9mEOxXkhOAUjCrBt2iYafclkmaIbgSdjJ+te7zKXeReqrc6P3VgIGUxU8kwmdSro0n1NjrXbzKQJhcw==
@@ -1821,7 +1821,7 @@ inflight@^1.0.4:
     once "^1.3.0"
     wrappy "1"
 
-inherits@2:
+inherits@2, inherits@^2.0.3:
   version "2.0.4"
   resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz"
   integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
@@ -3093,6 +3093,15 @@ read-pkg@^5.2.0:
     parse-json "^5.0.0"
     type-fest "^0.6.0"
 
+readable-stream@^3.3.0:
+  version "3.6.0"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
+  integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
+  dependencies:
+    inherits "^2.0.3"
+    string_decoder "^1.1.1"
+    util-deprecate "^1.0.1"
+
 regex-not@^1.0.0, regex-not@^1.0.2:
   version "1.0.2"
   resolved "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz"
@@ -3246,11 +3255,24 @@ rxjs@^6.5.3:
   dependencies:
     tslib "^1.9.0"
 
+s3-streaming-upload@^0.3.4:
+  version "0.3.4"
+  resolved "https://registry.yarnpkg.com/s3-streaming-upload/-/s3-streaming-upload-0.3.4.tgz#e505a2375baf0eb472e181f63fecba942929e32b"
+  integrity sha512-mm/OPbV5rUpjGtRFMhFBH158gjguwVgM4/81UcxDGF8jU1uhopjvsQJRt0hvmLqYBqNR6LsP9fdb0ZHkJrwztw==
+  dependencies:
+    aws-sdk "^2.814.0"
+    readable-stream "^3.3.0"
+
 safe-buffer@^5.0.1, safe-buffer@^5.1.2, safe-buffer@~5.1.1:
   version "5.1.2"
   resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz"
   integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
 
+safe-buffer@~5.2.0:
+  version "5.2.1"
+  resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
+  integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
+
 safe-regex@^1.1.0:
   version "1.1.0"
   resolved "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz"
@@ -3541,6 +3563,13 @@ string-width@^4.1.0, string-width@^4.2.0:
     is-fullwidth-code-point "^3.0.0"
     strip-ansi "^6.0.0"
 
+string_decoder@^1.1.1:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
+  integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==
+  dependencies:
+    safe-buffer "~5.2.0"
+
 strip-ansi@^5.1.0:
   version "5.2.0"
   resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz"
@@ -3818,6 +3847,11 @@ use@^3.1.0:
   resolved "https://registry.npmjs.org/use/-/use-3.1.1.tgz"
   integrity sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==
 
+util-deprecate@^1.0.1:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
+  integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=
+
 [email protected], uuid@^3.3.2:
   version "3.3.2"
   resolved "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz"