Improve block storage to use AWS S3

bosagora · Sep 30, 2024 · f1eea3b · f1eea3b
1 parent b359725
commit f1eea3b
Show file tree

Hide file tree

Showing 14 changed files with 1,257 additions and 26 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -60,4 +60,6 @@ jobs:
       - name: Tests of Server
         env:
           REPORT_GAS: true
+          NODE_S3_ACCESS_KEY: ${{ secrets.NODE_S3_ACCESS_KEY }}
+          NODE_S3_SECRET_KEY: ${{ secrets.NODE_S3_SECRET_KEY }}
         run: cd packages/server && yarn run test
diff --git a/packages/server/config/config.example.yaml b/packages/server/config/config.example.yaml
@@ -31,9 +31,14 @@ node:
   interval: 600
   max_txs: 256
   send_interval: 30
+  storage_type: "${NODE_STORAGE_TYPE}"
   ipfs_api_url: https://api-ipfs.bosagora.info
   ipfs_gateway_url: https://ipfs.bosagora.info
   ipfs_test: false
+  s3_region: "${NODE_S3_REGION}"
+  s3_access_key: "${NODE_S3_ACCESS_KEY}"
+  s3_secret_key: "${NODE_S3_SECRET_KEY}"
+  s3_bucket: "${NODE_S3_BUCKET}"
 
 scheduler:
   enable: true

diff --git a/packages/server/config/config.yaml b/packages/server/config/config.yaml
@@ -31,9 +31,14 @@ node:
   interval: ${NODE_INTERVAL}
   max_txs: ${NODE_MAX_TXS}
   send_interval: ${NODE_SEND_INTERVAL}
+  storage_type: "${NODE_STORAGE_TYPE}"
   ipfs_api_url: "${NODE_IPFS_API_URL}"
   ipfs_gateway_url: "${NODE_IPFS_GATEWAY_URL}"
   ipfs_test: ${NODE_IPFS_TEST}
+  s3_region: "${NODE_S3_REGION}"
+  s3_access_key: "${NODE_S3_ACCESS_KEY}"
+  s3_secret_key: "${NODE_S3_SECRET_KEY}"
+  s3_bucket: "${NODE_S3_BUCKET}"
 
 scheduler:
   enable: true

diff --git a/packages/server/config/config_test.yaml b/packages/server/config/config_test.yaml
@@ -31,9 +31,14 @@ node:
   interval: 5
   max_txs: 8
   send_interval: 1
+  storage_type: "${NODE_STORAGE_TYPE}"
   ipfs_api_url: http://localhost:5001
   ipfs_gateway_url: http://localhost:8080
   ipfs_test: true
+  s3_region: "${NODE_S3_REGION}"
+  s3_access_key: "${NODE_S3_ACCESS_KEY}"
+  s3_secret_key: "${NODE_S3_SECRET_KEY}"
+  s3_bucket: "${NODE_S3_BUCKET}"
 
 scheduler:
   enable: true

diff --git a/packages/server/env/.env.sample b/packages/server/env/.env.sample
@@ -32,10 +32,17 @@ PUBLISHER_KEY=0x8cb4f445f174e010b3a178bcc6d84a16ffa69c02a52e942e69df578fa1aa4610
 NODE_INTERVAL=60
 NODE_MAX_TXS=256
 NODE_SEND_INTERVAL=30
+
+NODE_STORAGE_TYPE=s3
 NODE_IPFS_API_URL=http://localhost:5001
 NODE_IPFS_GATEWAY_URL=http://localhost:8080
 NODE_IPFS_TEST=false
 
+NODE_S3_REGION=
+NODE_S3_ACCESS_KEY=
+NODE_S3_SECRET_KEY=
+NODE_S3_BUCKET=
+
 ACCESS_KEY0=0x2c93e943c0d7f6f1a42f53e116c52c40fe5c1b428506dc04b290f2a77580a342
 SENDER0=0x4501F7aF010Cef3DcEaAfbc7Bfb2B39dE57df54d
 WAITING0=86400

diff --git a/packages/server/package.json b/packages/server/package.json
@@ -66,6 +66,7 @@
         "@types/urijs": "^1.19.12"
     },
     "dependencies": {
+        "@aws-sdk/client-s3": "^3.658.1",
         "acc-save-purchase-contracts": "~1.0.2",
         "acc-save-purchase-sdk": "~2.2.0",
         "argparse": "^2.0.1",

diff --git a/packages/server/src/modules/index.ts b/packages/server/src/modules/index.ts
@@ -2,4 +2,4 @@ export { NetworkError } from "./network/ErrorTypes";
 export { Scheduler } from "./scheduler/Scheduler";
 export { WebService } from "./service/WebService";
 
-export { IPFSManager } from "./network/IPFSManager";
+export { IPFSManager } from "../service/network/IPFSManager";
diff --git a/packages/server/src/service/common/Config.ts b/packages/server/src/service/common/Config.ts
@@ -274,10 +274,15 @@ export class NodeConfig implements INodeConfig {
     public interval: number;
     public max_txs: number;
     public send_interval: number;
+    public storage_type: string;
     public ipfs_api_url: string;
     public ipfs_gateway_url: string;
     public ipfs_test: boolean;
 
+    public s3_region: string;
+    public s3_access_key: string;
+    public s3_secret_key: string;
+    public s3_bucket: string;
     /**
      * Constructor
      */
@@ -287,9 +292,15 @@ export class NodeConfig implements INodeConfig {
         this.interval = defaults.interval;
         this.max_txs = defaults.max_txs;
         this.send_interval = defaults.send_interval;
+        this.storage_type = defaults.storage_type.toLowerCase();
         this.ipfs_api_url = defaults.ipfs_api_url;
         this.ipfs_gateway_url = defaults.ipfs_gateway_url;
         this.ipfs_test = defaults.ipfs_test;
+
+        this.s3_region = defaults.s3_region;
+        this.s3_access_key = defaults.s3_access_key;
+        this.s3_secret_key = defaults.s3_secret_key;
+        this.s3_bucket = defaults.s3_bucket;
     }
 
     /**
@@ -300,9 +311,14 @@ export class NodeConfig implements INodeConfig {
             interval: 600,
             max_txs: 128,
             send_interval: 14,
+            storage_type: "s3",
             ipfs_api_url: "https://api-ipfs.bosagora.info",
             ipfs_gateway_url: "https://ipfs.bosagora.info",
             ipfs_test: true,
+            s3_region: "",
+            s3_access_key: "",
+            s3_secret_key: "",
+            s3_bucket: "",
         };
     }
 
@@ -314,9 +330,17 @@ export class NodeConfig implements INodeConfig {
         if (config.interval !== undefined) this.interval = Number(config.interval);
         if (config.max_txs !== undefined) this.max_txs = Number(config.max_txs);
         if (config.send_interval !== undefined) this.send_interval = Number(config.send_interval);
+
+        if (config.storage_type !== undefined) this.storage_type = config.storage_type.toLowerCase();
+
         if (config.ipfs_api_url !== undefined) this.ipfs_api_url = config.ipfs_api_url;
         if (config.ipfs_gateway_url !== undefined) this.ipfs_gateway_url = config.ipfs_gateway_url;
         if (config.ipfs_test !== undefined) this.ipfs_test = config.ipfs_test.toString().toLowerCase() === "true";
+
+        if (config.s3_region !== undefined) this.s3_region = config.s3_region;
+        if (config.s3_access_key !== undefined) this.s3_access_key = config.s3_access_key;
+        if (config.s3_secret_key !== undefined) this.s3_secret_key = config.s3_secret_key;
+        if (config.s3_bucket !== undefined) this.s3_bucket = config.s3_bucket;
     }
 }
 
@@ -349,9 +373,14 @@ export interface INodeConfig {
     interval: number;
     max_txs: number;
     send_interval: number;
+    storage_type: string;
     ipfs_api_url: string;
     ipfs_gateway_url: string;
     ipfs_test: boolean;
+    s3_region: string;
+    s3_access_key: string;
+    s3_secret_key: string;
+    s3_bucket: string;
 }
 
 /**

diff --git a/...server/src/modules/network/IPFSManager.ts → ...server/src/service/network/IPFSManager.ts b/...server/src/modules/network/IPFSManager.ts → ...server/src/service/network/IPFSManager.ts
@@ -1,12 +1,5 @@
-/**
- *  Includes classes that store data in IPFS
- *
- *  Copyright:
- *      Copyright (c) 2024 BOSAGORA Foundation All rights reserved.
- *
- *  License:
- *       MIT License. See LICENSE for details.
- */
+import { Config } from "../common/Config";
+import { IStorageManager } from "./IStorageManager";
 
 // tslint:disable-next-line:no-var-requires
 const IPFS = require("ipfs-mini");
@@ -20,16 +13,17 @@ const bs58 = require("bs58");
 /**
  * Store data in IPFS.
  */
-export class IPFSManager {
+export class IPFSManager implements IStorageManager {
     private ipfs: any;
     private test: boolean;
+    private config: Config;
 
     /**
      * Constructor
-     * @param api_url URL of the API for IPFS
      */
-    constructor(api_url: string) {
-        const uri = URI(api_url);
+    constructor(config: Config) {
+        this.config = config;
+        const uri = URI(config.node.ipfs_api_url);
         this.ipfs = new IPFS({ host: uri.hostname(), port: uri.port(), protocol: uri.protocol() });
         this.test = false;
     }
@@ -45,8 +39,9 @@ export class IPFSManager {
     /**
      * Store data in IPFS.
      * @param data Data to be stored.
+     * @param cid
      */
-    public add(data: string | Buffer): Promise<string> {
+    public add(data: string | Buffer, cid: string): Promise<string> {
         if (this.test) {
             return new Promise<string>((resolve, reject) => {
                 crypto.randomBytes(32, (err, buf) => {

diff --git a/packages/server/src/service/network/IStorageManager.ts b/packages/server/src/service/network/IStorageManager.ts
@@ -0,0 +1,4 @@
+export interface IStorageManager {
+    setTest(value: boolean): any;
+    add(data: string | Buffer, cid: string): Promise<string>;
+}
diff --git a/packages/server/src/service/network/S3Manager.ts b/packages/server/src/service/network/S3Manager.ts
@@ -0,0 +1,61 @@
+import { Config } from "../common/Config";
+import { IStorageManager } from "./IStorageManager";
+
+import { PutObjectCommand, S3Client } from "@aws-sdk/client-s3";
+
+/**
+ * Store data in IPFS.
+ */
+export class S3Manager implements IStorageManager {
+    private test: boolean;
+    private config: Config;
+    private s3_client: S3Client;
+
+    /**
+     * Constructor
+     */
+    constructor(config: Config) {
+        this.config = config;
+        this.s3_client = new S3Client({
+            region: this.config.node.s3_region,
+            credentials: {
+                accessKeyId: this.config.node.s3_access_key,
+                secretAccessKey: this.config.node.s3_secret_key,
+            },
+        });
+        this.test = false;
+    }
+
+    /**
+     * Specifies whether to use it for testing.
+     * @param value
+     */
+    public setTest(value: boolean) {
+        this.test = value;
+    }
+
+    /**
+     * Store data in IPFS.
+     * @param data Data to be stored.
+     * @param cid
+     */
+    public add(data: string | Buffer, cid: string): Promise<string> {
+        return new Promise<string>((resolve, reject) => {
+            const params = {
+                Body: data,
+                Bucket: this.config.node.s3_bucket,
+                Key: cid,
+            };
+            const command = new PutObjectCommand(params);
+            this.s3_client
+                .send(command)
+                .then(() => {
+                    return resolve(cid);
+                })
+                .catch((reason) => {
+                    console.error(reason);
+                    return reject(new Error(reason));
+                });
+        });
+    }
+}
diff --git a/packages/server/src/service/scheduler/Node.ts b/packages/server/src/service/scheduler/Node.ts
@@ -19,6 +19,8 @@ import { TransactionPool } from "./TransactionPool";
 import { Block, Hash, hashFull, Transaction, Utils } from "acc-save-purchase-sdk";
 import { ethers } from "hardhat";
 import { Metrics } from "../metrics/Metrics";
+import { IStorageManager } from "../network/IStorageManager";
+import { S3Manager } from "../network/S3Manager";
 
 /**
  * Definition of event type
@@ -41,7 +43,7 @@ export class Node extends Scheduler {
     /**
      * The object needed to store data in IPFS
      */
-    private _ipfs: IPFSManager | undefined;
+    private _blockStorage: IStorageManager | undefined;
 
     /**
      * The object needed to access the database
@@ -107,13 +109,13 @@ export class Node extends Scheduler {
     }
 
     /**
-     * Returns the value if this._ipfs is defined.
+     * Returns the value if this._blockStorage is defined.
      * Otherwise, exit the process.
      */
-    private get ipfs(): IPFSManager {
-        if (this._ipfs !== undefined) return this._ipfs;
+    private get blockStorage(): IStorageManager {
+        if (this._blockStorage !== undefined) return this._blockStorage;
         else {
-            logger.error("IPFSManager is not ready yet.");
+            logger.error("blockStorageManager is not ready yet.");
             process.exit(1);
         }
     }
@@ -158,8 +160,12 @@ export class Node extends Scheduler {
             }
         }
         if (this._config !== undefined) {
-            this._ipfs = new IPFSManager(this._config.node.ipfs_api_url);
-            this._ipfs.setTest(this._config.node.ipfs_test);
+            if (this._config.node.storage_type === "ipfs") {
+                this._blockStorage = new IPFSManager(this._config);
+            } else {
+                this._blockStorage = new S3Manager(this._config);
+            }
+            this._blockStorage.setTest(this._config.node.ipfs_test);
         }
     }
 
@@ -236,8 +242,8 @@ export class Node extends Scheduler {
                     let success: boolean = true;
 
                     try {
-                        // Save block to IPFS
-                        cid = await this.ipfs.add(JSON.stringify(block));
+                        // Save block
+                        cid = await this.blockStorage.add(JSON.stringify(block), hashFull(block).toString());
                         logger.info(`Saved block to IPFS - height: ${block.header.height.toString()}, CID: ${cid}`);
                     } catch {
                         success = false;

diff --git a/packages/server/test/1_blockchain/103_IPFSManager.test.ts b/packages/server/test/1_blockchain/103_IPFSManager.test.ts
@@ -11,13 +11,18 @@
 import { IPFSManager } from "../../src/modules";
 
 import * as assert from "assert";
+import { Config } from "../../src/service/common/Config";
+
+import path from "path";
 
 describe("Test of IPFSManager", () => {
-    const ipfs = new IPFSManager("http://localhost:5001");
+    const config = new Config();
+    config.readFromFile(path.resolve(process.cwd(), "config/config_test.yaml"));
+    const ipfs = new IPFSManager(config);
     ipfs.setTest(true);
 
     it("Add Contents", async () => {
-        const res = await ipfs.add("hello world!");
+        const res = await ipfs.add("hello world!", "");
         assert.ok(res !== null);
     });
 });