Merge pull request #39 from filedag-project/feat/milestone4

Feat/milestone4

.gitignore

@@ -18,7 +18,12 @@
 /dagpool
 /datanode
 /objectstore
-/dn-data
 /dp-data
+/dn-data
 /store-data
-/iam-tools
+/iam-tools
+
+# web
+/web/coverage
+/web/build
+/web/node_modules

Makefile

@@ -13,6 +13,8 @@ TAG_OBJECTSTORE ?= "filedag/objectstore:latest"
 
 build: clean dagpool datanode objectstore iamtools
 
+build-linux: clean dagpool-linux datanode-linux objectstore-linux iamtools-linux
+
 dagpool:
 	go build -ldflags "-s -w" -o ${DAGPOOL_TARGET} ./cmd/dagpool
 
@@ -25,6 +27,18 @@ objectstore:
 iamtools:
 	go build -ldflags "-s -w" -o ${IAMTOOLS_TARGET} ./cmd/tools/iam-tools
 
+dagpool-linux:
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w" -o ${DAGPOOL_TARGET} ./cmd/dagpool
+
+datanode-linux:
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w" -o ${DATANODE_TARGET} ./cmd/datanode
+
+objectstore-linux:
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w" -o ${OBJECTSTORE_TARGET} ./cmd/objectstore
+
+iamtools-linux:
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-s -w" -o ${IAMTOOLS_TARGET} ./cmd/tools/iam-tools
+
 docker-datanode:
 	docker build -q --no-cache -t $(TAG_DATANODE) . -f Dockerfile.datanode
 
@@ -45,4 +59,4 @@ clean:
 	-rm -f ${IAMTOOLS_TARGET}
 
 proto:
-	protoc --go_out=./dag/proto --go-grpc_out=./dag/proto dag/proto/*.proto --proto_path=./dag/proto
+	protoc --go_out=./dag/proto --go-grpc_out=./dag/proto dag/proto/*.proto --proto_path=./dag/proto

README-cn.md

@@ -42,14 +42,14 @@ FileDAG Storage 的开发将为上述问题提供一种解决方案
 #### Description:
       
 - 单DAG Node模式的开发:
-    - [x] 支持block store API
-    - [x] 为DAG Pool提供基础存储服务
+  - ✅ 支持block store API
+  - ✅ 为DAG Pool提供基础存储服务
 - DAG Pool:
-    - [x] 多用户访问
-    - [x] 身份验证机制
+  - ✅ 多用户访问
+  - ✅ 身份验证机制
 - 对象存储:
-    - [x] 实现用户、区域、桶、对象等基本数据结构
-    - [x] 实现用户认证的API
+  - ✅ 实现用户、区域、桶、对象等基本数据结构
+  - ✅ 实现用户认证的API
 
 ### Millstone 2
 
@@ -60,13 +60,13 @@ FileDAG Storage 的开发将为上述问题提供一种解决方案
 #### Description:
 
 - DAG Pool:
-    - [x] 数据块的引用记录
-    - [x] 数据pin策略
-    - [x] DAG Pool的可中断垃圾回收机制
+  - ✅ 数据块的引用记录
+  - ✅ 数据pin策略
+  - ✅ DAG Pool的可中断垃圾回收机制
 - 对象存储:
-    - [x] 桶相关操作的API
-    - [x] 对象操作的API
-    - [x] 权限操作API
+  - ✅ 桶相关操作的API
+  - ✅ 对象操作的API
+  - ✅ 权限操作API
 
 
 ### Milestone 3
@@ -78,12 +78,12 @@ FileDAG Storage 的开发将为上述问题提供一种解决方案
 #### Description:
 
 - DAG Node:
-    - [x] 基于Reed-Solomon Erasure Code开发数据容错技术
+  - ✅ 基于Reed-Solomon Erasure Code开发数据容错技术
 - DAG Pool:
-    - [x] 基于Redis Hash Slots，组织多个DAG Node构建存储集群
-    - [x] 提供存储节点的运行状况报告和全局一致性状态
-    - [x] 支持存储节点动态扩容
-    - [x] 支持存储节点动态扩展
+  - ✅ 基于Redis Hash Slots，组织多个DAG Node构建存储集群
+  - ✅ 提供存储节点的运行状况报告和全局一致性状态
+  - ✅ 支持存储节点动态扩容
+  - ✅ 支持存储节点动态扩展
 
 ### Milestone 4
 
@@ -93,13 +93,13 @@ FileDAG Storage 的开发将为上述问题提供一种解决方案
 
 #### Description:
 
-- 存储池统计概况面板
-- 用户界面
+- ✅ 存储池统计概况面板
+- ✅ 用户界面
 - 对象存储接口:
-    - 用户
-    - 用户访问操作
-    - 桶操作
-    - 权限设置
+  - ✅ 用户
+  - ✅ 用户访问操作
+  - ✅ 桶操作
+  - ✅ 权限设置
 
 ### Milestone 5
 

README.md

@@ -41,14 +41,14 @@ Build fundamental data structure, and the overall architecture of this project.
 #### Description:
 
 - Development of single DAG Node:
-    - [x] supports API of the block store
-    - [x] providers basic storage service for DAG Pool
+  - ✅ supports API of the block store
+  - ✅ providers basic storage service for DAG Pool
 - DAG Pool:
-    - [x] multi-user access
-    - [x] authentication mechanism
+  - ✅ multi-user access
+  - ✅ authentication mechanism
 - Object store:
-    - [x] implements basic data structure, such as user, region, bucket, and object
-    - [x] implements API of user authentication
+  - ✅ implements basic data structure, such as user, region, bucket, and object
+  - ✅ implements API of user authentication
 
 ### Millstone 2
 
@@ -59,13 +59,13 @@ Implement data management.
 #### Description:
 
 - DAG Pool:
-    - [x] reference records of data blocks
-    - [x] strategy of data pin
-    - [x] interruptible garbage collection mechanism of DAG Pool
+  - ✅ reference records of data blocks
+  - ✅ strategy of data pin
+  - ✅ interruptible garbage collection mechanism of DAG Pool
 - Object store:
-    - [x] API of bucket related operations
-    - [x] API of object manipulation
-    - [x] API of permission operation
+  - ✅ API of bucket related operations
+  - ✅ API of object manipulation
+  - ✅ API of permission operation
 
 
 ### Milestone 3
@@ -77,12 +77,12 @@ Realize clustered DAG Node and development of data fault tolerance.
 #### Description:
 
 - DAG Node:
-    - [x] develops data fault tolerance based on Reed-Solomon Erasure Code
+  - ✅ develops data fault tolerance based on Reed-Solomon Erasure Code
 - DAG Pool:
-    - [x] organizes multiple DAG Nodes to build a storage cluster based on Redis Hash Slots
-    - [x] provides health report of storage nodes and status of global consistency
-    - [x] supports dynamic expansion of storage nodes
-    - [x] supports dynamic scaling of storage nodes
+  - ✅ organizes multiple DAG Nodes to build a storage cluster based on Redis Hash Slots
+  - ✅ provides health report of storage nodes and status of global consistency
+  - ✅ supports dynamic expansion of storage nodes
+  - ✅ supports dynamic scaling of storage nodes
 
 ### Milestone 4
 
@@ -92,13 +92,13 @@ Develop the Control Panel of FileDAG Storage.
 
 #### Description:
 
-- Dashboard of storage pool statistics overview
-- User interface
+- ✅ Dashboard of storage pool statistics overview
+- ✅ User interface
 - Object Store interface:
-    - users
-    - user access operations
-    - bucket operations
-    - permission setting
+  - ✅ users
+  - ✅ user access operations
+  - ✅ bucket operations
+  - ✅ permission setting
 
 ### Milestone 5
 

cmd/objectstore/server.go

@@ -6,12 +6,13 @@ import (
 	"fmt"
 	dagpoolcli "github.com/filedag-project/filedag-storage/dag/pool/client"
 	"github.com/filedag-project/filedag-storage/objectservice/iam"
-	"github.com/filedag-project/filedag-storage/objectservice/iam/auth"
 	"github.com/filedag-project/filedag-storage/objectservice/iamapi"
+	"github.com/filedag-project/filedag-storage/objectservice/objmetadb"
+	"github.com/filedag-project/filedag-storage/objectservice/pkg/auth"
 	"github.com/filedag-project/filedag-storage/objectservice/s3api"
 	"github.com/filedag-project/filedag-storage/objectservice/store"
-	"github.com/filedag-project/filedag-storage/objectservice/uleveldb"
 	"github.com/filedag-project/filedag-storage/objectservice/utils"
+	"github.com/filedag-project/filedag-storage/objectservice/utils/httpstats"
 	"github.com/gorilla/mux"
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/ipfs/go-merkledag"
@@ -55,7 +56,7 @@ func startServer(cctx *cli.Context) {
 			"Root user length should be at least 3, and password length at least 8 characters")
 	}
 
-	db, err := uleveldb.OpenDb(datadir)
+	db, err := objmetadb.OpenDb(datadir)
 	if err != nil {
 		return
 	}
@@ -85,18 +86,43 @@ func startServer(cctx *cli.Context) {
 				log.Errorf("CleanObjectsInBucket error: %v", err)
 				continue
 			}
-			if err = bmSys.DeleteBucket(ctx, bkt.Name); err != nil {
+			if err = bmSys.DeleteBucket(ctx, bkt.Name, accessKey); err != nil {
 				log.Errorf("DeleteBucket error: %v", err)
 			}
 		}
 	}
+	bucketInfoFunc := func(ctx context.Context, accessKey string) []store.BucketInfo {
+		var bucketInfos []store.BucketInfo
+		bkts, err := bmSys.GetAllBucketsOfUser(ctx, accessKey)
+		if err != nil {
+			log.Errorf("GetAllBucketsOfUser error: %v", err)
+			return bucketInfos
+		}
+		for _, bkt := range bkts {
+			info, err := storageSys.GetAllObjectsInBucketInfo(ctx, bkt.Name)
+			if err != nil {
+				return nil
+			}
+			bucketInfos = append(bucketInfos, info)
+		}
+		return bucketInfos
+	}
+	storePoolStatsFunc := func(ctx context.Context) (store.DataUsageInfo, error) {
+		bkts, err := bmSys.GetAllBucketInfo(ctx)
+		if err != nil {
+			log.Errorf("GetAllBucketsOfUser error: %v", err)
+			return store.DataUsageInfo{}, nil
+		}
+		return storageSys.StoreStats(ctx, bkts.Bucket)
+	}
 	handler := s3api.CorsHandler(router)
-	s3api.NewS3Server(router, authSys, bmSys, storageSys)
-	iamapi.NewIamApiServer(router, authSys, cleanData)
-
+	httpStatsSys := httpstats.NewHttpStatsSys(db)
+	iamapi.NewIamApiServer(router, authSys, httpStatsSys, cleanData, bucketInfoFunc, storePoolStatsFunc)
+	s3api.NewS3Server(router, authSys, bmSys, storageSys, httpStatsSys)
+	go httpStatsSys.StoreApiLog(cctx.Context)
 	if strings.HasPrefix(listen, ":") {
 		for _, ip := range utils.MustGetLocalIP4().ToSlice() {
-			log.Infof("start sever at http://%v%v", ip, listen)
+			log.Infof("start server at http://%v%v", ip, listen)
 		}
 	} else {
 		log.Infof("start sever at http://%v", listen)

cmd/tools/iam-tools/iam_tools.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"fmt"
-	"github.com/filedag-project/filedag-storage/objectservice/iam/auth"
+	"github.com/filedag-project/filedag-storage/objectservice/pkg/auth"
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
@@ -74,6 +74,10 @@ var addUserCmd = &cli.Command{
 			Name:  "password",
 			Usage: "the password that you want to add",
 		},
+		&cli.StringFlag{
+			Name:  "capacity",
+			Usage: "the capacity of  the user you want to add",
+		},
 	},
 	Action: func(cctx *cli.Context) error {
 		apiAddr := cctx.String(ServerApi)
@@ -97,8 +101,9 @@ var addUserCmd = &cli.Command{
 		if !auth.IsSecretKeyValid(password) {
 			return errInvalidSecretKeyLength
 		}
+		capacity := cctx.String("capacity")
 
-		req, err := mustNewSignedV4Request(http.MethodPost, apiAddr+addUserUrl+"?accessKey="+username+"&secretKey="+password,
+		req, err := mustNewSignedV4Request(http.MethodPost, apiAddr+addUserUrl+"?accessKey="+username+"&secretKey="+password+"&capacity="+capacity,
 			0, nil, "s3", accessKey, secretKey)
 		if err != nil {
 			log.Errorf("mustNewSignedV4Request err: %v", err)

cmd/tools/iam-tools/request.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"github.com/filedag-project/filedag-storage/objectservice/iam/auth"
+	"github.com/filedag-project/filedag-storage/objectservice/pkg/auth"
 	"github.com/filedag-project/filedag-storage/objectservice/utils"
 	"io"
 	"net/http"

conf/node_config4.json

@@ -0,0 +1,10 @@
+{
+  "name": "dag_node4",
+  "nodes": [
+    "127.0.0.1:9020",
+    "127.0.0.1:9021",
+    "127.0.0.1:9022"
+  ],
+  "data_blocks": 2,
+  "parity_blocks": 1
+}

dag/config/config.go

@@ -5,7 +5,7 @@ import (
 	"time"
 )
 
-//PoolConfig is the configuration for the dag pool
+// PoolConfig is the configuration for the dag pool
 type PoolConfig struct {
 	Listen       string        `json:"listen"`
 	LeveldbPath  string        `json:"leveldb_path"`
@@ -14,13 +14,13 @@ type PoolConfig struct {
 	GcPeriod     time.Duration `json:"gc_period"`
 }
 
-//ClusterConfig is the configuration for a cluster
+// ClusterConfig is the configuration for a cluster
 type ClusterConfig struct {
 	Version int           `json:"version"`
 	Cluster []DagNodeInfo `json:"cluster"`
 }
 
-//DagNodeConfig is the configuration for a dag node
+// DagNodeConfig is the configuration for a dag node
 type DagNodeConfig struct {
 	Name         string   `json:"name"`
 	Nodes        []string `json:"nodes"`         // rpc address list of datanodes