From e3fd931f400fc185e8f502cc127a90d148824414 Mon Sep 17 00:00:00 2001 From: Janison Sivarajah Date: Fri, 27 Nov 2020 13:35:50 -0500 Subject: [PATCH] Remove mongo: point daemon to use fork of textile (#266) * Point to forked textile: init * Optional-mongo: cleanup makefile, point to latest version fo buckets fork * Optional-mongo: clean up remaining mongo refs * Optional-mongo: update readme --- .github/workflows/release.yml | 4 ---- .goreleaser.yml | 18 +++--------------- Makefile | 8 ++------ README.md | 10 +--------- cmd/space-daemon/main.go | 8 -------- config/config.go | 8 -------- config/map_config.go | 8 -------- core/env/env.go | 4 ---- core/textile/buckd.go | 31 ++++++++----------------------- go.mod | 1 + go.sum | 7 ++++++- 11 files changed, 21 insertions(+), 86 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b33a69f2..125acce7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,10 +51,6 @@ jobs: args: release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - MONGO_USR: ${{ secrets.MONGO_USR }} - MONGO_PW: ${{ secrets.MONGO_PW }} - MONGO_HOST: ${{ secrets.MONGO_HOST }} - MONGO_REPLICA_SET: ${{ secrets.MONGO_REPLICA_SET }} SERVICES_API_URL: ${{ secrets[steps.secretnames.outputs.SERVICES_API_URL] }} VAULT_API_URL: ${{ secrets[steps.secretnames.outputs.VAULT_API_URL] }} VAULT_SALT_SECRET: ${{ secrets[steps.secretnames.outputs.VAULT_SALT_SECRET] }} diff --git a/.goreleaser.yml b/.goreleaser.yml index 738b95e5..be8bdf33 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -11,11 +11,7 @@ builds: # env: # - CGO_ENABLED=0 ldflags: - - -s -w -X main.mongousr={{ .Env.MONGO_USR }} - - -X main.mongopw={{ .Env.MONGO_PW }} - - -X main.mongohost={{ .Env.MONGO_HOST }} - - -X main.mongorepset={{ .Env.MONGO_REPLICA_SET }} - - -X main.spaceapi={{ .Env.SERVICES_API_URL }} + - -s -w -X main.spaceapi={{ .Env.SERVICES_API_URL }} - -X main.vaultapi={{ .Env.VAULT_API_URL }} - -X main.vaultsaltsecret={{ .Env.VAULT_SALT_SECRET }} - -X main.spacehubauth={{ .Env.SERVICES_HUB_AUTH_URL }} @@ -35,11 +31,7 @@ builds: # env: # - CGO_ENABLED=0 ldflags: - - -s -w -X main.mongousr={{ .Env.MONGO_USR }} - - -X main.mongopw={{ .Env.MONGO_PW }} - - -X main.mongohost={{ .Env.MONGO_HOST }} - - -X main.mongorepset={{ .Env.MONGO_REPLICA_SET }} - - -X main.spaceapi={{ .Env.SERVICES_API_URL }} + - -s -w -X main.spaceapi={{ .Env.SERVICES_API_URL }} - -X main.vaultapi={{ .Env.VAULT_API_URL }} - -X main.vaultsaltsecret={{ .Env.VAULT_SALT_SECRET }} - -X main.spacehubauth={{ .Env.SERVICES_HUB_AUTH_URL }} @@ -61,11 +53,7 @@ builds: # env: # - CGO_ENABLED=1 ldflags: - - -s -w -X main.mongousr={{ .Env.MONGO_USR }} - - -X main.mongopw={{ .Env.MONGO_PW }} - - -X main.mongohost={{ .Env.MONGO_HOST }} - - -X main.mongorepset={{ .Env.MONGO_REPLICA_SET }} - - -X main.spaceapi={{ .Env.SERVICES_API_URL }} + - -s -w -X main.spaceapi={{ .Env.SERVICES_API_URL }} - -X main.vaultapi={{ .Env.VAULT_API_URL }} - -X main.vaultsaltsecret={{ .Env.VAULT_SALT_SECRET }} - -X main.spacehubauth={{ .Env.SERVICES_HUB_AUTH_URL }} diff --git a/Makefile b/Makefile index 2c6d966b..88ca079b 100644 --- a/Makefile +++ b/Makefile @@ -4,8 +4,6 @@ build: -ldflags \ "-X 'main.ipfsnodeaddr=${IPFS_NODE_ADDR}' \ -X 'main.ipfsnodepath=${IPFS_NODE_PATH}' \ - -X 'main.mongousr=${MONGO_USR}' \ - -X 'main.mongopw=${MONGO_PW}' \ -X 'main.spaceapi=${SERVICES_API_URL}' \ -X 'main.spacestoragesiteurl=${SPACE_STORAGE_SITE_URL}' \ -X 'main.vaultapi=${VAULT_API_URL}' \ @@ -15,10 +13,8 @@ build: -X 'main.textilehubma=${TXL_HUB_MA}' \ -X 'main.textilethreads=${TXL_THREADS_TARGET}' \ -X 'main.textilehubgatewayurl=${TXL_HUB_GATEWAY_URL}' \ - -X 'main.textileuserkey=${TXL_USER_KEY}' \ - -X 'main.textileusersecret=${TXL_USER_SECRET}' \ - -X 'main.mongohost=${MONGO_HOST}' \ - -X 'main.mongorepset=${MONGO_REPLICA_SET}'" \ + -X 'main.textileuserkey=${TXL_USER_KEY}' \ + -X 'main.textileusersecret=${TXL_USER_SECRET}'" \ cmd/space-daemon/main.go test: diff --git a/README.md b/README.md index b4c06f9a..af26fa63 100644 --- a/README.md +++ b/README.md @@ -76,20 +76,12 @@ These are centralized services that are optional, but offer additional convenien Our hosted Textile Hub requires authentication via public key for logging in. This service sends a challenge to Space Daemon, which signs the challenge with the private key of the user and in that way our hosted Textile Hub can allow the user to store data. -### MongoDB - -Currently, local Textile Threads require a running MongoDB database. Space Daemon by default connects to a hosted one, but this will be removed once Textile switches over to an embeddable data store. - ## Running from source After cloning this repo, you can run it from source by running `go run ./cmd/space-daemon -dev`. Consider that you will need the following environment variables exported in your system: ``` IPFS_ADDR=[Your IPFS node address] -MONGO_PW=[The password of a MongoDB database] -MONGO_USR=[The user of a MongoDB database] -MONGO_HOST=[The host of a MongoDB database] -MONGO_REPLICA_SET=[The replica set for a MongoDB database] SERVICES_API_URL=[The URL where Space Services API is located] VAULT_API_URL=[The URL where Space Vault API is located] VAULT_SALT_SECRET=[A random string used for kdf functions before storing keys to the vault] @@ -188,7 +180,7 @@ Secrets are set by adding them in Github and then specifying them in `release.ym If specified, the release file will dynamically generate the secret name based on the stage by adding a `_DEV` or `_PRD` suffix to the secret name only for the specificed environment variable. It will always use `_PRD` unless the tag ends in `-dev`. So for example tag `v0.0.15` will use PRD values, while `v0.0.15-dev` will use DEV values. -Stage specific secret names will only be used for secrets `release.yml` that point to the step output instead of the secret name directly (i.e., `SERVICES_API_URL: ${{ secrets[steps.secretnames.outputs.SERVICES_API_URL] }}` instead of `MONGO_REPLICA_SET: ${{ secrets.MONGO_REPLICA_SET }}`. +Stage specific secret names will only be used for secrets in `release.yml` that point to the step output instead of the secret name directly (i.e., `SERVICES_API_URL: ${{ secrets[steps.secretnames.outputs.SERVICES_API_URL] }}` instead of `SERVICES_API_URL: ${{ secrets.SERVICES_API_URL }}`. So to add a new secret: * If it's not stage specific then add the secret in GH with no suffix and in `release.yml`, refer to it based on the secret name. diff --git a/cmd/space-daemon/main.go b/cmd/space-daemon/main.go index 15cb52fe..1affe930 100644 --- a/cmd/space-daemon/main.go +++ b/cmd/space-daemon/main.go @@ -32,10 +32,6 @@ var ( ipfsnode = flag.Bool("ipfsnode", true, "run IPFS embedded into the daemon (defaults to true)") ipfsnodeaddr string ipfsnodepath string - mongousr string - mongopw string - mongohost string - mongorepset string spaceapi string spacestoragesiteurl string vaultapi string @@ -64,10 +60,6 @@ func main() { Ipfsnode: *ipfsnode == true, Ipfsnodeaddr: ipfsnodeaddr, Ipfsnodepath: ipfsnodepath, - Mongousr: mongousr, - Mongopw: mongopw, - Mongohost: mongohost, - Mongorepset: mongorepset, ServicesAPIURL: spaceapi, SpaceStorageSiteUrl: spacestoragesiteurl, VaultAPIURL: vaultapi, diff --git a/config/config.go b/config/config.go index 5f401fd6..a8894bdb 100644 --- a/config/config.go +++ b/config/config.go @@ -28,10 +28,6 @@ const ( Ipfsnode = "space/ipfsNode" Ipfsnodeaddr = "space/ipfsNodeAddr" Ipfsnodepath = "space/ipfsNodePath" - Mongousr = "space/mongoUsr" - Mongopw = "space/mongoPw" - Mongohost = "space/mongoHost" - Mongorepset = "space/mongoRepSet" MinThreadsConnection = "space/minThreadsConn" MaxThreadsConnection = "space/maxThreadsConn" ) @@ -45,10 +41,6 @@ type Flags struct { Ipfsnode bool Ipfsnodeaddr string Ipfsnodepath string - Mongousr string - Mongopw string - Mongohost string - Mongorepset string DevMode bool ServicesAPIURL string SpaceStorageSiteUrl string diff --git a/config/map_config.go b/config/map_config.go index 5aa1a8fe..d14a8969 100644 --- a/config/map_config.go +++ b/config/map_config.go @@ -32,10 +32,6 @@ func NewMap(envVal env.SpaceEnv, flags *Flags) Config { configStr[Ipfsaddr] = os.Getenv(env.IpfsAddr) configStr[Ipfsnodeaddr] = os.Getenv(env.IpfsNodeAddr) configStr[Ipfsnodepath] = os.Getenv(env.IpfsNodePath) - configStr[Mongousr] = os.Getenv(env.MongoUsr) - configStr[Mongopw] = os.Getenv(env.MongoPw) - configStr[Mongohost] = os.Getenv(env.MongoHost) - configStr[Mongorepset] = os.Getenv(env.MongoRepSet) configStr[SpaceServicesAPIURL] = os.Getenv(env.ServicesAPIURL) configStr[SpaceVaultAPIURL] = os.Getenv(env.VaultAPIURL) configStr[SpaceVaultSaltSecret] = os.Getenv(env.VaultSaltSecret) @@ -55,10 +51,6 @@ func NewMap(envVal env.SpaceEnv, flags *Flags) Config { configStr[Ipfsaddr] = flags.Ipfsaddr configStr[Ipfsnodeaddr] = flags.Ipfsnodeaddr configStr[Ipfsnodepath] = flags.Ipfsnodepath - configStr[Mongousr] = flags.Mongousr - configStr[Mongopw] = flags.Mongopw - configStr[Mongohost] = flags.Mongohost - configStr[Mongorepset] = flags.Mongorepset configStr[SpaceServicesAPIURL] = flags.ServicesAPIURL configStr[SpaceVaultAPIURL] = flags.VaultAPIURL configStr[SpaceVaultSaltSecret] = flags.VaultSaltSecret diff --git a/core/env/env.go b/core/env/env.go index c18967d9..4586ac16 100644 --- a/core/env/env.go +++ b/core/env/env.go @@ -13,10 +13,6 @@ const ( IpfsNode = "IPFS_NODE" IpfsNodeAddr = "IPFS_NODE_ADDR" IpfsNodePath = "IPFS_NODE_PATH" - MongoUsr = "MONGO_USR" - MongoPw = "MONGO_PW" - MongoHost = "MONGO_HOST" - MongoRepSet = "MONGO_REPLICA_SET" ServicesAPIURL = "SERVICES_API_URL" VaultAPIURL = "VAULT_API_URL" VaultSaltSecret = "VAULT_SALT_SECRET" diff --git a/core/textile/buckd.go b/core/textile/buckd.go index 25ee6208..7c26fe6f 100644 --- a/core/textile/buckd.go +++ b/core/textile/buckd.go @@ -15,10 +15,6 @@ import ( ) var IpfsAddr string -var MongoUsr string -var MongoPw string -var MongoHost string -var MongoRepSet string var MaxThreadsConn int var MinThreadsConn int @@ -38,10 +34,6 @@ func NewBuckd(cfg config.Config) *TextileBuckd { func (tb *TextileBuckd) Start(ctx context.Context) error { IpfsAddr = tb.cfg.GetString(config.Ipfsaddr, "/ip4/127.0.0.1/tcp/5001") - MongoUsr = tb.cfg.GetString(config.Mongousr, "") - MongoPw = tb.cfg.GetString(config.Mongopw, "") - MongoHost = tb.cfg.GetString(config.Mongohost, "") - MongoRepSet = tb.cfg.GetString(config.Mongorepset, "") MinThreadsConn = tb.cfg.GetInt(config.MinThreadsConnection, 50) MaxThreadsConn = tb.cfg.GetInt(config.MaxThreadsConnection, 100) @@ -56,29 +48,22 @@ func (tb *TextileBuckd) Start(ctx context.Context) error { // PLACEHOLDER: filecoin settings - // TODO: replace with embedded store - // addrMongoURI := "mongodb://" + MongoUsr + ":" + MongoPw + "@" + MongoHost - // HOTFIX: in some linux environments the - // above connstr does not work - addrMongoURI := "mongodb://" + MongoUsr + ":" + MongoPw + "@" + MongoHost + "/?ssl=true&replicaSet=" + MongoRepSet + "&authSource=admin&retryWrites=true&w=majority" - usr, err := user.Current() if err != nil { return err } textile, err := core.NewTextile(ctx, core.Config{ - RepoPath: usr.HomeDir + "/.buckd/repo", - AddrAPI: addrAPI, - AddrAPIProxy: addrAPIProxy, - AddrThreadsHost: addrThreadsHost, - AddrIPFSAPI: addrIpfsAPI, - AddrGatewayHost: addrGatewayHost, - AddrGatewayURL: addrGatewayURL, + RepoPath: usr.HomeDir + "/.buckd/repo", + CollectionRepoPath: usr.HomeDir + "/.buckd/collections", + AddrAPI: addrAPI, + AddrAPIProxy: addrAPIProxy, + AddrThreadsHost: addrThreadsHost, + AddrIPFSAPI: addrIpfsAPI, + AddrGatewayHost: addrGatewayHost, + AddrGatewayURL: addrGatewayURL, //AddrPowergateAPI: addrPowergateApi, - AddrMongoURI: addrMongoURI, //UseSubdomains: config.Viper.GetBool("gateway.subdomains"), - AddrMongoName: "buckets", //DNSDomain: dnsDomain, //DNSZoneID: dnsZoneID, //DNSToken: dnsToken, diff --git a/go.mod b/go.mod index 12697bb0..dc8253b8 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/FleekHQ/space-daemon go 1.14 replace github.com/textileio/go-threads => github.com/FleekHQ/go-threads v1.0.1-0.20201028195307-d9371c20fe66 +replace github.com/textileio/textile/v2 => github.com/FleekHQ/textile/v2 v2.0.0-20201127024116-cee5aaade92c replace github.com/libp2p/go-libp2p-pubsub => github.com/libp2p/go-libp2p-pubsub v0.3.2 replace github.com/libp2p/go-libp2p-core => github.com/libp2p/go-libp2p-core v0.6.1 diff --git a/go.sum b/go.sum index 65c9b45d..027c5e50 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,11 @@ github.com/FleekHQ/go-threads v1.0.1-0.20201019173032-43cf56e94181 h1:A6v9Sbh/1s github.com/FleekHQ/go-threads v1.0.1-0.20201019173032-43cf56e94181/go.mod h1:secctVT6GgkKoQPUz/Eg/FK05e3eiRdyhkkJMA1cuQg= github.com/FleekHQ/go-threads v1.0.1-0.20201028195307-d9371c20fe66 h1:JppBUaU4v56XxHj8fOMK4jjYCKCdiEMOrhXibu/Jbs8= github.com/FleekHQ/go-threads v1.0.1-0.20201028195307-d9371c20fe66/go.mod h1:mQgVlEzC++pJ0+EyrhW19y5ZHa8xogdYQq/8pLUvOm4= +github.com/FleekHQ/space-daemon v0.0.33/go.mod h1:dpCcfU+b6FP+whLdnPwaSAbcmVGQwYcNedsbEumE/Mc= +github.com/FleekHQ/textile/v2 v2.0.0-20201116173414-db43a5d7591f h1:Phw2fWt6eJ2SAVXJNZaGXqQyT3I5HDOxibM5ocqVLvM= +github.com/FleekHQ/textile/v2 v2.0.0-20201116173414-db43a5d7591f/go.mod h1:BXhNUwvl7dR0fvNi7xaKTSUEent09vJ7bhNYkMNzQ7c= +github.com/FleekHQ/textile/v2 v2.0.0-20201127024116-cee5aaade92c h1:Qu2C3VP+G0maRQYrOxyU0HTqVBNXAjh/5Uyw3Y/DIQ8= +github.com/FleekHQ/textile/v2 v2.0.0-20201127024116-cee5aaade92c/go.mod h1:telji7Pfrll6wugWd0cSk1u3FSsTESOPfL39r0QTloU= github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= @@ -213,12 +218,12 @@ github.com/cloudflare/cloudflare-go v0.11.6 h1:gErXaYucoS8aHdmoJnF4RMFiXJH449sk6 github.com/cloudflare/cloudflare-go v0.11.6/go.mod h1:lmCbgQdBeSQlMv0W0OSqoGgl8aFrgc5oXHhWMt47dh0= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b/go.mod h1:hU7vhtrqonEphNF+xt8/lHdaBprxmV1h8BOGrd9XwmQ= github.com/cockroachdb/pebble v0.0.0-20201001221639-879f3bfeef07/go.mod h1:hU7vhtrqonEphNF+xt8/lHdaBprxmV1h8BOGrd9XwmQ= github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20200228182428-0f16d7a0959c h1:8ahmSVELW1wghbjerVAyuEYD5+Dio66RYvSS0iGfL1M=