Remove mongo: point daemon to use fork of textile (#266)

* Point to forked textile: init

* Optional-mongo: cleanup makefile, point to latest version fo buckets fork

* Optional-mongo: clean up remaining mongo refs

* Optional-mongo: update readme

.github/workflows/release.yml

@@ -51,10 +51,6 @@ jobs:
           args: release
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          MONGO_USR: ${{ secrets.MONGO_USR }}
-          MONGO_PW: ${{ secrets.MONGO_PW }}
-          MONGO_HOST: ${{ secrets.MONGO_HOST }}
-          MONGO_REPLICA_SET: ${{ secrets.MONGO_REPLICA_SET }}
           SERVICES_API_URL: ${{ secrets[steps.secretnames.outputs.SERVICES_API_URL] }}
           VAULT_API_URL: ${{ secrets[steps.secretnames.outputs.VAULT_API_URL] }}
           VAULT_SALT_SECRET: ${{ secrets[steps.secretnames.outputs.VAULT_SALT_SECRET] }}

.goreleaser.yml

@@ -11,11 +11,7 @@ builds:
     # env:
     #   - CGO_ENABLED=0
     ldflags:
-      - -s -w -X main.mongousr={{ .Env.MONGO_USR }}
-      - -X main.mongopw={{ .Env.MONGO_PW }}
-      - -X main.mongohost={{ .Env.MONGO_HOST }}
-      - -X main.mongorepset={{ .Env.MONGO_REPLICA_SET }}
-      - -X main.spaceapi={{ .Env.SERVICES_API_URL }}
+      - -s -w -X  main.spaceapi={{ .Env.SERVICES_API_URL }}
       - -X main.vaultapi={{ .Env.VAULT_API_URL }}
       - -X main.vaultsaltsecret={{ .Env.VAULT_SALT_SECRET }}
       - -X main.spacehubauth={{ .Env.SERVICES_HUB_AUTH_URL }}
@@ -35,11 +31,7 @@ builds:
     # env:
     #   - CGO_ENABLED=0
     ldflags:
-      - -s -w -X main.mongousr={{ .Env.MONGO_USR }}
-      - -X main.mongopw={{ .Env.MONGO_PW }}
-      - -X main.mongohost={{ .Env.MONGO_HOST }}
-      - -X main.mongorepset={{ .Env.MONGO_REPLICA_SET }}
-      - -X main.spaceapi={{ .Env.SERVICES_API_URL }}
+      - -s -w -X main.spaceapi={{ .Env.SERVICES_API_URL }}
       - -X main.vaultapi={{ .Env.VAULT_API_URL }}
       - -X main.vaultsaltsecret={{ .Env.VAULT_SALT_SECRET }}
       - -X main.spacehubauth={{ .Env.SERVICES_HUB_AUTH_URL }}
@@ -61,11 +53,7 @@ builds:
     # env:
     #   - CGO_ENABLED=1
     ldflags:
-      - -s -w -X main.mongousr={{ .Env.MONGO_USR }}
-      - -X main.mongopw={{ .Env.MONGO_PW }}
-      - -X main.mongohost={{ .Env.MONGO_HOST }}
-      - -X main.mongorepset={{ .Env.MONGO_REPLICA_SET }}
-      - -X main.spaceapi={{ .Env.SERVICES_API_URL }}
+      - -s -w -X main.spaceapi={{ .Env.SERVICES_API_URL }}
       - -X main.vaultapi={{ .Env.VAULT_API_URL }}
       - -X main.vaultsaltsecret={{ .Env.VAULT_SALT_SECRET }}
       - -X main.spacehubauth={{ .Env.SERVICES_HUB_AUTH_URL }}

Makefile

@@ -4,8 +4,6 @@ build:
 	-ldflags \
 	"-X 'main.ipfsnodeaddr=${IPFS_NODE_ADDR}' \
 	-X 'main.ipfsnodepath=${IPFS_NODE_PATH}' \
-	-X 'main.mongousr=${MONGO_USR}' \
-	-X 'main.mongopw=${MONGO_PW}' \
 	-X 'main.spaceapi=${SERVICES_API_URL}' \
 	-X 'main.spacestoragesiteurl=${SPACE_STORAGE_SITE_URL}' \
 	-X 'main.vaultapi=${VAULT_API_URL}' \
@@ -15,10 +13,8 @@ build:
 	-X 'main.textilehubma=${TXL_HUB_MA}' \
 	-X 'main.textilethreads=${TXL_THREADS_TARGET}' \
 	-X 'main.textilehubgatewayurl=${TXL_HUB_GATEWAY_URL}' \
-  	-X 'main.textileuserkey=${TXL_USER_KEY}' \
-	-X 'main.textileusersecret=${TXL_USER_SECRET}' \
-	-X 'main.mongohost=${MONGO_HOST}' \
-	-X 'main.mongorepset=${MONGO_REPLICA_SET}'" \
+    -X 'main.textileuserkey=${TXL_USER_KEY}' \
+	-X 'main.textileusersecret=${TXL_USER_SECRET}'" \
 	cmd/space-daemon/main.go
 
 test:

README.md

@@ -76,20 +76,12 @@ These are centralized services that are optional, but offer additional convenien
 
 Our hosted Textile Hub requires authentication via public key for logging in. This service sends a challenge to Space Daemon, which signs the challenge with the private key of the user and in that way our hosted Textile Hub can allow the user to store data.
 
-### MongoDB
-
-Currently, local Textile Threads require a running MongoDB database. Space Daemon by default connects to a hosted one, but this will be removed once Textile switches over to an embeddable data store.
-
 ## Running from source
 
 After cloning this repo, you can run it from source by running `go run ./cmd/space-daemon -dev`. Consider that you will need the following environment variables exported in your system:
 
 ```
 IPFS_ADDR=[Your IPFS node address]
-MONGO_PW=[The password of a MongoDB database]
-MONGO_USR=[The user of a MongoDB database]
-MONGO_HOST=[The host of a MongoDB database]
-MONGO_REPLICA_SET=[The replica set for a MongoDB database]
 SERVICES_API_URL=[The URL where Space Services API is located]
 VAULT_API_URL=[The URL where Space Vault API is located]
 VAULT_SALT_SECRET=[A random string used for kdf functions before storing keys to the vault]
@@ -188,7 +180,7 @@ Secrets are set by adding them in Github and then specifying them in `release.ym
 
 If specified, the release file will dynamically generate the secret name based on the stage by adding a `_DEV` or `_PRD` suffix to the secret name only for the specificed environment variable. It will always use `_PRD` unless the tag ends in `-dev`.  So for example tag `v0.0.15` will use PRD values, while `v0.0.15-dev` will use DEV values.
 
-Stage specific secret names will only be used for secrets `release.yml` that point to the step output instead of the secret name directly (i.e., `SERVICES_API_URL: ${{ secrets[steps.secretnames.outputs.SERVICES_API_URL] }}` instead of `MONGO_REPLICA_SET: ${{ secrets.MONGO_REPLICA_SET }}`.
+Stage specific secret names will only be used for secrets in `release.yml` that point to the step output instead of the secret name directly (i.e., `SERVICES_API_URL: ${{ secrets[steps.secretnames.outputs.SERVICES_API_URL] }}` instead of `SERVICES_API_URL: ${{ secrets.SERVICES_API_URL }}`.
 
 So to add a new secret:
 * If it's not stage specific then add the secret in GH with no suffix and in `release.yml`, refer to it based on the secret name.

cmd/space-daemon/main.go

@@ -32,10 +32,6 @@ var (
 	ipfsnode             = flag.Bool("ipfsnode", true, "run IPFS embedded into the daemon (defaults to true)")
 	ipfsnodeaddr         string
 	ipfsnodepath         string
-	mongousr             string
-	mongopw              string
-	mongohost            string
-	mongorepset          string
 	spaceapi             string
 	spacestoragesiteurl  string
 	vaultapi             string
@@ -64,10 +60,6 @@ func main() {
 		Ipfsnode:             *ipfsnode == true,
 		Ipfsnodeaddr:         ipfsnodeaddr,
 		Ipfsnodepath:         ipfsnodepath,
-		Mongousr:             mongousr,
-		Mongopw:              mongopw,
-		Mongohost:            mongohost,
-		Mongorepset:          mongorepset,
 		ServicesAPIURL:       spaceapi,
 		SpaceStorageSiteUrl:  spacestoragesiteurl,
 		VaultAPIURL:          vaultapi,

config/config.go

@@ -28,10 +28,6 @@ const (
 	Ipfsnode                 = "space/ipfsNode"
 	Ipfsnodeaddr             = "space/ipfsNodeAddr"
 	Ipfsnodepath             = "space/ipfsNodePath"
-	Mongousr                 = "space/mongoUsr"
-	Mongopw                  = "space/mongoPw"
-	Mongohost                = "space/mongoHost"
-	Mongorepset              = "space/mongoRepSet"
 	MinThreadsConnection     = "space/minThreadsConn"
 	MaxThreadsConnection     = "space/maxThreadsConn"
 )
@@ -45,10 +41,6 @@ type Flags struct {
 	Ipfsnode             bool
 	Ipfsnodeaddr         string
 	Ipfsnodepath         string
-	Mongousr             string
-	Mongopw              string
-	Mongohost            string
-	Mongorepset          string
 	DevMode              bool
 	ServicesAPIURL       string
 	SpaceStorageSiteUrl  string

config/map_config.go

@@ -32,10 +32,6 @@ func NewMap(envVal env.SpaceEnv, flags *Flags) Config {
 		configStr[Ipfsaddr] = os.Getenv(env.IpfsAddr)
 		configStr[Ipfsnodeaddr] = os.Getenv(env.IpfsNodeAddr)
 		configStr[Ipfsnodepath] = os.Getenv(env.IpfsNodePath)
-		configStr[Mongousr] = os.Getenv(env.MongoUsr)
-		configStr[Mongopw] = os.Getenv(env.MongoPw)
-		configStr[Mongohost] = os.Getenv(env.MongoHost)
-		configStr[Mongorepset] = os.Getenv(env.MongoRepSet)
 		configStr[SpaceServicesAPIURL] = os.Getenv(env.ServicesAPIURL)
 		configStr[SpaceVaultAPIURL] = os.Getenv(env.VaultAPIURL)
 		configStr[SpaceVaultSaltSecret] = os.Getenv(env.VaultSaltSecret)
@@ -55,10 +51,6 @@ func NewMap(envVal env.SpaceEnv, flags *Flags) Config {
 		configStr[Ipfsaddr] = flags.Ipfsaddr
 		configStr[Ipfsnodeaddr] = flags.Ipfsnodeaddr
 		configStr[Ipfsnodepath] = flags.Ipfsnodepath
-		configStr[Mongousr] = flags.Mongousr
-		configStr[Mongopw] = flags.Mongopw
-		configStr[Mongohost] = flags.Mongohost
-		configStr[Mongorepset] = flags.Mongorepset
 		configStr[SpaceServicesAPIURL] = flags.ServicesAPIURL
 		configStr[SpaceVaultAPIURL] = flags.VaultAPIURL
 		configStr[SpaceVaultSaltSecret] = flags.VaultSaltSecret

core/env/env.go

@@ -13,10 +13,6 @@ const (
 	IpfsNode             = "IPFS_NODE"
 	IpfsNodeAddr         = "IPFS_NODE_ADDR"
 	IpfsNodePath         = "IPFS_NODE_PATH"
-	MongoUsr             = "MONGO_USR"
-	MongoPw              = "MONGO_PW"
-	MongoHost            = "MONGO_HOST"
-	MongoRepSet          = "MONGO_REPLICA_SET"
 	ServicesAPIURL       = "SERVICES_API_URL"
 	VaultAPIURL          = "VAULT_API_URL"
 	VaultSaltSecret      = "VAULT_SALT_SECRET"

core/textile/buckd.go

@@ -15,10 +15,6 @@ import (
 )
 
 var IpfsAddr string
-var MongoUsr string
-var MongoPw string
-var MongoHost string
-var MongoRepSet string
 var MaxThreadsConn int
 var MinThreadsConn int
 
@@ -38,10 +34,6 @@ func NewBuckd(cfg config.Config) *TextileBuckd {
 
 func (tb *TextileBuckd) Start(ctx context.Context) error {
 	IpfsAddr = tb.cfg.GetString(config.Ipfsaddr, "/ip4/127.0.0.1/tcp/5001")
-	MongoUsr = tb.cfg.GetString(config.Mongousr, "")
-	MongoPw = tb.cfg.GetString(config.Mongopw, "")
-	MongoHost = tb.cfg.GetString(config.Mongohost, "")
-	MongoRepSet = tb.cfg.GetString(config.Mongorepset, "")
 	MinThreadsConn = tb.cfg.GetInt(config.MinThreadsConnection, 50)
 	MaxThreadsConn = tb.cfg.GetInt(config.MaxThreadsConnection, 100)
 
@@ -56,29 +48,22 @@ func (tb *TextileBuckd) Start(ctx context.Context) error {
 
 	// PLACEHOLDER: filecoin settings
 
-	// TODO: replace with embedded store
-	// addrMongoURI := "mongodb://" + MongoUsr + ":" + MongoPw + "@" + MongoHost
-	// HOTFIX: in some linux environments the
-	// above connstr does not work
-	addrMongoURI := "mongodb://" + MongoUsr + ":" + MongoPw + "@" + MongoHost + "/?ssl=true&replicaSet=" + MongoRepSet + "&authSource=admin&retryWrites=true&w=majority"
-
 	usr, err := user.Current()
 	if err != nil {
 		return err
 	}
 
 	textile, err := core.NewTextile(ctx, core.Config{
-		RepoPath:        usr.HomeDir + "/.buckd/repo",
-		AddrAPI:         addrAPI,
-		AddrAPIProxy:    addrAPIProxy,
-		AddrThreadsHost: addrThreadsHost,
-		AddrIPFSAPI:     addrIpfsAPI,
-		AddrGatewayHost: addrGatewayHost,
-		AddrGatewayURL:  addrGatewayURL,
+		RepoPath:           usr.HomeDir + "/.buckd/repo",
+		CollectionRepoPath: usr.HomeDir + "/.buckd/collections",
+		AddrAPI:            addrAPI,
+		AddrAPIProxy:       addrAPIProxy,
+		AddrThreadsHost:    addrThreadsHost,
+		AddrIPFSAPI:        addrIpfsAPI,
+		AddrGatewayHost:    addrGatewayHost,
+		AddrGatewayURL:     addrGatewayURL,
 		//AddrPowergateAPI: addrPowergateApi,
-		AddrMongoURI: addrMongoURI,
 		//UseSubdomains:    config.Viper.GetBool("gateway.subdomains"),
-		AddrMongoName: "buckets",
 		//DNSDomain:        dnsDomain,
 		//DNSZoneID:        dnsZoneID,
 		//DNSToken:         dnsToken,

go.mod

@@ -3,6 +3,7 @@ module github.com/FleekHQ/space-daemon
 go 1.14
 
 replace github.com/textileio/go-threads => github.com/FleekHQ/go-threads v1.0.1-0.20201028195307-d9371c20fe66
+replace github.com/textileio/textile/v2 => github.com/FleekHQ/textile/v2 v2.0.0-20201127024116-cee5aaade92c
 
 replace github.com/libp2p/go-libp2p-pubsub => github.com/libp2p/go-libp2p-pubsub v0.3.2
 replace github.com/libp2p/go-libp2p-core => github.com/libp2p/go-libp2p-core v0.6.1

go.sum

@@ -61,6 +61,11 @@ github.com/FleekHQ/go-threads v1.0.1-0.20201019173032-43cf56e94181 h1:A6v9Sbh/1s
 github.com/FleekHQ/go-threads v1.0.1-0.20201019173032-43cf56e94181/go.mod h1:secctVT6GgkKoQPUz/Eg/FK05e3eiRdyhkkJMA1cuQg=
 github.com/FleekHQ/go-threads v1.0.1-0.20201028195307-d9371c20fe66 h1:JppBUaU4v56XxHj8fOMK4jjYCKCdiEMOrhXibu/Jbs8=
 github.com/FleekHQ/go-threads v1.0.1-0.20201028195307-d9371c20fe66/go.mod h1:mQgVlEzC++pJ0+EyrhW19y5ZHa8xogdYQq/8pLUvOm4=
+github.com/FleekHQ/space-daemon v0.0.33/go.mod h1:dpCcfU+b6FP+whLdnPwaSAbcmVGQwYcNedsbEumE/Mc=
+github.com/FleekHQ/textile/v2 v2.0.0-20201116173414-db43a5d7591f h1:Phw2fWt6eJ2SAVXJNZaGXqQyT3I5HDOxibM5ocqVLvM=
+github.com/FleekHQ/textile/v2 v2.0.0-20201116173414-db43a5d7591f/go.mod h1:BXhNUwvl7dR0fvNi7xaKTSUEent09vJ7bhNYkMNzQ7c=
+github.com/FleekHQ/textile/v2 v2.0.0-20201127024116-cee5aaade92c h1:Qu2C3VP+G0maRQYrOxyU0HTqVBNXAjh/5Uyw3Y/DIQ8=
+github.com/FleekHQ/textile/v2 v2.0.0-20201127024116-cee5aaade92c/go.mod h1:telji7Pfrll6wugWd0cSk1u3FSsTESOPfL39r0QTloU=
 github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
 github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ=
 github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
@@ -213,12 +218,12 @@ github.com/cloudflare/cloudflare-go v0.11.6 h1:gErXaYucoS8aHdmoJnF4RMFiXJH449sk6
 github.com/cloudflare/cloudflare-go v0.11.6/go.mod h1:lmCbgQdBeSQlMv0W0OSqoGgl8aFrgc5oXHhWMt47dh0=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
-github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
 github.com/cockroachdb/pebble v0.0.0-20200916222308-4e219a90ba5b/go.mod h1:hU7vhtrqonEphNF+xt8/lHdaBprxmV1h8BOGrd9XwmQ=
 github.com/cockroachdb/pebble v0.0.0-20201001221639-879f3bfeef07/go.mod h1:hU7vhtrqonEphNF+xt8/lHdaBprxmV1h8BOGrd9XwmQ=
 github.com/cockroachdb/redact v0.0.0-20200622112456-cd282804bbd3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
+github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.0.0-20200228182428-0f16d7a0959c h1:8ahmSVELW1wghbjerVAyuEYD5+Dio66RYvSS0iGfL1M=