Chore: Integration Test & Fixtures Setup (#268)

.github/workflows/test.yml

@@ -2,10 +2,9 @@
 on: [pull_request]
 name: Test
 jobs:
-  test:
+  unit-test:
     strategy:
       matrix:
-        #go-version: [1.13.x, 1.14.x]
         go-version: [1.14.x]
         platform: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.platform }}
@@ -16,5 +15,70 @@ jobs:
           go-version: ${{ matrix.go-version }}
       - name: Checkout code
         uses: actions/checkout@v2
-      - name: Test
-        run: go test -race ./...
+      - name: Unit Test
+        if: ${{ matrix.platform != 'windows-latest' }}
+        run: go test -race -coverprofile=coverage/unitcoverage.out $(go list ./... | grep -v integration_tests)
+      - name: Unit Test (Win)
+        if: ${{ matrix.platform == 'windows-latest' }} # skipping coverage collection on windows
+        run: go test -race $(go list ./... | grep -v integration_tests)
+      - name: Coveralls
+        if: ${{ matrix.platform != 'windows-latest' }}
+        uses: shogo82148/actions-goveralls@v1
+        with:
+          flag-name: unit-test-${{ matrix.platform }}
+          path-to-profile: coverage/unitcoverage.out
+          parallel: true
+  integration-test:
+    strategy:
+      matrix:
+        go-version: [ 1.14.x ]
+#        platform: [ ubuntu-latest, macos-latest, windows-latest ]
+        platform: [ macos-latest ]
+    runs-on: ${{ matrix.platform }}
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      SERVICES_API_URL: ${{ secrets.outputs.SERVICES_API_URL_DEV }}
+      VAULT_API_URL: ${{ secrets.VAULT_API_URL_DEV }}
+      VAULT_SALT_SECRET: ${{ secrets.VAULT_SALT_SECRET_DEV }}
+      SERVICES_HUB_AUTH_URL: ${{ secrets.SERVICES_HUB_AUTH_URL_DEV }}
+      TXL_HUB_TARGET: ${{ secrets.TXL_HUB_TARGET_DEV }}
+      TXL_HUB_MA: ${{ secrets.TXL_HUB_MA_DEV }}
+      TXL_THREADS_TARGET: ${{ secrets.TXL_THREADS_TARGET_DEV }}
+      TXL_HUB_GATEWAY_URL: ${{ secrets.TXL_HUB_GATEWAY_URL_DEV }}
+      TXL_USER_KEY: ${{ secrets.TXL_USER_KEY_DEV }}
+      TXL_USER_SECRET: ${{ secrets.TXL_USER_SECRET_DEV }}
+      SPACE_STORAGE_SITE_URL: ${{ secrets.SPACE_STORAGE_SITE_URL_DEV }}
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Install gnome-keyring (Ubuntu)
+        if: ${{ matrix.platform == 'ubuntu-latest' }}
+        run: sudo apt-get install pass gnome-keyring dbus-x11
+      - name: Verify gnome-keyring is installed (Ubuntu)
+        if: ${{ matrix.platform == 'ubuntu-latest' }}
+        run: gnome-keyring-daemon -V
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Integration Test
+        if: ${{ matrix.platform != 'windows-latest' }}
+        run: go test -v -timeout 60m -coverprofile=coverage/integrationcoverage.out ./integration_tests/...
+      - name: Integration Test (Win)
+        if: ${{ matrix.platform == 'windows-latest' }} # skipping coverage collection on windows
+        run: go test -v -timeout 60m ./integration_tests/...
+      - name: Coveralls
+        if: ${{ matrix.platform != 'windows-latest' }}
+        uses: shogo82148/actions-goveralls@v1
+        with:
+          flag-name: integration-test-${{ matrix.platform }}
+          path-to-profile: coverage/integrationcoverage.out
+          parallel: true
+  submit-coverage:
+    needs: [unit-test, integration-test]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Coveralls Finished
+        uses: shogo82148/actions-goveralls@v1
+        with:
+          parallel-finished: true

.gitignore

@@ -10,6 +10,8 @@
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+coverage/*
+!coverage/.gitkeep
 
 # Dependency directories (remove the comment below to include it)
 # vendor/

Makefile

@@ -19,7 +19,16 @@ build:
 	cmd/space-daemon/main.go
 
 test:
-	go test ./...
+	go test $$(go list ./... | grep -v integration_tests)
+
+test_coverage:
+	go test -coverprofile=coverage/unitcoverage.out $$(go list ./... | grep -v integration_tests)
+
+integration_test:
+	go test -v -p 1 ./integration_tests/...
+
+integration_test_coverage:
+	go test -v -p 1 -coverprofile=coverage/integrationcoverage.out ./integration_tests/...
 
 proto_gen:
 	protoc -I grpc/pb/ -I grpc/proto/ -I./devtools/googleapis grpc/proto/space.proto --go_out=plugins=grpc:grpc/pb

app/app.go

@@ -3,13 +3,11 @@ package app
 import (
 	"context"
 	"fmt"
-	"os"
-	"os/signal"
-	"syscall"
 
 	"github.com/FleekHQ/space-daemon/core/space/fuse/installer"
 
 	"github.com/FleekHQ/space-daemon/core/search/bleve"
+	"github.com/pkg/errors"
 
 	"github.com/FleekHQ/space-daemon/core"
 	"github.com/FleekHQ/space-daemon/grpc"
@@ -41,11 +39,11 @@ import (
 
 // Shutdown logic follows this example https://gist.github.com/akhenakh/38dbfea70dc36964e23acc19777f3869
 type App struct {
-	eg             *errgroup.Group
-	components     *stack.Stack
-	cfg            config.Config
-	env            env.SpaceEnv
-	isShuttingDown bool
+	eg         *errgroup.Group
+	components *stack.Stack
+	cfg        config.Config
+	env        env.SpaceEnv
+	IsRunning  bool
 }
 
 type componentMap struct {
@@ -55,10 +53,10 @@ type componentMap struct {
 
 func New(cfg config.Config, env env.SpaceEnv) *App {
 	return &App{
-		components:     stack.New(),
-		cfg:            cfg,
-		env:            env,
-		isShuttingDown: false,
+		components: stack.New(),
+		cfg:        cfg,
+		env:        env,
+		IsRunning:  false,
 	}
 }
 
@@ -68,13 +66,11 @@ func New(cfg config.Config, env env.SpaceEnv) *App {
 // added to the apps list of tracked components using the `Run()` function, but if the component has a blocking
 // start/run function it should be tracked with the `RunAsync()` function and call the blocking function in the
 // input function block.
-func (a *App) Start(ctx context.Context) error {
-	a.eg, ctx = errgroup.WithContext(ctx)
+func (a *App) Start() error {
+	var ctx context.Context
+	a.eg, ctx = errgroup.WithContext(context.Background())
 
-	// setup to detect interruption
-	interrupt := make(chan os.Signal, 1)
-	signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM)
-	defer signal.Stop(interrupt)
+	log.SetLogLevel(a.cfg.GetString(config.LogLevel, "debug"))
 
 	// init appStore
 	appStore := store.New(
@@ -203,19 +199,9 @@ func (a *App) Start(ctx context.Context) error {
 	}
 
 	log.Info("Daemon ready")
+	a.IsRunning = true
 
-	// wait for interruption or done signal
-	select {
-	case <-interrupt:
-		log.Debug("Got interrupt signal")
-		// TODO: Track multiple interrupts in a goroutine to force exit for app.
-		break
-	case <-ctx.Done():
-		log.Debug("Got context done signal")
-		break
-	}
-
-	return a.Shutdown()
+	return nil
 }
 
 // Run registers this component to be cleaned up on Shutdown
@@ -264,6 +250,10 @@ func (a *App) RunAsync(name string, component core.AsyncComponent, fn func() err
 // Run() or RunAsync() functions
 func (a *App) Shutdown() error {
 	log.Info("Daemon shutdown started")
+	if !a.IsRunning {
+		return errors.New("app is not running")
+	}
+
 	for a.components.Len() > 0 {
 		m, ok := a.components.Pop().(*componentMap)
 		if ok {
@@ -276,5 +266,6 @@ func (a *App) Shutdown() error {
 
 	err := a.eg.Wait()
 	log.Info("Shutdown complete")
+	a.IsRunning = false
 	return err
 }

cmd/space-daemon/main.go

@@ -1,14 +1,15 @@
 package main
 
 import (
-	"context"
 	"flag"
 	"fmt"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
+	"os/signal"
 	"runtime"
 	"runtime/pprof"
+	"syscall"
 
 	"github.com/FleekHQ/space-daemon/tracing"
 
@@ -99,24 +100,33 @@ func main() {
 	env := env.New()
 
 	// load configs
-	cfg := config.NewMap(env, cf)
-
-	// setup context
-	ctx := context.Background()
+	cfg := config.NewMap(cf)
 
 	rlimit.SetRLimit()
 
 	spaceApp := app.New(cfg, env)
-	// this blocks and returns on exist
-	err := spaceApp.Start(ctx)
+
+	err := spaceApp.Start()
+	if err != nil {
+		log.Error("Application startup failed", err)
+		returnCode = 1
+	}
+
+	// setup to detect interruption
+	interrupt := make(chan os.Signal, 1)
+	signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM)
+	defer signal.Stop(interrupt)
+
+	<-interrupt // wait for interrupt and then shutdown app
+	err = spaceApp.Shutdown()
 
 	if *memprofile != "" {
 		cleanupMemProfile := runMemProfiler(*memprofile)
 		defer cleanupMemProfile()
 	}
 
 	if err != nil {
-		log.Error("Application startup failed", err)
+		log.Error("Application shutdown failed", err)
 		returnCode = 1
 	}
 }

config/config.go

@@ -30,29 +30,45 @@ const (
 	Ipfsnodepath             = "space/ipfsNodePath"
 	MinThreadsConnection     = "space/minThreadsConn"
 	MaxThreadsConnection     = "space/maxThreadsConn"
+	BuckdPath                = "space/BuckdPath"
+	BuckdApiMaAddr           = "space/BuckdApiMaAddr"
+	BuckdApiProxyMaAddr      = "space/BuckdApiProxyMaAddr"
+	BuckdThreadsHostMaAddr   = "Space/BuckdThreadsHostMaAddr"
+	BuckdGatewayPort         = "Space/BuckdGatewayPort"
+	LogLevel                 = "Space/LogLevel"
 )
 
 var (
 	ErrConfigNotLoaded = errors.New("config file was not loaded correctly or it does not exist")
 )
 
 type Flags struct {
-	Ipfsaddr             string
-	Ipfsnode             bool
-	Ipfsnodeaddr         string
-	Ipfsnodepath         string
-	DevMode              bool
-	ServicesAPIURL       string
-	SpaceStorageSiteUrl  string
-	VaultAPIURL          string
-	VaultSaltSecret      string
-	ServicesHubAuthURL   string
-	TextileHubTarget     string
-	TextileHubMa         string
-	TextileThreadsTarget string
-	TextileHubGatewayUrl string
-	TextileUserKey       string
-	TextileUserSecret    string
+	Ipfsaddr               string
+	Ipfsnode               bool
+	Ipfsnodeaddr           string
+	Ipfsnodepath           string
+	DevMode                bool
+	ServicesAPIURL         string
+	SpaceStorageSiteUrl    string
+	VaultAPIURL            string
+	VaultSaltSecret        string
+	ServicesHubAuthURL     string
+	TextileHubTarget       string
+	TextileHubMa           string
+	TextileThreadsTarget   string
+	TextileHubGatewayUrl   string
+	TextileUserKey         string
+	TextileUserSecret      string
+	SpaceStorePath         string
+	RpcServerPort          int
+	RpcProxyServerPort     int
+	RestProxyServerPort    int
+	BuckdPath              string
+	BuckdApiMaAddr         string
+	BuckdApiProxyMaAddr    string
+	BuckdThreadsHostMaAddr string
+	BuckdGatewayPort       int
+	LogLevel               string
 }
 
 // Config used to fetch config information

config/map_config.go

@@ -14,14 +14,15 @@ type mapConfig struct {
 	configBool map[string]bool
 }
 
-func NewMap(envVal env.SpaceEnv, flags *Flags) Config {
+func NewMap(flags *Flags) Config {
 	configStr := make(map[string]string)
 	configInt := make(map[string]int)
 	configBool := make(map[string]bool)
 
 	usr, _ := user.Current()
 
 	// default values
+	configStr[LogLevel] = flags.LogLevel
 	configStr[SpaceStorePath] = filepath.Join(usr.HomeDir, ".fleek-space")
 	configStr[MountFuseDrive] = "false"
 	configStr[FuseDriveName] = "Space"
@@ -65,6 +66,33 @@ func NewMap(envVal env.SpaceEnv, flags *Flags) Config {
 		configStr[TextileUserKey] = flags.TextileUserKey
 		configStr[TextileUserSecret] = flags.TextileUserSecret
 		configBool[Ipfsnode] = flags.Ipfsnode
+		if flags.SpaceStorePath != "" {
+			configStr[SpaceStorePath] = flags.SpaceStorePath
+		}
+		if flags.RpcServerPort != 0 {
+			configInt[SpaceServerPort] = flags.RpcServerPort
+		}
+		if flags.RpcProxyServerPort != 0 {
+			configInt[SpaceProxyServerPort] = flags.RpcProxyServerPort
+		}
+		if flags.RestProxyServerPort != 0 {
+			configInt[SpaceRestProxyServerPort] = flags.RestProxyServerPort
+		}
+		if flags.BuckdPath != "" {
+			configStr[BuckdPath] = flags.BuckdPath
+		}
+		if flags.BuckdApiMaAddr != "" {
+			configStr[BuckdApiMaAddr] = flags.BuckdApiMaAddr
+		}
+		if flags.BuckdApiProxyMaAddr != "" {
+			configStr[BuckdApiProxyMaAddr] = flags.BuckdApiProxyMaAddr
+		}
+		if flags.BuckdThreadsHostMaAddr != "" {
+			configStr[BuckdThreadsHostMaAddr] = flags.BuckdThreadsHostMaAddr
+		}
+		if flags.BuckdGatewayPort != 0 {
+			configInt[BuckdGatewayPort] = flags.BuckdGatewayPort
+		}
 	}
 
 	// Temp fix until we move to viper