chore: Make keyring non-interactive (#3026)

## Relevant issue(s)

Resolves #2995 

## Description

This PR makes the keyring interactions non-interactive by requiring the
keyring password to be set as an environment variable secret. It also
adds support for that secret to be stored in a `.env` file in the
working directory or in a file at a path defined by the `--secret-file`
flag.

Making the keyring non-interactive is necessary to support automated
deployments.

.gitignore

@@ -58,3 +58,6 @@ crash.log
 
 # Ignore .zip files, such as Lambda Function code slugs.
 **.zip
+
+# Ignore .env files containing sensitive information.
+.env

README.md

@@ -68,7 +68,11 @@ The following keys are loaded from the keyring on start:
 - `peer-key` Ed25519 private key (required)
 - `encryption-key` AES-128, AES-192, or AES-256 key (optional)
 
-To randomly generate the required keys, run the following command:
+A secret to unlock the keyring is required on start and must be provided via the `DEFRADB_KEYRING_SECRET` environment variable. If a `.env` file is available in the working directory, the secret can be stored there or via a file at a path defined by the `--keyring-secret-file` flag.
+
+The keys will be randomly generated on the inital start of the node if they are not found.
+
+Alternatively, to randomly generate the required keys, run the following command:
 
 ```
 defradb keyring generate

cli/config.go

@@ -16,6 +16,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/joho/godotenv"
 	"github.com/sourcenetwork/corelog"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
@@ -52,6 +53,7 @@ var configFlags = map[string]string{
 	"url":                "api.address",
 	"max-txn-retries":    "datastore.maxtxnretries",
 	"store":              "datastore.store",
+	"no-encryption":      "datastore.noencryption",
 	"valuelogfilesize":   "datastore.badger.valuelogfilesize",
 	"peers":              "net.peers",
 	"p2paddr":            "net.p2paddresses",
@@ -65,6 +67,7 @@ var configFlags = map[string]string{
 	"no-keyring":         "keyring.disabled",
 	"source-hub-address": "acp.sourceHub.address",
 	"development":        "development",
+	"secret-file":        "secretfile",
 }
 
 // configDefaults contains default values for config entries.
@@ -92,6 +95,7 @@ var configDefaults = map[string]any{
 	"log.output":                        "stderr",
 	"log.source":                        false,
 	"log.stacktrace":                    false,
+	"secretfile":                        ".env",
 }
 
 // defaultConfig returns a new config with default values.
@@ -159,6 +163,12 @@ func loadConfig(rootdir string, flags *pflag.FlagSet) (*viper.Viper, error) {
 		}
 	}
 
+	// load environment variables from .env file if one exists
+	err = godotenv.Load(cfg.GetString("secretfile"))
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return nil, err
+	}
+
 	// set logging config
 	corelog.SetConfig(corelog.Config{
 		Level:            cfg.GetString("log.level"),

cli/errors.go

@@ -16,14 +16,6 @@ import (
 	"github.com/sourcenetwork/defradb/errors"
 )
 
-const errKeyringHelp = `%w
-
-Did you forget to initialize the keyring?
-
-Use the following command to generate the required keys: 
-  defradb keyring generate
-`
-
 const (
 	errInvalidLensConfig        string = "invalid lens configuration"
 	errSchemaVersionNotOfSchema string = "the given schema version is from a different schema"
@@ -41,6 +33,7 @@ var (
 	ErrViewAddMissingArgs         = errors.New("please provide a base query and output SDL for this view")
 	ErrPolicyFileArgCanNotBeEmpty = errors.New("policy file argument can not be empty")
 	ErrPurgeForceFlagRequired     = errors.New("run this command again with --force if you really want to purge all data")
+	ErrMissingKeyringSecret       = errors.New("missing keyring secret")
 )
 
 func NewErrRequiredFlagEmpty(longName string, shortName string) error {
@@ -62,7 +55,3 @@ func NewErrSchemaVersionNotOfSchema(schemaRoot string, schemaVersionID string) e
 		errors.NewKV("SchemaVersionID", schemaVersionID),
 	)
 }
-
-func NewErrKeyringHelp(inner error) error {
-	return fmt.Errorf(errKeyringHelp, inner)
-}

cli/keyring_export.go

@@ -21,6 +21,10 @@ func MakeKeyringExportCommand() *cobra.Command {
 		Long: `Export a private key.
 Prints the hexadecimal representation of a private key.
 
+The DEFRA_KEYRING_SECRET environment variable must be set to unlock the keyring.
+This can also be done with a .env file in the working directory or at a path
+defined with the --keyring-secret-file flag.
+
 Example:
   defradb keyring export encryption-key`,
 		Args: cobra.ExactArgs(1),

cli/keyring_export_test.go

@@ -13,21 +13,20 @@ package cli
 import (
 	"bytes"
 	"encoding/hex"
+	"os"
 	"strings"
 	"testing"
 
 	"github.com/sourcenetwork/defradb/crypto"
 
-	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestKeyringExport(t *testing.T) {
 	rootdir := t.TempDir()
-	readPassword = func(_ *cobra.Command, _ string) ([]byte, error) {
-		return []byte("secret"), nil
-	}
+	err := os.Setenv("DEFRA_KEYRING_SECRET", "password")
+	require.NoError(t, err)
 
 	keyBytes, err := crypto.GenerateAES256()
 	require.NoError(t, err)

cli/keyring_generate.go

@@ -26,13 +26,17 @@ func MakeKeyringGenerateCommand() *cobra.Command {
 Randomly generate and store private keys in the keyring.
 By default peer and encryption keys will be generated.
 
+The DEFRA_KEYRING_SECRET environment variable must be set to unlock the keyring.
+This can also be done with a .env file in the working directory or at a path
+defined with the --keyring-secret-file flag.
+
 WARNING: This will overwrite existing keys in the keyring.
 
 Example:
   defradb keyring generate
 
 Example: with no encryption key
-  defradb keyring generate --no-encryption-key
+  defradb keyring generate --no-encryption
 
 Example: with no peer key
   defradb keyring generate --no-peer-key
@@ -69,7 +73,7 @@ Example: with system keyring
 			return nil
 		},
 	}
-	cmd.Flags().BoolVar(&noEncryptionKey, "no-encryption-key", false,
+	cmd.Flags().BoolVar(&noEncryptionKey, "no-encryption", false,
 		"Skip generating an encryption key. Encryption at rest will be disabled")
 	cmd.Flags().BoolVar(&noPeerKey, "no-peer-key", false,
 		"Skip generating a peer key.")

cli/keyring_generate_test.go

@@ -11,24 +11,23 @@
 package cli
 
 import (
+	"os"
 	"path/filepath"
 	"testing"
 
-	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestKeyringGenerate(t *testing.T) {
 	rootdir := t.TempDir()
-	readPassword = func(_ *cobra.Command, _ string) ([]byte, error) {
-		return []byte("secret"), nil
-	}
+	err := os.Setenv("DEFRA_KEYRING_SECRET", "password")
+	require.NoError(t, err)
 
 	cmd := NewDefraCommand()
 	cmd.SetArgs([]string{"keyring", "generate", "--rootdir", rootdir})
 
-	err := cmd.Execute()
+	err = cmd.Execute()
 	require.NoError(t, err)
 
 	assert.FileExists(t, filepath.Join(rootdir, "keys", encryptionKeyName))
@@ -37,14 +36,13 @@ func TestKeyringGenerate(t *testing.T) {
 
 func TestKeyringGenerateNoEncryptionKey(t *testing.T) {
 	rootdir := t.TempDir()
-	readPassword = func(_ *cobra.Command, _ string) ([]byte, error) {
-		return []byte("secret"), nil
-	}
+	err := os.Setenv("DEFRA_KEYRING_SECRET", "password")
+	require.NoError(t, err)
 
 	cmd := NewDefraCommand()
-	cmd.SetArgs([]string{"keyring", "generate", "--no-encryption-key", "--rootdir", rootdir})
+	cmd.SetArgs([]string{"keyring", "generate", "--no-encryption", "--rootdir", rootdir})
 
-	err := cmd.Execute()
+	err = cmd.Execute()
 	require.NoError(t, err)
 
 	assert.NoFileExists(t, filepath.Join(rootdir, "keys", encryptionKeyName))
@@ -53,14 +51,13 @@ func TestKeyringGenerateNoEncryptionKey(t *testing.T) {
 
 func TestKeyringGenerateNoPeerKey(t *testing.T) {
 	rootdir := t.TempDir()
-	readPassword = func(_ *cobra.Command, _ string) ([]byte, error) {
-		return []byte("secret"), nil
-	}
+	err := os.Setenv("DEFRA_KEYRING_SECRET", "password")
+	require.NoError(t, err)
 
 	cmd := NewDefraCommand()
 	cmd.SetArgs([]string{"keyring", "generate", "--no-peer-key", "--rootdir", rootdir})
 
-	err := cmd.Execute()
+	err = cmd.Execute()
 	require.NoError(t, err)
 
 	assert.FileExists(t, filepath.Join(rootdir, "keys", encryptionKeyName))

cli/keyring_import.go

@@ -23,6 +23,10 @@ func MakeKeyringImportCommand() *cobra.Command {
 		Long: `Import a private key.
 Store an externally generated key in the keyring.
 
+The DEFRA_KEYRING_SECRET environment variable must be set to unlock the keyring.
+This can also be done with a .env file in the working directory or at a path
+defined with the --keyring-secret-file flag.
+
 Example:
   defradb keyring import encryption-key 0000000000000000`,
 		Args: cobra.ExactArgs(2),

cli/keyring_import_test.go

@@ -12,21 +12,20 @@ package cli
 
 import (
 	"encoding/hex"
+	"os"
 	"path/filepath"
 	"testing"
 
 	"github.com/sourcenetwork/defradb/crypto"
 
-	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestKeyringImport(t *testing.T) {
 	rootdir := t.TempDir()
-	readPassword = func(_ *cobra.Command, _ string) ([]byte, error) {
-		return []byte("secret"), nil
-	}
+	err := os.Setenv("DEFRA_KEYRING_SECRET", "password")
+	require.NoError(t, err)
 
 	keyBytes, err := crypto.GenerateAES256()
 	require.NoError(t, err)

cli/root.go

@@ -102,5 +102,9 @@ Start a DefraDB node, interact with a local or remote node, and much more.
 		cfg.GetString(configFlags["source-hub-address"]),
 		"The SourceHub address authorized by the client to make SourceHub transactions on behalf of the actor",
 	)
+	cmd.PersistentFlags().String(
+		"secret-file",
+		cfg.GetString(configFlags["secret-file"]),
+		"Path to the file containing secrets")
 	return cmd
 }

cli/start.go

@@ -18,6 +18,7 @@ import (
 	"github.com/sourcenetwork/immutable"
 	"github.com/spf13/cobra"
 
+	"github.com/sourcenetwork/defradb/crypto"
 	"github.com/sourcenetwork/defradb/errors"
 	"github.com/sourcenetwork/defradb/event"
 	"github.com/sourcenetwork/defradb/http"
@@ -97,17 +98,38 @@ func MakeStartCommand() *cobra.Command {
 			if !cfg.GetBool("keyring.disabled") {
 				kr, err := openKeyring(cmd)
 				if err != nil {
-					return NewErrKeyringHelp(err)
+					return err
 				}
-				// load the required peer key
+				// load the required peer key or generate one if it doesn't exist
 				peerKey, err := kr.Get(peerKeyName)
-				if err != nil {
-					return NewErrKeyringHelp(err)
+				if err != nil && errors.Is(err, keyring.ErrNotFound) {
+					peerKey, err = crypto.GenerateEd25519()
+					if err != nil {
+						return err
+					}
+					err = kr.Set(peerKeyName, peerKey)
+					if err != nil {
+						return err
+					}
+					log.Info("generated peer key")
+				} else if err != nil {
+					return err
 				}
 				opts = append(opts, net.WithPrivateKey(peerKey))
+
 				// load the optional encryption key
 				encryptionKey, err := kr.Get(encryptionKeyName)
-				if err != nil && !errors.Is(err, keyring.ErrNotFound) {
+				if err != nil && errors.Is(err, keyring.ErrNotFound) && !cfg.GetBool("datastore.noencryption") {
+					encryptionKey, err = crypto.GenerateAES256()
+					if err != nil {
+						return err
+					}
+					err = kr.Set(encryptionKeyName, encryptionKey)
+					if err != nil {
+						return err
+					}
+					log.Info("generated encryption key")
+				} else if err != nil && !errors.Is(err, keyring.ErrNotFound) {
 					return err
 				}
 				opts = append(opts, node.WithBadgerEncryptionKey(encryptionKey))
@@ -224,5 +246,9 @@ func MakeStartCommand() *cobra.Command {
 		cfg.GetBool(configFlags["development"]),
 		"Enables a set of features that make development easier but should not be enabled in production",
 	)
+	cmd.Flags().Bool(
+		"no-encryption",
+		cfg.GetBool(configFlags["no-encryption"]),
+		"Skip generating an encryption key. Encryption at rest will be disabled. WARNING: This cannot be undone.")
 	return cmd
 }

cli/utils.go

@@ -16,14 +16,12 @@ import (
 	"encoding/json"
 	"os"
 	"path/filepath"
-	"syscall"
 	"time"
 
 	"github.com/decred/dcrd/dcrec/secp256k1/v4"
 	"github.com/sourcenetwork/immutable"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"golang.org/x/term"
 
 	acpIdentity "github.com/sourcenetwork/defradb/acp/identity"
 	"github.com/sourcenetwork/defradb/client"
@@ -58,14 +56,6 @@ const (
 	authTokenExpiration = time.Minute * 15
 )
 
-// readPassword reads a user input password without echoing it to the terminal.
-var readPassword = func(cmd *cobra.Command, msg string) ([]byte, error) {
-	cmd.Print(msg)
-	pass, err := term.ReadPassword(int(syscall.Stdin))
-	cmd.Println("")
-	return pass, err
-}
-
 // mustGetContextDB returns the db for the current command context.
 //
 // If a db is not set in the current context this function panics.
@@ -214,10 +204,11 @@ func openKeyring(cmd *cobra.Command) (keyring.Keyring, error) {
 	if err := os.MkdirAll(path, 0755); err != nil {
 		return nil, err
 	}
-	prompt := keyring.PromptFunc(func(s string) ([]byte, error) {
-		return readPassword(cmd, s)
-	})
-	return keyring.OpenFileKeyring(path, prompt)
+	secret := []byte(cfg.GetString("keyring.secret"))
+	if len(secret) == 0 {
+		return nil, ErrMissingKeyringSecret
+	}
+	return keyring.OpenFileKeyring(path, secret)
 }
 
 func writeJSON(cmd *cobra.Command, out any) error {

docs/config.md

@@ -21,6 +21,10 @@ The number of retries to make in the event of a transaction conflict. Defaults t
 
 Currently this is only used within the P2P system and will not affect operations initiated by users.
 
+## `datastore.noencryption`
+
+Skip generating an encryption key. Encryption at rest will be disabled. **WARNING**: This cannot be undone.
+
 ## `datastore.badger.path`
 
 The path to the database data file(s). Defaults to `data`.
@@ -156,3 +160,7 @@ transactions created by the node is stored. Required when using `acp.type`:`sour
 The SourceHub address of the actor that client-side actions should permit to make SourceHub actions on
 their behalf.  This is a client-side only config param.  It is required if the client wishes to make
 SourceHub ACP requests in order to create protected data.
+
+## `secretfile`
+
+Path to the file containing secrets. Defaults to `.env`.