diff --git a/commands/cluster_command_launcher.go b/commands/cluster_command_launcher.go index 101cd76..6fdc2eb 100644 --- a/commands/cluster_command_launcher.go +++ b/commands/cluster_command_launcher.go @@ -59,6 +59,8 @@ const ( eonModeKey = "eonMode" configParamFlag = "config-param" configParamKey = "configParam" + configParamFileFlag = "config-param-file" + configParamFileKey = "configParamFile" logPathFlag = "log-path" logPathKey = "logPath" keyFileFlag = "key-file" @@ -80,10 +82,15 @@ const ( subclusterFlag = "subcluster" addNodeFlag = "new-hosts" sandboxFlag = "sandbox" + saveRpFlag = "save-restore-point" + isolateMetadataFlag = "isolate-metadata" + createStorageLocationsFlag = "create-storage-locations" sandboxKey = "sandbox" connFlag = "conn" connKey = "conn" stopNodeFlag = "stop-hosts" + reIPFileFlag = "re-ip-file" + removeNodeFlag = "remove" // VER-90436: restart -> start startNodeFlag = "restart" startHostFlag = "start-hosts" @@ -542,13 +549,31 @@ func hideLocalFlags(cmd *cobra.Command, flags []string) { } } -// markFlagsRequired will mark local flags as required -func markFlagsRequired(cmd *cobra.Command, flags []string) { +// markFlagsRequired marks given flags as required +func markFlagsRequired(cmd *cobra.Command, flags ...string) { for _, flag := range flags { err := cmd.MarkFlagRequired(flag) if err != nil { fmt.Printf("Warning: fail to mark flag %q required, details: %v\n", flag, err) } + + // emphasize [Required] in the help message + f := cmd.Flags().Lookup(flag) + if f != nil { // empty flag means not found + f.Usage = "[Required] " + f.Usage + } + } +} + +// markFlagsOneRequired marks one of the given flags as required +func markFlagsOneRequired(cmd *cobra.Command, flags []string) { + cmd.MarkFlagsOneRequired(flags...) + for _, flag := range flags { + f := cmd.Flags().Lookup(flag) + if f != nil { // empty flag means not found + oneRequiredGroup := f.Annotations["cobra_annotation_one_required"] + f.Usage = fmt.Sprintf("(One of %v is required) ", oneRequiredGroup) + f.Usage + } } } diff --git a/commands/cmd_add_node.go b/commands/cmd_add_node.go index 8c3bfb3..5f97caa 100644 --- a/commands/cmd_add_node.go +++ b/commands/cmd_add_node.go @@ -48,7 +48,7 @@ func makeCmdAddNode() *cobra.Command { newCmd, addNodeSubCmd, "Add host(s) to an existing database", - `This subcommand adds one or more hosts to an existing database. + `This command adds one or more hosts to an existing database. You must provide the --new-hosts option followed by one or more hosts to add as a comma-separated list. @@ -57,7 +57,7 @@ You cannot add hosts to a sandbox subcluster in an Eon Mode database. Use the --node-names option to address issues resulting from a failed node addition attempt. It's crucial to include all expected nodes in the catalog -when using this option. This subcommand removes any surplus nodes from the +when using this option. This command removes any surplus nodes from the catalog, provided they are down, before commencing the node addition process. Omitting the option will skip this node trimming process. @@ -79,7 +79,7 @@ Examples: newCmd.setLocalFlags(cmd) // require hosts to add - markFlagsRequired(cmd, []string{addNodeFlag}) + markFlagsRequired(cmd, addNodeFlag) return cmd } @@ -199,7 +199,7 @@ func (c *CmdAddNode) Run(vcc vclusterops.ClusterCommands) error { } // write db info to vcluster config file - err := writeConfig(&vdb) + err := writeConfig(&vdb, true /*forceOverwrite*/) if err != nil { vcc.PrintWarning("fail to write config file, details: %s", err) } diff --git a/commands/cmd_add_subcluster.go b/commands/cmd_add_subcluster.go index e86dce0..5328bf2 100644 --- a/commands/cmd_add_subcluster.go +++ b/commands/cmd_add_subcluster.go @@ -49,7 +49,7 @@ func makeCmdAddSubcluster() *cobra.Command { newCmd, addSCSubCmd, "Add a subcluster", - `This subcommand adds a new subcluster to an existing Eon Mode database. + `This command adds a new subcluster to an existing Eon Mode database. You must provide a subcluster name with the --subcluster option. @@ -89,7 +89,7 @@ Examples: newCmd.setHiddenFlags(cmd) // require name of subcluster to add - markFlagsRequired(cmd, []string{subclusterFlag}) + markFlagsRequired(cmd, subclusterFlag) // hide eon mode flag since we expect it to come from config file, not from user input hideLocalFlags(cmd, []string{eonModeFlag}) @@ -222,7 +222,7 @@ func (c *CmdAddSubcluster) Run(vcc vclusterops.ClusterCommands) error { return err } // update db info in the config file - err = writeConfig(&vdb) + err = writeConfig(&vdb, true /*forceOverwrite*/) if err != nil { vcc.PrintWarning("fail to write config file, details: %s", err) } diff --git a/commands/cmd_base.go b/commands/cmd_base.go index 4032908..ea1d6f9 100644 --- a/commands/cmd_base.go +++ b/commands/cmd_base.go @@ -16,8 +16,10 @@ package commands import ( + "encoding/json" "fmt" "os" + "path/filepath" "strings" "github.com/spf13/cobra" @@ -28,7 +30,9 @@ import ( ) const ( - outputFilePerm = 0644 + filePerm = 0644 + configDirPerm = 0755 + defConfigParamFileName = "config_param.json" ) /* CmdBase @@ -43,6 +47,7 @@ type CmdBase struct { // to a file instead of being displayed in stdout. This is the file the output will // be written to output string + configParamFile string passwordFile string readPasswordFromPrompt bool } @@ -70,7 +75,7 @@ func (c *CmdBase) setCommonFlags(cmd *cobra.Command, flags []string) { if len(flags) == 0 { return } - setConfigFlags(cmd, flags) + c.setConfigFlags(cmd, flags) if util.StringInArray(passwordFlag, flags) { c.setPasswordFlags(cmd) } @@ -132,7 +137,7 @@ func (c *CmdBase) setCommonFlags(cmd *cobra.Command, flags []string) { // setConfigFlags sets the config flag as well as all the common flags that // can also be set with values from the config file -func setConfigFlags(cmd *cobra.Command, flags []string) { +func (c *CmdBase) setConfigFlags(cmd *cobra.Command, flags []string) { if util.StringInArray(dbNameFlag, flags) { cmd.Flags().StringVarP( &dbOptions.DBName, @@ -209,7 +214,122 @@ func setConfigFlags(cmd *cobra.Command, flags []string) { configParamFlag, map[string]string{}, "Comma-separated list of NAME=VALUE pairs of existing configuration parameters") + cmd.Flags().StringVar( + &c.configParamFile, + configParamFileFlag, + "", + "Path to config parameter file") + } +} + +func (c *CmdBase) initConfigParam() error { + // We need to find the path to the config param. The order of precedence is as follows: + // 1. Option + // 2. Default locations + // a. /opt/vertica/config/config_param.json if running vcluster in /opt/vertica/bin + // b. $HOME/.config/vcluster/config_param.json otherwise + // + // If none of these things are true, then we run the cli without a config param file. + + if c.configParamFile != "" { + return nil + } + + // Pick a default config param file + + // If we are running vcluster from /opt/vertica/bin, we'll assume we + // have installed the vertica package on this machine and so can assume + // /opt/vertica/config exists too. + vclusterExePath, err := os.Executable() + if err != nil { + return err + } + if vclusterExePath == defaultExecutablePath { + if util.CheckPathExist(rpmConfDir) { + c.configParamFile = fmt.Sprintf("%s/%s", rpmConfDir, defConfigParamFileName) + return nil + } } + // Finally default to the .config directory in the users home. This is used + // by many CLI applications. + cfgDir, err := os.UserConfigDir() + if err != nil { + return err + } + // Ensure the config directory exists. + path := filepath.Join(cfgDir, "vcluster") + err = os.MkdirAll(path, configDirPerm) + if err != nil { + // Just abort if we don't have write access to the config path + return err + } + c.configParamFile = fmt.Sprintf("%s/%s", path, defConfigParamFileName) + return nil +} + +// setConfigParam sets the configuration parameters from config param file +func (c *CmdBase) setConfigParam(opt *vclusterops.DatabaseOptions) error { + err := c.initConfigParam() + if err != nil { + return err + } + + if c.configParamFile == "" { + return nil + } + configParam, err := c.getConfigParamFromFile(c.configParamFile) + if err != nil { + return err + } + for name, val := range configParam { + // allow users to overwrite params in file with --config-param + if _, ok := opt.ConfigurationParameters[name]; ok { + continue + } + opt.ConfigurationParameters[name] = val + } + return nil +} + +func (c *CmdBase) writeConfigParam(configParam map[string]string, forceOverwrite bool) error { + if !c.parser.Changed(configParamFlag) { + // no new config param specified, no need to write + return nil + } + if c.configParamFile == "" { + return fmt.Errorf("config param file path is empty") + } + if util.CheckPathExist(c.configParamFile) && !forceOverwrite { + return fmt.Errorf("file %s exist, consider using --force-overwrite-file to overwrite the file", c.configParamFile) + } + configParamBytes, err := json.Marshal(&configParam) + if err != nil { + return fmt.Errorf("fail to marshal configuration parameters, details: %w", err) + } + err = os.WriteFile(c.configParamFile, configParamBytes, filePerm) + if err != nil { + return fmt.Errorf("fail to write configuration parameters file, details: %w", err) + } + return nil +} + +func (c *CmdBase) getConfigParamFromFile(configParamFile string) (map[string]string, error) { + if !util.CheckPathExist(configParamFile) { + return nil, nil + } + // Read config param from file + configParamBytes, err := os.ReadFile(configParamFile) + if err != nil { + return nil, fmt.Errorf("error reading config param from file %q: %w", configParamFile, err) + } + + var configParam map[string]string + err = json.Unmarshal(configParamBytes, &configParam) + if err != nil { + return nil, fmt.Errorf("error reading config param from file %q: %w", configParamFile, err) + } + + return configParam, nil } // setPasswordFlags sets all the password flags @@ -334,7 +454,7 @@ func (c *CmdBase) initCmdOutputFile() (*os.File, error) { if c.output == "" { return nil, fmt.Errorf("output-file cannot be empty") } - return os.OpenFile(c.output, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, outputFilePerm) + return os.OpenFile(c.output, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, filePerm) } // getCertFilesFromPaths will update cert and key file from cert path options diff --git a/commands/cmd_config_recover.go b/commands/cmd_config_recover.go index c24e3be..74ea2bb 100644 --- a/commands/cmd_config_recover.go +++ b/commands/cmd_config_recover.go @@ -70,7 +70,7 @@ Examples: ) // require db-name, hosts, catalog-path, and data-path - markFlagsRequired(cmd, []string{dbNameFlag, hostsFlag, catalogPathFlag}) + markFlagsRequired(cmd, dbNameFlag, hostsFlag, catalogPathFlag) // local flags newCmd.setLocalFlags(cmd) @@ -120,7 +120,7 @@ func (c *CmdConfigRecover) Run(vcc vclusterops.ClusterCommands) error { } // write db info to vcluster config file vdb.FirstStartAfterRevive = c.recoverConfigOptions.AfterRevive - err = writeConfig(&vdb) + err = writeConfig(&vdb, true /*forceOverwrite*/) if err != nil { return fmt.Errorf("fail to write config file, details: %s", err) } diff --git a/commands/cmd_create_connection.go b/commands/cmd_create_connection.go index 23c3ab9..cb8f116 100644 --- a/commands/cmd_create_connection.go +++ b/commands/cmd_create_connection.go @@ -42,7 +42,7 @@ func makeCmdCreateConnection() *cobra.Command { newCmd, createConnectionSubCmd, "create the content of the connection file", - `This subcommand is used to create the content of the connection file. + `This command is used to create the content of the connection file. You must specify the database name and host list. If the database has a password, you need to provide password. If the database uses @@ -59,7 +59,7 @@ Examples: // local flags newCmd.setLocalFlags(cmd) - markFlagsRequired(cmd, []string{dbNameFlag, hostsFlag, connFlag}) + markFlagsRequired(cmd, dbNameFlag, hostsFlag, connFlag) return cmd } diff --git a/commands/cmd_create_db.go b/commands/cmd_create_db.go index c9e3dcd..6bf69bf 100644 --- a/commands/cmd_create_db.go +++ b/commands/cmd_create_db.go @@ -46,7 +46,7 @@ func makeCmdCreateDB() *cobra.Command { newCmd, createDBSubCmd, "Create a database", - `This subcommand creates a database on a set of hosts. + `This command creates a database on a set of hosts. You must specify the database name, host list, catalog path, and data path. @@ -109,11 +109,7 @@ Examples: newCmd.setHiddenFlags(cmd) // require db-name - markFlagsRequired(cmd, []string{dbNameFlag}) - - // VER-92676: we may add a function setRequiredFlags() in cmd_base.go - // in which we can set catalog-path, data-path, ... - // then call that function in cmd_create_db and cmd_revive_db + markFlagsRequired(cmd, dbNameFlag, hostsFlag, catalogPathFlag, dataPathFlag) return cmd } @@ -199,6 +195,12 @@ func (c *CmdCreateDB) setLocalFlags(cmd *cobra.Command) { false, "Force removal of existing directories before creating the database", ) + cmd.Flags().BoolVar( + &c.createDBOptions.ForceOverwriteFile, + "force-overwrite-file", + false, + "Force overwrite of existing config and config param files", + ) cmd.Flags().BoolVar( &c.createDBOptions.SkipPackageInstall, "skip-package-install", @@ -257,7 +259,16 @@ func (c *CmdCreateDB) validateParse(logger vlog.Printer) error { return err } - return c.setDBPassword(&c.createDBOptions.DatabaseOptions) + err = c.setDBPassword(&c.createDBOptions.DatabaseOptions) + if err != nil { + return err + } + + err = c.initConfigParam() + if err != nil { + return err + } + return nil } func (c *CmdCreateDB) Run(vcc vclusterops.ClusterCommands) error { @@ -268,10 +279,15 @@ func (c *CmdCreateDB) Run(vcc vclusterops.ClusterCommands) error { } // write db info to vcluster config file - err := writeConfig(&vdb) + err := writeConfig(&vdb, c.createDBOptions.ForceOverwriteFile) if err != nil { fmt.Printf("Warning: Fail to write config file, details: %s\n", err) } + // write config parameters to vcluster config param file + err = c.writeConfigParam(c.createDBOptions.ConfigurationParameters, c.createDBOptions.ForceOverwriteFile) + if err != nil { + vcc.PrintWarning("fail to write config param file, details: %s", err) + } vcc.PrintInfo("Created a database with name [%s]", vdb.Name) return nil } diff --git a/commands/cmd_drop_db.go b/commands/cmd_drop_db.go index baebe59..7c21b53 100644 --- a/commands/cmd_drop_db.go +++ b/commands/cmd_drop_db.go @@ -41,7 +41,7 @@ func makeCmdDropDB() *cobra.Command { newCmd, dropDBSubCmd, "Drop a database", - `This subcommand drops a stopped database. + `This command drops a stopped database. For an Eon database, communal storage is not deleted. You can recover the dropped database with revive_db. diff --git a/commands/cmd_install_packages.go b/commands/cmd_install_packages.go index fdd4d8f..8d24947 100644 --- a/commands/cmd_install_packages.go +++ b/commands/cmd_install_packages.go @@ -46,7 +46,7 @@ func makeCmdInstallPackages() *cobra.Command { newCmd, installPkgSubCmd, "Install default package(s) in database", - `This subcommand installs default packages in the database. + `This command installs default packages in the database. You must provide the --hosts option followed by all hosts in the database as a comma-separated list. diff --git a/commands/cmd_list_all_nodes.go b/commands/cmd_list_all_nodes.go index d28c0b4..e8494f8 100644 --- a/commands/cmd_list_all_nodes.go +++ b/commands/cmd_list_all_nodes.go @@ -44,7 +44,7 @@ func makeListAllNodes() *cobra.Command { newCmd, listAllNodesSubCmd, "List all nodes in the database", - `This subcommand queries the status of the nodes in the database and prints + `This command queries the status of the nodes in the database and prints whether they are up or down. To provide its status, each host must run the spread daemon. diff --git a/commands/cmd_manage_config.go b/commands/cmd_manage_config.go index 6e0c599..3f63494 100644 --- a/commands/cmd_manage_config.go +++ b/commands/cmd_manage_config.go @@ -31,7 +31,7 @@ func makeCmdManageConfig() *cobra.Command { cmd := makeSimpleCobraCmd( manageConfigSubCmd, "Display or recover the contents of the config file", - `This subcommand displays or recovers the contents of the config file.`) + `This command displays or recovers the contents of the config file.`) cmd.AddCommand(makeCmdConfigShow()) cmd.AddCommand(makeCmdConfigRecover()) diff --git a/commands/cmd_re_ip.go b/commands/cmd_re_ip.go index 0605ada..385e4ce 100644 --- a/commands/cmd_re_ip.go +++ b/commands/cmd_re_ip.go @@ -43,7 +43,7 @@ func makeCmdReIP() *cobra.Command { newCmd, reIPSubCmd, "Re-ip database nodes", - `This subcommand changes the IP addresses of database nodes in the catalog. + `This command changes the IP addresses of database nodes in the catalog. The database must be down to change the IP addresses with re_ip. If the database is up, you must run restart_node after re_ip for the @@ -74,8 +74,8 @@ Examples: newCmd.setLocalFlags(cmd) // require re-ip-file - markFlagsRequired(cmd, []string{"re-ip-file"}) - markFlagsFileName(cmd, map[string][]string{"re-ip-file": {"json"}}) + markFlagsRequired(cmd, reIPFileFlag) + markFlagsFileName(cmd, map[string][]string{reIPFileFlag: {"json"}}) return cmd } @@ -84,7 +84,7 @@ Examples: func (c *CmdReIP) setLocalFlags(cmd *cobra.Command) { cmd.Flags().StringVar( &c.reIPFilePath, - "re-ip-file", + reIPFileFlag, "", "Path of the re-ip file", ) @@ -111,6 +111,11 @@ func (c *CmdReIP) validateParse(logger vlog.Printer) error { if err != nil { return err } + + err = c.setConfigParam(&c.reIPOptions.DatabaseOptions) + if err != nil { + return err + } return c.reIPOptions.ReadReIPFile(c.reIPFilePath) } @@ -138,12 +143,18 @@ func (c *CmdReIP) Run(vcc vclusterops.ClusterCommands) error { // update config file after running re_ip if canUpdateConfig { c.UpdateConfig(dbConfig) - err = dbConfig.write(options.ConfigPath) + err = dbConfig.write(options.ConfigPath, true /*forceOverwrite*/) if err != nil { fmt.Printf("Warning: fail to update config file, details %v\n", err) } } + // write config parameters to vcluster config param file + err = c.writeConfigParam(options.ConfigurationParameters, true /*forceOverwrite*/) + if err != nil { + vcc.PrintWarning("fail to write config param file, details: %s", err) + } + return nil } diff --git a/commands/cmd_remove_node.go b/commands/cmd_remove_node.go index fc8640b..2abd473 100644 --- a/commands/cmd_remove_node.go +++ b/commands/cmd_remove_node.go @@ -44,7 +44,7 @@ func makeCmdRemoveNode() *cobra.Command { newCmd, removeNodeSubCmd, "Remove host(s) from an existing database", - `This subcommand removes one or more nodes from an existing database. + `This command removes one or more nodes from an existing database. You must provide the --remove option followed by one or more hosts to remove as a comma-separated list. @@ -68,7 +68,7 @@ Examples: newCmd.setLocalFlags(cmd) // require hosts to remove - markFlagsRequired(cmd, []string{"remove"}) + markFlagsRequired(cmd, removeNodeFlag) return cmd } @@ -77,7 +77,7 @@ Examples: func (c *CmdRemoveNode) setLocalFlags(cmd *cobra.Command) { cmd.Flags().StringSliceVar( &c.removeNodeOptions.HostsToRemove, - "remove", + removeNodeFlag, []string{}, "Comma-separated list of host(s) to remove from the database", ) @@ -144,7 +144,7 @@ func (c *CmdRemoveNode) Run(vcc vclusterops.ClusterCommands) error { } // write db info to vcluster config file - err = writeConfig(&vdb) + err = writeConfig(&vdb, true /*forceOverwrite*/) if err != nil { vcc.PrintWarning("fail to write config file, details: %s", err) } diff --git a/commands/cmd_remove_subcluster.go b/commands/cmd_remove_subcluster.go index ee1e8e3..68b4c75 100644 --- a/commands/cmd_remove_subcluster.go +++ b/commands/cmd_remove_subcluster.go @@ -42,7 +42,7 @@ func makeCmdRemoveSubcluster() *cobra.Command { newCmd, removeSCSubCmd, "Remove a subcluster", - `This subcommand removes a subcluster from an existing Eon Mode database. + `This command removes a subcluster from an existing Eon Mode database. You must provide the subcluster name with the --subcluster option. @@ -66,7 +66,7 @@ Examples: newCmd.setLocalFlags(cmd) // require name of subcluster to remove - markFlagsRequired(cmd, []string{subclusterFlag}) + markFlagsRequired(cmd, subclusterFlag) // hide eon mode flag since we expect it to come from config file, not from user input hideLocalFlags(cmd, []string{eonModeFlag}) @@ -134,7 +134,7 @@ func (c *CmdRemoveSubcluster) Run(vcc vclusterops.ClusterCommands) error { } // write db info to vcluster config file - err = writeConfig(&vdb) + err = writeConfig(&vdb, true /*forceOverwrite*/) if err != nil { vcc.PrintWarning("fail to write config file, details: %s", err) } diff --git a/commands/cmd_replication.go b/commands/cmd_replication.go index 06cef2a..c14cf3e 100644 --- a/commands/cmd_replication.go +++ b/commands/cmd_replication.go @@ -23,7 +23,7 @@ func makeCmdReplication() *cobra.Command { cmd := makeSimpleCobraCmd( replicationSubCmd, "Handle database replication", - `This subcommand starts database replication or displays the status of an + `This command starts database replication or displays the status of an in-progress replication operation.`) cmd.AddCommand(makeCmdStartReplication()) diff --git a/commands/cmd_restart_node.go b/commands/cmd_restart_node.go index d1fad4a..ca44185 100644 --- a/commands/cmd_restart_node.go +++ b/commands/cmd_restart_node.go @@ -49,7 +49,7 @@ func makeCmdRestartNodes() *cobra.Command { newCmd, restartNodeSubCmd, "Restart nodes in the database", - `This subcommand starts individual nodes in a running cluster. This + `This command starts individual nodes in a running cluster. This differs from start_db, which starts Vertica after cluster quorum is lost. You can pass --restart a comma-separated list of NODE_NAME=IP_TO_RESTART pairs @@ -82,7 +82,7 @@ Examples: newCmd.setLocalFlags(cmd) // require nodes or hosts to restart - cmd.MarkFlagsOneRequired([]string{startNodeFlag, startHostFlag}...) + markFlagsOneRequired(cmd, []string{startNodeFlag, startHostFlag}) return cmd } diff --git a/commands/cmd_revive_db.go b/commands/cmd_revive_db.go index 631c700..4e21489 100644 --- a/commands/cmd_revive_db.go +++ b/commands/cmd_revive_db.go @@ -43,7 +43,7 @@ func makeCmdReviveDB() *cobra.Command { newCmd, reviveDBSubCmd, "Revive a database", - `This subcommand revives an Eon Mode database on the specified hosts or restores + `This command revives an Eon Mode database on the specified hosts or restores an Eon Mode database to the specified restore point. The --communal-storage-location option is required. If access to communal @@ -86,7 +86,7 @@ Examples: newCmd.setLocalFlags(cmd) // require db-name and communal-storage-location - markFlagsRequired(cmd, []string{dbNameFlag, communalStorageLocationFlag}) + markFlagsRequired(cmd, dbNameFlag, communalStorageLocationFlag) return cmd } @@ -162,7 +162,16 @@ func (c *CmdReviveDB) validateParse(logger vlog.Printer) error { return nil } - return c.ValidateParseBaseOptions(&c.reviveDBOptions.DatabaseOptions) + err = c.ValidateParseBaseOptions(&c.reviveDBOptions.DatabaseOptions) + if err != nil { + return err + } + + err = c.setConfigParam(&c.reviveDBOptions.DatabaseOptions) + if err != nil { + return err + } + return nil } func (c *CmdReviveDB) Run(vcc vclusterops.ClusterCommands) error { @@ -181,11 +190,17 @@ func (c *CmdReviveDB) Run(vcc vclusterops.ClusterCommands) error { // write db info to vcluster config file vdb.FirstStartAfterRevive = true - err = writeConfig(vdb) + err = writeConfig(vdb, true /*forceOverwrite*/) if err != nil { vcc.PrintWarning("fail to write config file, details: %s", err) } + // write config parameters to vcluster config param file + err = c.writeConfigParam(c.reviveDBOptions.ConfigurationParameters, true /*forceOverwrite*/) + if err != nil { + vcc.PrintWarning("fail to write config param file, details: %s", err) + } + vcc.PrintInfo("Successfully revived database %s", c.reviveDBOptions.DBName) return nil diff --git a/commands/cmd_sandbox.go b/commands/cmd_sandbox.go index f5a1f37..76426cf 100644 --- a/commands/cmd_sandbox.go +++ b/commands/cmd_sandbox.go @@ -49,7 +49,7 @@ func makeCmdSandboxSubcluster() *cobra.Command { newCmd, sandboxSubCmd, "Sandbox a subcluster", - `This subcommand sandboxes a subcluster in an existing Eon Mode database. + `This command sandboxes a subcluster in an existing Eon Mode database. Only secondary subclusters can be sandboxed. All hosts in the subcluster that you want to sandbox must be up. @@ -76,7 +76,7 @@ Examples: newCmd.setLocalFlags(cmd) // require name of subcluster to sandbox as well as the sandbox name - markFlagsRequired(cmd, []string{subclusterFlag, sandboxFlag}) + markFlagsRequired(cmd, subclusterFlag, sandboxFlag) return cmd } @@ -95,6 +95,24 @@ func (c *CmdSandboxSubcluster) setLocalFlags(cmd *cobra.Command) { "", "The name of the sandbox", ) + cmd.Flags().BoolVar( + &c.sbOptions.SaveRp, + saveRpFlag, + false, + "A restore point is saved when creating the sandbox", + ) + cmd.Flags().BoolVar( + &c.sbOptions.Imeta, + isolateMetadataFlag, + false, + "The metadata of sandboxed subcluster is isolated", + ) + cmd.Flags().BoolVar( + &c.sbOptions.Sls, + createStorageLocationsFlag, + false, + "The sandbox create its own storage locations", + ) } func (c *CmdSandboxSubcluster) Parse(inputArgv []string, logger vlog.Printer) error { @@ -150,7 +168,7 @@ func (c *CmdSandboxSubcluster) Run(vcc vclusterops.ClusterCommands) error { return nil } - writeErr := dbConfig.write(options.ConfigPath) + writeErr := dbConfig.write(options.ConfigPath, true /*forceOverwrite*/) if writeErr != nil { vcc.PrintWarning("fail to write the config file, details: " + writeErr.Error()) return nil diff --git a/commands/cmd_scrutinize.go b/commands/cmd_scrutinize.go index 8bae24a..f50bd57 100644 --- a/commands/cmd_scrutinize.go +++ b/commands/cmd_scrutinize.go @@ -86,7 +86,7 @@ func makeCmdScrutinize() *cobra.Command { newCmd, scrutinizeSubCmd, "Scrutinize a database", - `This subcommand runs scrutinize to collect diagnostic information about a + `This command runs scrutinize to collect diagnostic information about a database. Vertica support might request that you run scrutinize when resolving a support diff --git a/commands/cmd_show_restore_points.go b/commands/cmd_show_restore_points.go index 919059c..aab138c 100644 --- a/commands/cmd_show_restore_points.go +++ b/commands/cmd_show_restore_points.go @@ -42,7 +42,7 @@ func makeCmdShowRestorePoints() *cobra.Command { newCmd, showRestorePointsSubCmd, "Query and list restore point(s) in archive(s)", - `This subcommand queries and displays restore points in archives. + `This command queries and displays restore points in archives. The --start-timestamp and --end-timestamp options limit the restore points query by creation timestamp. Both options accept UTC timestamps in date-time @@ -147,7 +147,16 @@ func (c *CmdShowRestorePoints) validateParse(logger vlog.Printer) error { if err != nil { return err } - return c.setDBPassword(&c.showRestorePointsOptions.DatabaseOptions) + + err = c.setDBPassword(&c.showRestorePointsOptions.DatabaseOptions) + if err != nil { + return err + } + err = c.setConfigParam(&c.showRestorePointsOptions.DatabaseOptions) + if err != nil { + return err + } + return nil } func (c *CmdShowRestorePoints) Analyze(logger vlog.Printer) error { diff --git a/commands/cmd_start_db.go b/commands/cmd_start_db.go index f14ab21..54780ea 100644 --- a/commands/cmd_start_db.go +++ b/commands/cmd_start_db.go @@ -16,6 +16,8 @@ package commands import ( + "fmt" + "github.com/spf13/cobra" "github.com/vertica/vcluster/vclusterops" "github.com/vertica/vcluster/vclusterops/util" @@ -47,9 +49,9 @@ func makeCmdStartDB() *cobra.Command { newCmd, startDBSubCmd, "Start a database", - `This subcommand starts a database on a set of hosts. + `This command starts a database on a set of hosts. -Starts Vertica on each host and establishes cluster quorum. This subcommand is +Starts Vertica on each host and establishes cluster quorum. This command is similar to restart_node, except start_db assumes that cluster quorum has been lost. @@ -65,6 +67,12 @@ Examples: # Start a database with config file using password authentication vcluster start_db --password testpassword \ --config /opt/vertica/config/vertica_cluster.yaml + # Start a database partially with config file on a sandbox + vcluster start_db --password testpassword \ + --config /home/dbadmin/vertica_cluster.yaml --sandbox "sand" + # Start a database partially with config file on a sandbox + vcluster start_db --password testpassword \ + --config /home/dbadmin/vertica_cluster.yaml --main-cluster-only `, []string{dbNameFlag, hostsFlag, communalStorageLocationFlag, ipv6Flag, configFlag, catalogPathFlag, passwordFlag, eonModeFlag, configParamFlag}, @@ -90,6 +98,19 @@ func (c *CmdStartDB) setLocalFlags(cmd *cobra.Command) { ) // Update description of hosts flag locally for a detailed hint cmd.Flags().Lookup(hostsFlag).Usage = "Comma-separated list of hosts in database. This is used to start sandboxed hosts" + + cmd.Flags().StringVar( + &c.startDBOptions.Sandbox, + sandboxFlag, + "", + "Name of the sandbox to start", + ) + cmd.Flags().BoolVar( + &c.startDBOptions.MainCluster, + "main-cluster-only", + false, + "Start the database on main cluster, but don't start any of the sandboxes", + ) } // setHiddenFlags will set the hidden flags the command has. @@ -160,18 +181,53 @@ func (c *CmdStartDB) validateParse(logger vlog.Printer) error { if err != nil { return err } - return c.setDBPassword(&c.startDBOptions.DatabaseOptions) -} + err = c.setDBPassword(&c.startDBOptions.DatabaseOptions) + if err != nil { + return err + } + + err = c.setConfigParam(&c.startDBOptions.DatabaseOptions) + if err != nil { + return err + } + return nil +} +func filterInputHosts(options *vclusterops.VStartDatabaseOptions, dbConfig *DatabaseConfig) []string { + filteredHosts := []string{} + for _, n := range dbConfig.Nodes { + // Collect sandbox hosts + if options.Sandbox == n.Sandbox && n.Sandbox != util.MainClusterSandbox { + filteredHosts = append(filteredHosts, n.Address) + } + // Collect main cluster hosts + if options.MainCluster && n.Sandbox == util.MainClusterSandbox { + filteredHosts = append(filteredHosts, n.Address) + } + } + if len(options.Hosts) > 0 { + return util.SliceCommon(filteredHosts, options.Hosts) + } + return filteredHosts +} func (c *CmdStartDB) Run(vcc vclusterops.ClusterCommands) error { vcc.V(1).Info("Called method Run()") options := c.startDBOptions + if options.Sandbox != "" && options.MainCluster { + return fmt.Errorf("cannot use both --sandbox and --main-cluster-only options together ") + } dbConfig, readConfigErr := readConfig() if readConfigErr == nil { + if options.Sandbox != util.MainClusterSandbox || options.MainCluster { + options.RawHosts = filterInputHosts(options, dbConfig) + } options.FirstStartAfterRevive = dbConfig.FirstStartAfterRevive } else { vcc.PrintWarning("fail to read config file", "error", readConfigErr) + if options.MainCluster || options.Sandbox != util.MainClusterSandbox { + return fmt.Errorf("cannot start the database partially without config file") + } } vdb, err := vcc.VStartDatabase(options) @@ -179,19 +235,35 @@ func (c *CmdStartDB) Run(vcc vclusterops.ClusterCommands) error { vcc.LogError(err, "failed to start the database") return err } - - vcc.PrintInfo("Successfully start the database %s", options.DBName) + msg := fmt.Sprintf("Started database %s", options.DBName) + if options.Sandbox != "" { + sandboxMsg := fmt.Sprintf(" on sandbox %s", options.Sandbox) + vcc.PrintInfo(msg + sandboxMsg) + return nil + } + if options.MainCluster { + startMsg := " on the main cluster" + vcc.PrintInfo(msg + startMsg) + return nil + } + vcc.PrintInfo(msg) // for Eon database, update config file to fill nodes' subcluster information if readConfigErr == nil && options.IsEon { // write db info to vcluster config file vdb.FirstStartAfterRevive = false - err := writeConfig(vdb) + err = writeConfig(vdb, true /*forceOverwrite*/) if err != nil { vcc.PrintWarning("fail to update config file, details: %s", err) } } + // write config parameters to vcluster config param file + err = c.writeConfigParam(options.ConfigurationParameters, true /*forceOverwrite*/) + if err != nil { + vcc.PrintWarning("fail to write config param file, details: %s", err) + } + return nil } diff --git a/commands/cmd_start_replication.go b/commands/cmd_start_replication.go index 5cf63d0..05e6e4d 100644 --- a/commands/cmd_start_replication.go +++ b/commands/cmd_start_replication.go @@ -94,7 +94,7 @@ Examples: // local flags newCmd.setLocalFlags(cmd) - // either target dbname/hosts or connection file must be provided + // either target dbname+hosts or connection file must be provided cmd.MarkFlagsOneRequired(targetConnFlag, targetDBNameFlag) cmd.MarkFlagsOneRequired(targetConnFlag, targetHostsFlag) @@ -139,7 +139,11 @@ func (c *CmdStartReplication) setLocalFlags(cmd *cobra.Command) { &globals.connFile, targetConnFlag, "", - "Path to the connection file") + "[Required] The connection file created with the create_connection command, "+ + "containing the database name, hosts, and password (if any) for the target database. "+ + "Alternatively, you can provide this information manually with --target-db-name, "+ + "--target-hosts, and --target-password-file", + ) markFlagsFileName(cmd, map[string][]string{targetConnFlag: {"yaml"}}) // password flags cmd.Flags().StringVar( diff --git a/commands/cmd_start_subcluster.go b/commands/cmd_start_subcluster.go index 514ab5c..bd64823 100644 --- a/commands/cmd_start_subcluster.go +++ b/commands/cmd_start_subcluster.go @@ -43,7 +43,7 @@ func makeCmdStartSubcluster() *cobra.Command { newCmd, startSCSubCmd, "Start a subcluster", - `This subcommand starts a stopped subcluster in a running Eon database. + `This command starts a stopped subcluster in a running Eon database. You must provide the subcluster name with the --subcluster option. @@ -63,7 +63,7 @@ Examples: newCmd.setLocalFlags(cmd) // require name of subcluster to start - markFlagsRequired(cmd, []string{subclusterFlag}) + markFlagsRequired(cmd, subclusterFlag) // hide eon mode flag since we expect it to come from config file, not from user input hideLocalFlags(cmd, []string{eonModeFlag}) diff --git a/commands/cmd_stop_db.go b/commands/cmd_stop_db.go index 28a03a2..926a85a 100644 --- a/commands/cmd_stop_db.go +++ b/commands/cmd_stop_db.go @@ -48,7 +48,7 @@ func makeCmdStopDB() *cobra.Command { newCmd, stopDBSubCmd, "Stop a database", - `This subcommand stops a database or sandbox. + `This command stops a database or sandbox. Examples: # Stop a database with config file using password authentication diff --git a/commands/cmd_stop_node.go b/commands/cmd_stop_node.go index 41a9124..c7d6884 100644 --- a/commands/cmd_stop_node.go +++ b/commands/cmd_stop_node.go @@ -39,7 +39,7 @@ func makeCmdStopNode() *cobra.Command { newCmd, stopNodeCmd, "Stop a list of node(s)", - `This subcommand stops a node or list or nodes from an existing database. + `This command stops a node or list or nodes from an existing database. You must provide the host list with the --stop-hosts option followed by one or more hosts to stop as a comma-separated list. @@ -60,7 +60,7 @@ Examples: newCmd.setLocalFlags(cmd) // require hosts to stop - markFlagsRequired(cmd, []string{stopNodeFlag}) + markFlagsRequired(cmd, stopNodeFlag) return cmd } diff --git a/commands/cmd_stop_subcluster.go b/commands/cmd_stop_subcluster.go index 016c48e..c2ffa33 100644 --- a/commands/cmd_stop_subcluster.go +++ b/commands/cmd_stop_subcluster.go @@ -47,7 +47,7 @@ func makeCmdStopSubcluster() *cobra.Command { newCmd, stopSCSubCmd, "Stop a subcluster", - `This subcommand stops a subcluster from an existing Eon Mode database. + `This command stops a subcluster from an existing Eon Mode database. You must provide the subcluster name with the --subcluster option. @@ -77,7 +77,7 @@ Examples: newCmd.setLocalFlags(cmd) // require name of subcluster to add - markFlagsRequired(cmd, []string{subclusterFlag}) + markFlagsRequired(cmd, subclusterFlag) // hide eon mode flag since we expect it to come from config file, not from user input hideLocalFlags(cmd, []string{eonModeFlag}) diff --git a/commands/cmd_unsandbox.go b/commands/cmd_unsandbox.go index c5a2ddc..3e8507e 100644 --- a/commands/cmd_unsandbox.go +++ b/commands/cmd_unsandbox.go @@ -50,7 +50,7 @@ func makeCmdUnsandboxSubcluster() *cobra.Command { newCmd, unsandboxSubCmd, "Unsandbox a subcluster", - `This subcommand unsandboxes a subcluster in an existing Eon Mode database. + `This command unsandboxes a subcluster in an existing Eon Mode database. When you unsandbox a subcluster, its hosts shut down and restart as part of the main cluster. @@ -80,7 +80,7 @@ Examples: newCmd.setLocalFlags(cmd) // require name of subcluster to unsandbox - markFlagsRequired(cmd, []string{subclusterFlag}) + markFlagsRequired(cmd, subclusterFlag) return cmd } @@ -143,7 +143,7 @@ func (c *CmdUnsandboxSubcluster) Run(vcc vclusterops.ClusterCommands) error { return nil } - writeErr := dbConfig.write(options.ConfigPath) + writeErr := dbConfig.write(options.ConfigPath, true /*forceOverwrite*/) if writeErr != nil { vcc.PrintWarning("fail to write the config file, details: " + writeErr.Error()) return nil diff --git a/commands/vcluster_config.go b/commands/vcluster_config.go index eaf3273..0cb5d36 100644 --- a/commands/vcluster_config.go +++ b/commands/vcluster_config.go @@ -34,6 +34,7 @@ const ( defConfigFileName = "vertica_cluster.yaml" currentConfigFileVersion = "1.0" configFilePerm = 0644 + rpmConfDir = "/opt/vertica/config" ) // Config is the struct of vertica_cluster.yaml @@ -114,7 +115,6 @@ func initConfigImpl(vclusterExePath string, ensureOptVerticaConfigExists, ensure // have installed the vertica package on this machine and so can assume // /opt/vertica/config exists too. if vclusterExePath == defaultExecutablePath { - const rpmConfDir = "/opt/vertica/config" _, err := os.Stat(rpmConfDir) if ensureOptVerticaConfigExists && err != nil { if os.IsNotExist(err) { @@ -188,7 +188,7 @@ func loadConfigToViper() error { // writeConfig can write database information to vertica_cluster.yaml. // It will be called in the end of some subcommands that will change the db state. -func writeConfig(vdb *vclusterops.VCoordinationDatabase) error { +func writeConfig(vdb *vclusterops.VCoordinationDatabase, forceOverwrite bool) error { if dbOptions.ConfigPath == "" { return fmt.Errorf("configuration file path is empty") } @@ -199,7 +199,7 @@ func writeConfig(vdb *vclusterops.VCoordinationDatabase) error { } // update db config with the given database info - err = dbConfig.write(dbOptions.ConfigPath) + err = dbConfig.write(dbOptions.ConfigPath, forceOverwrite) if err != nil { return err } @@ -286,7 +286,10 @@ func readConfig() (dbConfig *DatabaseConfig, err error) { // any write error encountered. The viper in-built write function cannot // work well(the order of keys cannot be customized) so we used yaml.Marshal() // and os.WriteFile() to write the config file. -func (c *DatabaseConfig) write(configFilePath string) error { +func (c *DatabaseConfig) write(configFilePath string, forceOverwrite bool) error { + if util.CheckPathExist(configFilePath) && !forceOverwrite { + return fmt.Errorf("file %s exist, consider using --force-overwrite-file to overwrite the file", configFilePath) + } var config Config config.Version = currentConfigFileVersion config.Database = *c diff --git a/vclusterops/add_subcluster.go b/vclusterops/add_subcluster.go index b6dbcad..c6b2404 100644 --- a/vclusterops/add_subcluster.go +++ b/vclusterops/add_subcluster.go @@ -180,7 +180,7 @@ func (options *VAddSubclusterOptions) validateAnalyzeOptions(logger vlog.Printer if err != nil { return err } - return options.setUsePassword(logger) + return options.setUsePasswordAndValidateUsernameIfNeeded(logger) } // VAddSubcluster adds to a running database a new subcluster with provided options. diff --git a/vclusterops/alter_subcluster_type.go b/vclusterops/alter_subcluster_type.go index 3a0ee9b..d86a36a 100644 --- a/vclusterops/alter_subcluster_type.go +++ b/vclusterops/alter_subcluster_type.go @@ -72,9 +72,15 @@ func (options *VAlterSubclusterTypeOptions) validateParseOptions(logger vlog.Pri return err } - // need to provide a password or certs + // need to provide a password or key and certs if options.Password == nil && (options.Cert == "" || options.Key == "") { - return fmt.Errorf("must provide a password or certs") + // validate key and cert files in local file system + _, err = getCertFilePaths() + if err != nil { + // in case that the key or cert files do not exist + return fmt.Errorf("must provide a password, key and certificates explicitly," + + " or key and certificate files in the default paths") + } } if options.SCName == "" { diff --git a/vclusterops/cluster_op.go b/vclusterops/cluster_op.go index dc55fbd..6eeb593 100644 --- a/vclusterops/cluster_op.go +++ b/vclusterops/cluster_op.go @@ -240,6 +240,13 @@ func (op *opBase) parseAndCheckMapResponse(host, responseContent string) (opResp return responseObj, err } +func (op *opBase) parseAndCheckStringResponse(host, responseContent string) (string, error) { + var responseStr string + err := op.parseAndCheckResponse(host, responseContent, &responseStr) + + return responseStr, err +} + func (op *opBase) setClusterHTTPRequestName() { op.clusterHTTPRequest.Name = op.name } @@ -513,6 +520,7 @@ type ClusterCommands interface { VUnsandbox(options *VUnsandboxOptions) error VStopSubcluster(options *VStopSubclusterOptions) error VAlterSubclusterType(options *VAlterSubclusterTypeOptions) error + VPromoteSandboxToMain(options *VPromoteSandboxToMainOptions) error VRenameSubcluster(options *VRenameSubclusterOptions) error VFetchNodesDetails(options *VFetchNodesDetailsOptions) (NodesDetails, error) } diff --git a/vclusterops/create_db.go b/vclusterops/create_db.go index bdbbe2a..8859d08 100644 --- a/vclusterops/create_db.go +++ b/vclusterops/create_db.go @@ -43,6 +43,7 @@ type VCreateDatabaseOptions struct { // part 3: optional info ForceCleanupOnFailure bool // whether force remove existing directories on failure ForceRemovalAtCreation bool // whether force remove existing directories before creating the database + ForceOverwriteFile bool // whether force overwrite existing config and config param files SkipPackageInstall bool // whether skip package installation TimeoutNodeStartupSeconds int // timeout in seconds for polling node start up state diff --git a/vclusterops/fetch_nodes_details.go b/vclusterops/fetch_nodes_details.go index 304e014..f1c10b8 100644 --- a/vclusterops/fetch_nodes_details.go +++ b/vclusterops/fetch_nodes_details.go @@ -159,7 +159,7 @@ func (vcc *VClusterCommands) produceFetchNodesDetailsInstructions(options *VFetc var instructions []clusterOp // when password is specified, we will use username/password to call https endpoints - err := options.setUsePassword(vcc.Log) + err := options.setUsePasswordAndValidateUsernameIfNeeded(vcc.Log) if err != nil { return instructions, err } diff --git a/vclusterops/helpers.go b/vclusterops/helpers.go index ea599b9..e259c1c 100644 --- a/vclusterops/helpers.go +++ b/vclusterops/helpers.go @@ -229,7 +229,7 @@ func (vcc VClusterCommands) getVDBFromRunningDBIncludeSandbox(vdb *VCoordination // getVDBFromRunningDB will retrieve db configurations by calling https endpoints of a running db func (vcc VClusterCommands) getVDBFromRunningDBImpl(vdb *VCoordinationDatabase, options *DatabaseOptions, allowUseSandboxRes bool, sandbox string, updateNodeState bool) error { - err := options.setUsePassword(vcc.Log) + err := options.setUsePasswordAndValidateUsernameIfNeeded(vcc.Log) if err != nil { return fmt.Errorf("fail to set userPassword while retrieving database configurations, %w", err) } @@ -270,7 +270,7 @@ func (vcc VClusterCommands) getVDBFromRunningDBImpl(vdb *VCoordinationDatabase, // getClusterInfoFromRunningDB will retrieve db configurations by calling https endpoints of a running db func (vcc VClusterCommands) getClusterInfoFromRunningDB(vdb *VCoordinationDatabase, options *DatabaseOptions) error { - err := options.setUsePassword(vcc.Log) + err := options.setUsePasswordAndValidateUsernameIfNeeded(vcc.Log) if err != nil { return fmt.Errorf("fail to set userPassword while retrieving cluster configurations, %w", err) } diff --git a/vclusterops/https_convert_sandbox_to_main_op.go b/vclusterops/https_convert_sandbox_to_main_op.go new file mode 100644 index 0000000..e087b40 --- /dev/null +++ b/vclusterops/https_convert_sandbox_to_main_op.go @@ -0,0 +1,120 @@ +/* + (c) Copyright [2023-2024] Open Text. + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package vclusterops + +import ( + "errors" + "fmt" + + "github.com/vertica/vcluster/vclusterops/util" +) + +type httpsConvertSandboxToMainOp struct { + opBase + opHTTPSBase + sandbox string +} + +func makeHTTPSConvertSandboxToMainOp(hosts []string, userName string, + httpsPassword *string, useHTTPPassword bool, sandbox string) (httpsConvertSandboxToMainOp, error) { + op := httpsConvertSandboxToMainOp{} + op.name = "HTTPSConvertSandboxToMainOp" + op.description = "Convert local sandbox to main cluster" + op.hosts = hosts + op.sandbox = sandbox + op.useHTTPPassword = useHTTPPassword + + if useHTTPPassword { + err := util.ValidateUsernameAndPassword(op.name, useHTTPPassword, userName) + if err != nil { + return op, err + } + op.userName = userName + op.httpsPassword = httpsPassword + } + + return op, nil +} + +func (op *httpsConvertSandboxToMainOp) setupClusterHTTPRequest(hosts []string) error { + for _, host := range hosts { + httpRequest := hostHTTPRequest{} + httpRequest.Method = PostMethod + httpRequest.buildHTTPSEndpoint("sandbox/convert") + if op.useHTTPPassword { + httpRequest.Password = op.httpsPassword + httpRequest.Username = op.userName + } + op.clusterHTTPRequest.RequestCollection[host] = httpRequest + } + + return nil +} + +func (op *httpsConvertSandboxToMainOp) prepare(execContext *opEngineExecContext) error { + if len(op.hosts) == 0 { + return fmt.Errorf("[%s] cannot find any up hosts in the sandbox %s", op.name, op.sandbox) + } + execContext.dispatcher.setup(op.hosts) + + return op.setupClusterHTTPRequest(op.hosts) +} + +func (op *httpsConvertSandboxToMainOp) execute(execContext *opEngineExecContext) error { + if err := op.runExecute(execContext); err != nil { + return err + } + + return op.processResult(execContext) +} + +func (op *httpsConvertSandboxToMainOp) processResult(_ *opEngineExecContext) error { + var allErrs error + + for host, result := range op.clusterHTTPRequest.ResultCollection { + op.logResponse(host, result) + + if result.isUnauthorizedRequest() { + // skip checking response from other nodes because we will get the same error there + return result.err + } + if !result.isPassing() { + allErrs = errors.Join(allErrs, result.err) + // try processing other hosts' responses when the current host has some server errors + continue + } + + // decode the json-format response + // The successful response object will be a dictionary: + /* + { + "detail": "The sandbox has been converted to main successfully." + } + */ + _, err := op.parseAndCheckMapResponse(host, result.content) + if err != nil { + return fmt.Errorf(`[%s] fail to parse result on host %s, details: %w`, op.name, host, err) + } + + return nil + } + + return allErrs +} + +func (op *httpsConvertSandboxToMainOp) finalize(_ *opEngineExecContext) error { + return nil +} diff --git a/vclusterops/https_get_up_nodes_op.go b/vclusterops/https_get_up_nodes_op.go index cbd056b..57eba81 100644 --- a/vclusterops/https_get_up_nodes_op.go +++ b/vclusterops/https_get_up_nodes_op.go @@ -34,6 +34,7 @@ const ( InstallPackageCmd UnsandboxCmd ManageConnectionDrainingCmd + SetConfigurationParametersCmd ) type CommandType int @@ -219,7 +220,8 @@ func (op *httpsGetUpNodesOp) processResult(execContext *opEngineExecContext) err func isCompleteScanRequired(cmdType CommandType) bool { return cmdType == SandboxCmd || cmdType == StopDBCmd || cmdType == UnsandboxCmd || cmdType == StopSubclusterCmd || - cmdType == ManageConnectionDrainingCmd + cmdType == ManageConnectionDrainingCmd || + cmdType == SetConfigurationParametersCmd } func (op *httpsGetUpNodesOp) finalize(_ *opEngineExecContext) error { @@ -322,6 +324,7 @@ func (op *httpsGetUpNodesOp) collectUpHosts(nodesStates nodesStateInfo, host str upHosts.Add(node.Address) upScInfo[node.Address] = node.Subcluster if op.cmdType == ManageConnectionDrainingCmd || + op.cmdType == SetConfigurationParametersCmd || op.cmdType == StopDBCmd { sandboxInfo[node.Address] = node.Sandbox } diff --git a/vclusterops/https_sandbox_subcluster_op.go b/vclusterops/https_sandbox_subcluster_op.go index 093c188..58045e7 100644 --- a/vclusterops/https_sandbox_subcluster_op.go +++ b/vclusterops/https_sandbox_subcluster_op.go @@ -29,11 +29,14 @@ type httpsSandboxingOp struct { hostRequestBodyMap map[string]string scName string sandboxName string + SaveRp bool + Imeta bool + Sls bool } // This op is used to sandbox the given subcluster `scName` as `sandboxName` func makeHTTPSandboxingOp(logger vlog.Printer, scName, sandboxName string, - useHTTPPassword bool, userName string, httpsPassword *string) (httpsSandboxingOp, error) { + useHTTPPassword bool, userName string, httpsPassword *string, saveRp, imeta, sls bool) (httpsSandboxingOp, error) { op := httpsSandboxingOp{} op.name = "HTTPSSansboxingOp" op.description = "Convert subcluster into sandbox in catalog system" @@ -41,6 +44,9 @@ func makeHTTPSandboxingOp(logger vlog.Printer, scName, sandboxName string, op.useHTTPPassword = useHTTPPassword op.scName = scName op.sandboxName = sandboxName + op.SaveRp = saveRp + op.Imeta = imeta + op.Sls = sls if useHTTPPassword { err := util.ValidateUsernameAndPassword(op.name, useHTTPPassword, userName) @@ -74,7 +80,9 @@ func (op *httpsSandboxingOp) setupClusterHTTPRequest(hosts []string) error { func (op *httpsSandboxingOp) setupRequestBody() error { op.hostRequestBodyMap = make(map[string]string) op.hostRequestBodyMap["sandbox"] = op.sandboxName - + op.hostRequestBodyMap["save-restore-point"] = util.BoolToStr(op.SaveRp) + op.hostRequestBodyMap["create-storage-locations"] = util.BoolToStr(op.Sls) + op.hostRequestBodyMap["isolate-metadata"] = util.BoolToStr(op.Imeta) return nil } diff --git a/vclusterops/manage_connection_draining.go b/vclusterops/manage_connection_draining.go index 3dc7eb8..79f0fe7 100644 --- a/vclusterops/manage_connection_draining.go +++ b/vclusterops/manage_connection_draining.go @@ -70,11 +70,22 @@ func (opt *VManageConnectionDrainingOptions) validateParseOptions(logger vlog.Pr return err } - err = opt.validateBaseOptions(commandManageConnections, logger) + err = opt.validateBaseOptions(commandManageConnectionDraining, logger) if err != nil { return err } + // need to provide a password or key and certs + if opt.Password == nil && (opt.Cert == "" || opt.Key == "") { + // validate key and cert files in local file system + _, err := getCertFilePaths() + if err != nil { + // in case that the key or cert files do not exist + return fmt.Errorf("must provide a password, key and certificates explicitly," + + " or key and certificate files in the default paths") + } + } + return opt.validateExtraOptions(logger) } @@ -114,13 +125,19 @@ func (opt *VManageConnectionDrainingOptions) validateAnalyzeOptions(log vlog.Pri if err := opt.validateParseOptions(log); err != nil { return err } - err := opt.analyzeOptions() - if err != nil { + if err := opt.analyzeOptions(); err != nil { + return err + } + if err := opt.setUsePassword(log); err != nil { return err } - return opt.setUsePasswordForLocalDBConnection(log) + // username is always required when local db connection is made + return opt.validateUserName(log) } +// VManageConnectionDraining manages connection draining of nodes by pausing, redirecting, or +// resuming connections. It returns any error encountered. +// //nolint:dupl func (vcc VClusterCommands) VManageConnectionDraining(options *VManageConnectionDrainingOptions) error { // validate and analyze all options @@ -161,14 +178,16 @@ func (vcc VClusterCommands) produceManageConnectionDrainingInstructions( // get up hosts in all sandboxes httpsGetUpNodesOp, err := makeHTTPSGetUpNodesOp(options.DBName, options.Hosts, - options.usePassword, options.UserName, options.Password, ManageConnectionDrainingCmd) + options.usePassword, options.UserName, options.Password, + ManageConnectionDrainingCmd) if err != nil { return instructions, err } nmaManageConnectionsOp, err := makeNMAManageConnectionsOp(options.Hosts, - options.UserName, options.DBName, options.Sandbox, options.SCName, options.RedirectHostname, - options.Action, options.Password, options.usePassword) + options.UserName, options.DBName, options.Sandbox, options.SCName, + options.RedirectHostname, options.Action, options.Password, + options.usePassword) if err != nil { return instructions, err } diff --git a/vclusterops/manage_connection_draining_test.go b/vclusterops/manage_connection_draining_test.go index 016eca9..f70a253 100644 --- a/vclusterops/manage_connection_draining_test.go +++ b/vclusterops/manage_connection_draining_test.go @@ -26,15 +26,15 @@ func TestVManageConnectionsOptions_validateParseOptions(t *testing.T) { logger := vlog.Printer{} opt := VManageConnectionDrainingOptionsFactory() - testPassword := "test-password" - testSCName := "test-sc" - testDBName := "testdbname" - testUserName := "test-username" - testRedirectHostname := "test-redirect-hostname" + testPassword := "draining-test-password" + testSCName := "draining-test-sc" + testDBName := "draining_test_dbname" + testUserName := "draining-test-username" + testRedirectHostname := "draining-test-redirect-hostname" opt.SCName = testSCName opt.IsEon = true - opt.RawHosts = append(opt.RawHosts, "test-raw-host") + opt.RawHosts = append(opt.RawHosts, "draining-test-raw-host") opt.DBName = testDBName opt.UserName = testUserName opt.Password = &testPassword @@ -49,8 +49,14 @@ func TestVManageConnectionsOptions_validateParseOptions(t *testing.T) { err = opt.validateParseOptions(logger) assert.NoError(t, err) - // negative: no database name + // negative: Eon mode not set opt.UserName = testUserName + opt.IsEon = false + err = opt.validateParseOptions(logger) + assert.Error(t, err) + + // negative: no database name + opt.IsEon = true opt.DBName = "" err = opt.validateParseOptions(logger) assert.Error(t, err) diff --git a/vclusterops/nma_manage_connections_op.go b/vclusterops/nma_manage_connections_op.go index 35f0a86..9a0deb6 100644 --- a/vclusterops/nma_manage_connections_op.go +++ b/vclusterops/nma_manage_connections_op.go @@ -19,8 +19,6 @@ import ( "encoding/json" "errors" "fmt" - - "github.com/vertica/vcluster/vclusterops/util" ) type nmaManageConnectionsOp struct { @@ -31,12 +29,6 @@ type nmaManageConnectionsOp struct { initiator string } -type sqlEndpointData struct { - DBUsername string `json:"username"` - DBPassword string `json:"password"` - DBName string `json:"dbname"` -} - type manageConnectionsData struct { sqlEndpointData SubclusterName string `json:"subclustername"` @@ -62,20 +54,10 @@ func makeNMAManageConnectionsOp(hosts []string, return op, nil } -func createSQLEndpointData(username, dbName string, useDBPassword bool, password *string) sqlEndpointData { - sqlConnectionData := sqlEndpointData{} - sqlConnectionData.DBUsername = username - sqlConnectionData.DBName = dbName - if useDBPassword { - sqlConnectionData.DBPassword = *password - } - return sqlConnectionData -} - func (op *nmaManageConnectionsOp) setupRequestBody( username, dbName, subclusterName, redirectHostname string, password *string, useDBPassword bool) error { - err := util.ValidateSQLEndpointData(op.name, + err := ValidateSQLEndpointData(op.name, useDBPassword, username, password, dbName) if err != nil { return err @@ -139,7 +121,7 @@ func (op *nmaManageConnectionsOp) processResult(_ *opEngineExecContext) error { op.logResponse(host, result) if result.isPassing() { - _, err := op.parseAndCheckMapResponse(host, result.content) + _, err := op.parseAndCheckStringResponse(host, result.content) if err != nil { allErrs = errors.Join(allErrs, err) } diff --git a/vclusterops/nma_manage_connections_op_test.go b/vclusterops/nma_manage_connections_op_test.go index 06049d3..045fa70 100644 --- a/vclusterops/nma_manage_connections_op_test.go +++ b/vclusterops/nma_manage_connections_op_test.go @@ -26,11 +26,11 @@ func TestNmaManageConnectionsOp_SetupRequestBody(t *testing.T) { op := &nmaManageConnectionsOp{} op.action = ActionRedirect - username := "test-user" - dbName := "test-db" - subclusterName := "test-subcluster" - redirectHostname := "test-redirect" - password := "test-password-op" + username := "draining-test-user-op" + dbName := "draining-test-db-op" + subclusterName := "draining-test-subcluster-op" + redirectHostname := "draining-test-redirect-op" + password := "draining-test-password-op" useDBPassword := true err := op.setupRequestBody(username, dbName, subclusterName, redirectHostname, &password, useDBPassword) diff --git a/vclusterops/nma_set_config_parameter_op.go b/vclusterops/nma_set_config_parameter_op.go new file mode 100644 index 0000000..f206e1a --- /dev/null +++ b/vclusterops/nma_set_config_parameter_op.go @@ -0,0 +1,131 @@ +/* + (c) Copyright [2023-2024] Open Text. + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package vclusterops + +import ( + "encoding/json" + "errors" + "fmt" +) + +type nmaSetConfigurationParameterOp struct { + opBase + hostRequestBody string + sandbox string + initiator string +} + +type setConfigurationParameterData struct { + sqlEndpointData + ConfigParameter string `json:"config_parameter"` + Value string `json:"value"` + Level string `json:"level"` +} + +func makeNMASetConfigurationParameterOp(hosts []string, + username, dbName, sandbox, configParameter, value, level string, + password *string, useHTTPPassword bool) (nmaSetConfigurationParameterOp, error) { + op := nmaSetConfigurationParameterOp{} + op.name = "NMASetConfigurationParameterOp" + op.description = "Set configuration parameter value" + op.hosts = hosts + op.sandbox = sandbox + + err := op.setupRequestBody(username, dbName, configParameter, value, level, password, useHTTPPassword) + if err != nil { + return op, err + } + + return op, nil +} + +func (op *nmaSetConfigurationParameterOp) setupRequestBody( + username, dbName, configParameter, value, level string, password *string, + useDBPassword bool) error { + err := ValidateSQLEndpointData(op.name, + useDBPassword, username, password, dbName) + if err != nil { + return err + } + setConfigData := setConfigurationParameterData{} + setConfigData.sqlEndpointData = createSQLEndpointData(username, dbName, useDBPassword, password) + setConfigData.ConfigParameter = configParameter + setConfigData.Value = value + setConfigData.Level = level + + dataBytes, err := json.Marshal(setConfigData) + if err != nil { + return fmt.Errorf("[%s] fail to marshal request data to JSON string, detail %w", op.name, err) + } + + op.hostRequestBody = string(dataBytes) + + op.logger.Info("request data", "op name", op.name, "hostRequestBody", op.hostRequestBody) + + return nil +} + +func (op *nmaSetConfigurationParameterOp) setupClusterHTTPRequest(initiator string) error { + httpRequest := hostHTTPRequest{} + httpRequest.Method = PutMethod + httpRequest.buildNMAEndpoint("configuration/set") + httpRequest.RequestData = op.hostRequestBody + op.clusterHTTPRequest.RequestCollection[initiator] = httpRequest + + return nil +} + +func (op *nmaSetConfigurationParameterOp) prepare(execContext *opEngineExecContext) error { + // select an up host in the sandbox as the initiator + initiator, err := getInitiatorInSandbox(op.sandbox, op.hosts, execContext.upHostsToSandboxes) + if err != nil { + return err + } + op.initiator = initiator + execContext.dispatcher.setup([]string{op.initiator}) + return op.setupClusterHTTPRequest(op.initiator) +} + +func (op *nmaSetConfigurationParameterOp) execute(execContext *opEngineExecContext) error { + if err := op.runExecute(execContext); err != nil { + return err + } + + return op.processResult(execContext) +} + +func (op *nmaSetConfigurationParameterOp) finalize(_ *opEngineExecContext) error { + return nil +} + +func (op *nmaSetConfigurationParameterOp) processResult(_ *opEngineExecContext) error { + var allErrs error + + for host, result := range op.clusterHTTPRequest.ResultCollection { + op.logResponse(host, result) + + if result.isPassing() { + _, err := op.parseAndCheckStringResponse(host, result.content) + if err != nil { + allErrs = errors.Join(allErrs, err) + } + } else { + allErrs = errors.Join(allErrs, result.err) + } + } + + return allErrs +} diff --git a/vclusterops/nma_set_config_parameter_op_test.go b/vclusterops/nma_set_config_parameter_op_test.go new file mode 100644 index 0000000..1fbb17d --- /dev/null +++ b/vclusterops/nma_set_config_parameter_op_test.go @@ -0,0 +1,59 @@ +/* + (c) Copyright [2023-2024] Open Text. + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package vclusterops + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNmaSetConfigurationParameterOp_SetupRequestBody(t *testing.T) { + op := &nmaSetConfigurationParameterOp{} + + username := "config-test-user-op" + dbName := "config-test-db-op" + configParameter := "config-test-param-op" + value := "config-test-value-op" + level := "config-test-level-op" + password := "config-test-password-op" + useDBPassword := true + + err := op.setupRequestBody(username, dbName, configParameter, value, level, &password, useDBPassword) + assert.NoError(t, err) + + expectedData := setConfigurationParameterData{ + ConfigParameter: configParameter, + Value: value, + Level: level, + sqlEndpointData: createSQLEndpointData(username, dbName, useDBPassword, &password), + } + + expectedBytes, _ := json.Marshal(expectedData) + expectedRequestBody := string(expectedBytes) + + assert.Equal(t, expectedRequestBody, op.hostRequestBody) + + err = op.setupRequestBody("", dbName, configParameter, value, level, &password, useDBPassword) + assert.Error(t, err) + + err = op.setupRequestBody(username, "", configParameter, value, level, &password, useDBPassword) + assert.Error(t, err) + + err = op.setupRequestBody(username, dbName, configParameter, value, level, nil, useDBPassword) + assert.Error(t, err) +} diff --git a/vclusterops/promote_sandbox_to_main.go b/vclusterops/promote_sandbox_to_main.go new file mode 100644 index 0000000..70b106f --- /dev/null +++ b/vclusterops/promote_sandbox_to_main.go @@ -0,0 +1,157 @@ +/* + (c) Copyright [2023-2024] Open Text. + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package vclusterops + +import ( + "fmt" + + "github.com/vertica/vcluster/vclusterops/util" + "github.com/vertica/vcluster/vclusterops/vlog" +) + +type VPromoteSandboxToMainOptions struct { + // basic db info + DatabaseOptions + // Name of the sandbox to promote to main + SandboxName string +} + +func VPromoteSandboxToMainFactory() VPromoteSandboxToMainOptions { + opt := VPromoteSandboxToMainOptions{} + // set default values to the params + opt.setDefaultValues() + return opt +} + +func (opt *VPromoteSandboxToMainOptions) validateEonOptions(_ vlog.Printer) error { + if !opt.IsEon { + return fmt.Errorf("promote a sandbox to main is only supported in Eon mode") + } + return nil +} + +func (opt *VPromoteSandboxToMainOptions) validateParseOptions(logger vlog.Printer) error { + err := opt.validateEonOptions(logger) + if err != nil { + return err + } + + // need to provide a password or certs in source database + if opt.Password == nil && (opt.Cert == "" || opt.Key == "") { + return fmt.Errorf("must provide a password or a key-certificate pair") + } + + return opt.validateBaseOptions(commandPromoteSandboxToMain, logger) +} + +// analyzeOptions will modify some options based on what is chosen +func (opt *VPromoteSandboxToMainOptions) analyzeOptions() (err error) { + // we analyze host names when it is set in user input, otherwise we use hosts in yaml config + if len(opt.RawHosts) > 0 { + // resolve RawHosts to be IP addresses + hostAddresses, err := util.ResolveRawHostsToAddresses(opt.RawHosts, opt.IPv6) + if err != nil { + return err + } + opt.Hosts = hostAddresses + } + return nil +} + +func (opt *VPromoteSandboxToMainOptions) validateAnalyzeOptions(logger vlog.Printer) error { + if err := opt.validateParseOptions(logger); err != nil { + return err + } + if err := opt.analyzeOptions(); err != nil { + return err + } + if err := opt.setUsePassword(logger); err != nil { + return err + } + // username is always required when local db connection is made + return opt.validateUserName(logger) +} + +// VPromoteSandboxToMain can convert local sandbox to main cluster. The conversion is supported only for +// special sandboxes: without meta-isolation and communal (prefix) isolation. Those can be created +// with the: "sls=false;imeta=false" options +func (vcc VClusterCommands) VPromoteSandboxToMain(options *VPromoteSandboxToMainOptions) error { + /* + * - Produce Instructions + * - Create a VClusterOpEngine + * - Give the instructions to the VClusterOpEngine to run + */ + + // validate and analyze options + err := options.validateAnalyzeOptions(vcc.Log) + if err != nil { + return err + } + + // retrieve information from the database to accurately determine the state of each node in both the main cluster and sandbox + vdb := makeVCoordinationDatabase() + err = vcc.getVDBFromRunningDBIncludeSandbox(&vdb, &options.DatabaseOptions, options.SandboxName) + if err != nil { + return err + } + + // produce sandbox to main cluster instructions + instructions, err := vcc.promoteSandboxToMainInstructions(options, &vdb) + if err != nil { + return fmt.Errorf("fail to produce instructions, %w", err) + } + + // create a VClusterOpEngine, and add certs to the engine + certs := httpsCerts{key: options.Key, cert: options.Cert, caCert: options.CaCert} + clusterOpEngine := makeClusterOpEngine(instructions, &certs) + + // give the instructions to the VClusterOpEngine to run + runError := clusterOpEngine.run(vcc.Log) + if runError != nil { + return fmt.Errorf("fail to promote a sandbox to main cluster: %w", runError) + } + + return nil +} + +// The generated instructions will later perform the following operations necessary +// for a successful alter subcluster type operation: +// - promote sandbox to main using one of the up nodes in the sandbox subcluster +func (vcc VClusterCommands) promoteSandboxToMainInstructions(options *VPromoteSandboxToMainOptions, + vdb *VCoordinationDatabase) ([]clusterOp, error) { + var instructions []clusterOp + + var upHost string + for _, node := range vdb.HostNodeMap { + if node.State == util.NodeDownState { + continue + } + // the up host is used to promote the sandbox to main cluster + // should be the one in the sandbox + if node.Sandbox == options.SandboxName && node.Sandbox != "" { + upHost = node.Address + break + } + } + sandboxHost := []string{upHost} + httpsConvertSandboxToMainOp, err := makeHTTPSConvertSandboxToMainOp(sandboxHost, + options.UserName, options.Password, options.usePassword, options.SandboxName) + if err != nil { + return nil, err + } + instructions = append(instructions, &httpsConvertSandboxToMainOp) + return instructions, nil +} diff --git a/vclusterops/promote_sandbox_to_main_test.go b/vclusterops/promote_sandbox_to_main_test.go new file mode 100644 index 0000000..ce54335 --- /dev/null +++ b/vclusterops/promote_sandbox_to_main_test.go @@ -0,0 +1,54 @@ +/* + (c) Copyright [2023-2024] Open Text. + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package vclusterops + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vertica/vcluster/vclusterops/vlog" +) + +func TestPromoteSandboxToMainOptions_validateParseOptions(t *testing.T) { + logger := vlog.Printer{} + + opt := VPromoteSandboxToMainFactory() + testPassword := "test-password-3" + + opt.IsEon = true + opt.RawHosts = append(opt.RawHosts, "test-raw-host") + opt.DBName = testDBName + opt.UserName = testUserName + opt.Password = &testPassword + + err := opt.validateParseOptions(logger) + assert.NoError(t, err) + + opt.UserName = "" + err = opt.validateParseOptions(logger) + assert.NoError(t, err) + + // negative: no database name + opt.UserName = testUserName + opt.DBName = "" + err = opt.validateParseOptions(logger) + assert.ErrorContains(t, err, "must specify a database name") + + // negative: enterprise database + opt.IsEon = false + err = opt.validateParseOptions(logger) + assert.ErrorContains(t, err, "promote a sandbox to main is only supported in Eon mode") +} diff --git a/vclusterops/re_ip.go b/vclusterops/re_ip.go index 73325f5..a4bb8ef 100644 --- a/vclusterops/re_ip.go +++ b/vclusterops/re_ip.go @@ -213,7 +213,7 @@ func (vcc VClusterCommands) produceReIPInstructions(options *VReIPOptions, vdb * nmaHealthOp := makeNMAHealthOp(hosts) // need username for https operations - err := options.setUsePassword(vcc.Log) + err := options.setUsePasswordAndValidateUsernameIfNeeded(vcc.Log) if err != nil { return instructions, err } diff --git a/vclusterops/remove_node.go b/vclusterops/remove_node.go index acd0422..fbc77a2 100644 --- a/vclusterops/remove_node.go +++ b/vclusterops/remove_node.go @@ -104,7 +104,7 @@ func (options *VRemoveNodeOptions) validateAnalyzeOptions(log vlog.Printer) erro if err != nil { return err } - return options.setUsePassword(log) + return options.setUsePasswordAndValidateUsernameIfNeeded(log) } func (vcc VClusterCommands) VRemoveNode(options *VRemoveNodeOptions) (VCoordinationDatabase, error) { diff --git a/vclusterops/remove_subcluster.go b/vclusterops/remove_subcluster.go index 5543253..2cb51cd 100644 --- a/vclusterops/remove_subcluster.go +++ b/vclusterops/remove_subcluster.go @@ -124,7 +124,7 @@ func (options *VRemoveScOptions) validateAnalyzeOptions(logger vlog.Printer) err if err != nil { return err } - return options.setUsePassword(logger) + return options.setUsePasswordAndValidateUsernameIfNeeded(logger) } // VRemoveSubcluster removes a subcluster. It returns updated database catalog information and any error encountered. diff --git a/vclusterops/rename_subcluster.go b/vclusterops/rename_subcluster.go index db61299..92ca1c9 100644 --- a/vclusterops/rename_subcluster.go +++ b/vclusterops/rename_subcluster.go @@ -55,9 +55,15 @@ func (options *VRenameSubclusterOptions) validateParseOptions(logger vlog.Printe return err } - // need to provide a password or certs + // need to provide a password or key and certs if options.Password == nil && (options.Cert == "" || options.Key == "") { - return fmt.Errorf("must provide a password or certs") + // validate key and cert files in local file system + _, err = getCertFilePaths() + if err != nil { + // in case that the key or cert files do not exist + return fmt.Errorf("must provide a password, key and certificates explicitly," + + " or key and certificate files in the default paths") + } } if options.SCName == "" { diff --git a/vclusterops/replication.go b/vclusterops/replication.go index 89c6c5c..9d04a85 100644 --- a/vclusterops/replication.go +++ b/vclusterops/replication.go @@ -72,9 +72,15 @@ func (options *VReplicationDatabaseOptions) validateExtraOptions() error { return err } - // need to provide a password or certs in source database + // need to provide a password or key and certs if options.Password == nil && (options.Cert == "" || options.Key == "") { - return fmt.Errorf("must provide a password or certs") + // validate key and cert files in local file system + _, err = getCertFilePaths() + if err != nil { + // in case that the key or cert files do not exist + return fmt.Errorf("must provide a password, key and certificates explicitly," + + " or key and certificate files in the default paths") + } } // need to provide a password or TLSconfig if source and target username are different @@ -198,7 +204,7 @@ func (vcc VClusterCommands) produceDBReplicationInstructions(options *VReplicati var instructions []clusterOp // need username for https operations in source database - err := options.setUsePassword(vcc.Log) + err := options.setUsePasswordAndValidateUsernameIfNeeded(vcc.Log) if err != nil { return instructions, err } diff --git a/vclusterops/sandbox.go b/vclusterops/sandbox.go index 8c217f5..c6d4d34 100644 --- a/vclusterops/sandbox.go +++ b/vclusterops/sandbox.go @@ -28,6 +28,12 @@ type VSandboxOptions struct { SCName string SCHosts []string SCRawHosts []string + // indicate whether a restore point is created when create the sandbox + SaveRp bool + // indicate whether the metadata of sandbox should be isolated + Imeta bool + // indicate whether the sandbox should create its own storage locations + Sls bool // The expected node names with their IPs in the subcluster, the user of vclusterOps needs // to make sure the provided values are correct. This option will be used to do re-ip in // the target sandbox. @@ -156,7 +162,7 @@ func (vcc *VClusterCommands) produceSandboxSubclusterInstructions(options *VSand // Run Sandboxing httpsSandboxSubclusterOp, err := makeHTTPSandboxingOp(vcc.Log, options.SCName, options.SandboxName, - usePassword, username, options.Password) + usePassword, username, options.Password, options.SaveRp, options.Imeta, options.Sls) if err != nil { return instructions, err } diff --git a/vclusterops/scrutinize.go b/vclusterops/scrutinize.go index 0bde889..fa492b2 100644 --- a/vclusterops/scrutinize.go +++ b/vclusterops/scrutinize.go @@ -180,7 +180,7 @@ func (options *VScrutinizeOptions) analyzeOptions(logger vlog.Printer) (err erro return err } - err = options.setUsePassword(logger) + err = options.setUsePasswordAndValidateUsernameIfNeeded(logger) return err } diff --git a/vclusterops/set_config_parameter.go b/vclusterops/set_config_parameter.go new file mode 100644 index 0000000..12bfa29 --- /dev/null +++ b/vclusterops/set_config_parameter.go @@ -0,0 +1,166 @@ +/* + (c) Copyright [2023-2024] Open Text. + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package vclusterops + +import ( + "errors" + "fmt" + + "github.com/vertica/vcluster/vclusterops/util" + "github.com/vertica/vcluster/vclusterops/vlog" +) + +type VSetConfigurationParameterOptions struct { + /* part 1: basic db info */ + DatabaseOptions + + /* part 2: set configuration parameters options */ + Sandbox string + ConfigParameter string + // set value literally to "null" to clear the value of a config parameter + Value string + Level string +} + +func VSetConfigurationParameterOptionsFactory() VSetConfigurationParameterOptions { + opt := VSetConfigurationParameterOptions{} + // set default values to the params + opt.setDefaultValues() + + return opt +} + +func (opt *VSetConfigurationParameterOptions) validateParseOptions(logger vlog.Printer) error { + err := opt.validateBaseOptions(commandSetConfigurationParameter, logger) + if err != nil { + return err + } + + // need to provide a password or key and certs + if opt.Password == nil && (opt.Cert == "" || opt.Key == "") { + // validate key and cert files in local file system + _, err := getCertFilePaths() + if err != nil { + // in case that the key or cert files do not exist + return fmt.Errorf("must provide a password, key and certificates explicitly," + + " or key and certificate files in the default paths") + } + } + + return opt.validateExtraOptions(logger) +} + +func (opt *VSetConfigurationParameterOptions) validateExtraOptions(logger vlog.Printer) error { + if opt.ConfigParameter == "" { + errStr := "configuration parameter must not be empty" + logger.PrintError(errStr) + return errors.New(errStr) + } + // opt.Value could be empty (which is not equivalent to "null") + // opt.Level could be empty (which means database level) + return nil +} + +func (opt *VSetConfigurationParameterOptions) analyzeOptions() (err error) { + // we analyze host names when it is set in user input, otherwise we use hosts in yaml config + if len(opt.RawHosts) > 0 { + // resolve RawHosts to be IP addresses + opt.Hosts, err = util.ResolveRawHostsToAddresses(opt.RawHosts, opt.IPv6) + if err != nil { + return err + } + opt.normalizePaths() + } + return nil +} + +func (opt *VSetConfigurationParameterOptions) validateAnalyzeOptions(log vlog.Printer) error { + if err := opt.validateParseOptions(log); err != nil { + return err + } + if err := opt.analyzeOptions(); err != nil { + return err + } + if err := opt.setUsePassword(log); err != nil { + return err + } + // username is always required when local db connection is made + return opt.validateUserName(log) +} + +// VSetConfigurationParameters sets or clears the value of a database configuration parameter. +// It returns any error encountered. +func (vcc VClusterCommands) VSetConfigurationParameters(options *VSetConfigurationParameterOptions) error { + // validate and analyze all options + err := options.validateAnalyzeOptions(vcc.Log) + if err != nil { + return err + } + + // produce set configuration parameters instructions + instructions, err := vcc.produceSetConfigurationParameterInstructions(options) + if err != nil { + return fmt.Errorf("fail to produce instructions, %w", err) + } + + // Create a VClusterOpEngine, and add certs to the engine + certs := httpsCerts{key: options.Key, cert: options.Cert, caCert: options.CaCert} + clusterOpEngine := makeClusterOpEngine(instructions, &certs) + + // Give the instructions to the VClusterOpEngine to run + runError := clusterOpEngine.run(vcc.Log) + if runError != nil { + return fmt.Errorf("fail to set configuration parameter: %w", runError) + } + + return nil +} + +// The generated instructions will later perform the following operations necessary +// for a successful set configuration parameter action. +// - Check NMA connectivity +// - Check UP nodes and sandboxes info +// - Send set configuration parameter request +func (vcc VClusterCommands) produceSetConfigurationParameterInstructions( + options *VSetConfigurationParameterOptions) ([]clusterOp, error) { + var instructions []clusterOp + + // get up hosts in all sandboxes + httpsGetUpNodesOp, err := makeHTTPSGetUpNodesOp(options.DBName, options.Hosts, + options.usePassword, options.UserName, options.Password, + SetConfigurationParametersCmd) + if err != nil { + return instructions, err + } + + nmaHealthOp := makeNMAHealthOp(options.Hosts) + + nmaSetConfigOp, err := makeNMASetConfigurationParameterOp(options.Hosts, + options.UserName, options.DBName, options.Sandbox, + options.ConfigParameter, options.Value, options.Level, + options.Password, options.usePassword) + if err != nil { + return instructions, err + } + + instructions = append(instructions, + &nmaHealthOp, + &httpsGetUpNodesOp, + &nmaSetConfigOp, + ) + + return instructions, nil +} diff --git a/vclusterops/set_config_parameter_test.go b/vclusterops/set_config_parameter_test.go new file mode 100644 index 0000000..615efe0 --- /dev/null +++ b/vclusterops/set_config_parameter_test.go @@ -0,0 +1,82 @@ +/* + (c) Copyright [2023-2024] Open Text. + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package vclusterops + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vertica/vcluster/vclusterops/vlog" +) + +func TestVSetConfigurationParameterOptions_validateParseOptions(t *testing.T) { + logger := vlog.Printer{} + + opt := VSetConfigurationParameterOptionsFactory() + testPassword := "config-test-password" + testSandbox := "config-test-sandbox" + testDBName := "config_test_dbname" + testUserName := "config-test-username" + testConfigParameter := "config-test-parameter" + testValue := "config-test-value" + testLevel := "config-test-level" + + opt.Sandbox = testSandbox + opt.RawHosts = append(opt.RawHosts, "config-test-raw-host") + opt.DBName = testDBName + opt.UserName = testUserName + opt.Password = &testPassword + opt.ConfigParameter = testConfigParameter + opt.Value = testValue + opt.Level = testLevel + + err := opt.validateParseOptions(logger) + assert.NoError(t, err) + + // positive: no username (in which case default OS username will be used) + opt.UserName = "" + err = opt.validateParseOptions(logger) + assert.NoError(t, err) + + // positive: value is "null" + opt.UserName = testUserName + opt.Value = "null" + err = opt.validateParseOptions(logger) + assert.NoError(t, err) + + // positive: value is empty + opt.Value = "" + err = opt.validateParseOptions(logger) + assert.NoError(t, err) + + // positive: empty level + opt.Value = testValue + opt.Level = "" + err = opt.validateParseOptions(logger) + assert.NoError(t, err) + + // negative: no database name + opt.Level = testLevel + opt.DBName = "" + err = opt.validateParseOptions(logger) + assert.Error(t, err) + + // negative: no configuration parameter + opt.DBName = testDBName + opt.ConfigParameter = "" + err = opt.validateParseOptions(logger) + assert.Error(t, err) +} diff --git a/vclusterops/sql_endpoint_common.go b/vclusterops/sql_endpoint_common.go new file mode 100644 index 0000000..43860b1 --- /dev/null +++ b/vclusterops/sql_endpoint_common.go @@ -0,0 +1,48 @@ +/* + (c) Copyright [2023-2024] Open Text. + Licensed under the Apache License, Version 2.0 (the "License"); + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package vclusterops + +import "fmt" + +type sqlEndpointData struct { + DBUsername string `json:"username"` + DBPassword string `json:"password"` + DBName string `json:"dbname"` +} + +func createSQLEndpointData(username, dbName string, useDBPassword bool, password *string) sqlEndpointData { + sqlConnectionData := sqlEndpointData{} + sqlConnectionData.DBUsername = username + sqlConnectionData.DBName = dbName + if useDBPassword { + sqlConnectionData.DBPassword = *password + } + return sqlConnectionData +} + +func ValidateSQLEndpointData(opName string, useDBPassword bool, userName string, + password *string, dbName string) error { + if userName == "" { + return fmt.Errorf("[%s] should always provide a username for local database connection", opName) + } + if dbName == "" { + return fmt.Errorf("[%s] should always provide a database name for local database connection", opName) + } + if useDBPassword && password == nil { + return fmt.Errorf("[%s] should properly set the password when a password is configured", opName) + } + return nil +} diff --git a/vclusterops/start_db.go b/vclusterops/start_db.go index 4156ca3..3266c3c 100644 --- a/vclusterops/start_db.go +++ b/vclusterops/start_db.go @@ -37,6 +37,8 @@ type VStartDatabaseOptions struct { StatePollingTimeout int // whether trim the input host list based on the catalog info TrimHostList bool + Sandbox string // Start db on given sandbox + MainCluster bool // Start db on main cluster only // If the path is set, the NMA will store the Vertica start command at the path // instead of executing it. This is useful in containerized environments where // you may not want to have both the NMA and Vertica server in the same container. @@ -244,7 +246,7 @@ func (vcc VClusterCommands) produceStartDBPreCheck(options *VStartDatabaseOption nmaHealthOp := makeNMAHealthOp(options.Hosts) // need username for https operations - err := options.setUsePassword(vcc.Log) + err := options.setUsePasswordAndValidateUsernameIfNeeded(vcc.Log) if err != nil { return instructions, err } diff --git a/vclusterops/start_node.go b/vclusterops/start_node.go index 26da453..5621632 100644 --- a/vclusterops/start_node.go +++ b/vclusterops/start_node.go @@ -271,7 +271,7 @@ func (vcc VClusterCommands) produceStartNodesInstructions(startNodeInfo *VStartN nmaHealthOp := makeNMAHealthOp(options.Hosts) // need username for https operations - err := options.setUsePassword(vcc.Log) + err := options.setUsePasswordAndValidateUsernameIfNeeded(vcc.Log) if err != nil { return instructions, err } diff --git a/vclusterops/start_subcluster.go b/vclusterops/start_subcluster.go index 8bc81e7..344f05b 100644 --- a/vclusterops/start_subcluster.go +++ b/vclusterops/start_subcluster.go @@ -105,7 +105,7 @@ func (options *VStartScOptions) validateAnalyzeOptions(logger vlog.Printer) erro if err != nil { return err } - return options.setUsePassword(logger) + return options.setUsePasswordAndValidateUsernameIfNeeded(logger) } // VStartSubcluster start nodes in a subcluster. It returns any error encountered. diff --git a/vclusterops/util/util.go b/vclusterops/util/util.go index d187521..1537480 100644 --- a/vclusterops/util/util.go +++ b/vclusterops/util/util.go @@ -96,6 +96,13 @@ func CheckNotEmpty(a string) bool { return a != "" } +func BoolToStr(b bool) string { + if b { + return "true" + } + return "false" +} + func CheckAllEmptyOrNonEmpty(vars ...string) bool { // Initialize flags for empty and non-empty conditions allEmpty := true @@ -394,20 +401,6 @@ func ValidateUsernameAndPassword(opName string, useHTTPPassword bool, userName s return nil } -func ValidateSQLEndpointData(opName string, useDBPassword bool, userName string, - password *string, dbName string) error { - if userName == "" { - return fmt.Errorf("[%s] should always provide a username for local database connection", opName) - } - if dbName == "" { - return fmt.Errorf("[%s] should always provide a database name for local database connection", opName) - } - if useDBPassword && password == nil { - return fmt.Errorf("[%s] should properly set the password when a password is configured", opName) - } - return nil -} - const ( FileExist = 0 FileNotExist = 1 diff --git a/vclusterops/vcluster_database_options.go b/vclusterops/vcluster_database_options.go index 060a218..ef6feb5 100644 --- a/vclusterops/vcluster_database_options.go +++ b/vclusterops/vcluster_database_options.go @@ -85,29 +85,31 @@ const ( ) const ( - commandCreateDB = "create_db" - commandDropDB = "drop_db" - commandStopDB = "stop_db" - commandStartDB = "start_db" - commandAddNode = "add_node" - commandRemoveNode = "remove_node" - commandStopNode = "stop_node" - commandRestartNode = "restart_node" - commandAddSubcluster = "add_subcluster" - commandRemoveSubcluster = "remove_subcluster" - commandStopSubcluster = "stop_subcluster" - commandStartSubcluster = "start_subcluster" - commandSandboxSC = "sandbox_subcluster" - commandUnsandboxSC = "unsandbox_subcluster" - commandShowRestorePoints = "show_restore_points" - commandInstallPackages = "install_packages" - commandConfigRecover = "manage_config_recover" - commandManageConnections = "manage_connections" - commandReplicationStart = "replication_start" - commandFetchNodesDetails = "fetch_nodes_details" - commandAlterSubclusterType = "alter_subcluster_type" - commandRenameSc = "rename_subcluster" - commandReIP = "re_ip" + commandCreateDB = "create_db" + commandDropDB = "drop_db" + commandStopDB = "stop_db" + commandStartDB = "start_db" + commandAddNode = "add_node" + commandRemoveNode = "remove_node" + commandStopNode = "stop_node" + commandRestartNode = "restart_node" + commandAddSubcluster = "add_subcluster" + commandRemoveSubcluster = "remove_subcluster" + commandStopSubcluster = "stop_subcluster" + commandStartSubcluster = "start_subcluster" + commandSandboxSC = "sandbox_subcluster" + commandUnsandboxSC = "unsandbox_subcluster" + commandShowRestorePoints = "show_restore_points" + commandInstallPackages = "install_packages" + commandConfigRecover = "manage_config_recover" + commandManageConnectionDraining = "manage_connection_draining" + commandSetConfigurationParameter = "set_configuration_parameter" + commandReplicationStart = "replication_start" + commandPromoteSandboxToMain = "promote_sandbox_to_main" + commandFetchNodesDetails = "fetch_nodes_details" + commandAlterSubclusterType = "alter_subcluster_type" + commandRenameSc = "rename_subcluster" + commandReIP = "re_ip" ) func DatabaseOptionsFactory() DatabaseOptions { @@ -257,7 +259,7 @@ func (opt *DatabaseOptions) validateUserName(log vlog.Printer) error { return nil } -func (opt *DatabaseOptions) setUsePassword(log vlog.Printer) error { +func (opt *DatabaseOptions) setUsePasswordAndValidateUsernameIfNeeded(log vlog.Printer) error { // when password is specified, // we will use username/password to call https endpoints opt.usePassword = false @@ -272,16 +274,11 @@ func (opt *DatabaseOptions) setUsePassword(log vlog.Printer) error { return nil } -func (opt *DatabaseOptions) setUsePasswordForLocalDBConnection(log vlog.Printer) error { +func (opt *DatabaseOptions) setUsePassword(_ vlog.Printer) error { opt.usePassword = false if opt.Password != nil { opt.usePassword = true } - // username is always required when local db connection is made - err := opt.validateUserName(log) - if err != nil { - return err - } return nil }