From 6ff223aae1c8dc5c61aae7be04362a3e31af836b Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 24 Jul 2024 10:12:54 -0600 Subject: [PATCH 01/67] Removed 'dora' API --- internal/api/dora/dora.go | 75 --------------------------------------- 1 file changed, 75 deletions(-) delete mode 100644 internal/api/dora/dora.go diff --git a/internal/api/dora/dora.go b/internal/api/dora/dora.go deleted file mode 100644 index b113cf6..0000000 --- a/internal/api/dora/dora.go +++ /dev/null @@ -1,75 +0,0 @@ -package dora - -import ( - "encoding/json" - "fmt" - - "github.com/OpenCHAMI/magellan/internal/util" - - "github.com/jmoiron/sqlx" -) - -const ( - Host = "http://localhost" - DbType = "sqlite3" - DbPath = "../data/assets.db" - BaseEndpoint = "/v1" - Port = 8000 -) - -type ScannedResult struct { - id string - site any - cidr string - ip string - port int - protocol string - scanner string - state string - updated string -} - -func makeEndpointUrl(endpoint string) string { - return Host + ":" + fmt.Sprint(Port) + BaseEndpoint + endpoint -} - -// Scan for BMC assets uing dora scanner -func ScanForAssets() error { - - return nil -} - -// Query dora API to get scanned ports -func QueryScannedPorts() error { - // Perform scan and collect from dora server - url := makeEndpointUrl("/scanned_ports") - _, body, err := util.MakeRequest(nil, url, "GET", nil, nil) - if err != nil { - return fmt.Errorf("failed todiscover assets: %v", err) - } - - // get data from JSON - var res map[string]any - if err := json.Unmarshal(body, &res); err != nil { - return fmt.Errorf("failed tounmarshal response body: %v", err) - } - data := res["data"] - - fmt.Println(data) - - return nil -} - -// Loads scanned ports directly from DB -func LoadScannedPortsFromDB(dbPath string, dbType string) { - db, _ := sqlx.Open(dbType, dbPath) - sql := `SELECT * FROM scanned_port WHERE state='open'` - rows, _ := db.Query(sql) - for rows.Next() { - var r ScannedResult - rows.Scan( - &r.id, &r.site, &r.cidr, &r.ip, &r.port, &r.protocol, &r.scanner, - &r.state, &r.updated, - ) - } -} From 6873ffd1cb0d0fda0670f3cf83385cfdebfcdd27 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 24 Jul 2024 10:13:57 -0600 Subject: [PATCH 02/67] Moved SMD-related API to pkg --- {internal/api => pkg}/smd/smd.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {internal/api => pkg}/smd/smd.go (100%) diff --git a/internal/api/smd/smd.go b/pkg/smd/smd.go similarity index 100% rename from internal/api/smd/smd.go rename to pkg/smd/smd.go From 30787702473203a930bb24498cbffc838fd4ab03 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 24 Jul 2024 10:15:06 -0600 Subject: [PATCH 03/67] Removed commented out code --- internal/collect.go | 132 +++++--------------------------------------- 1 file changed, 14 insertions(+), 118 deletions(-) diff --git a/internal/collect.go b/internal/collect.go index f5e6480..17f7815 100644 --- a/internal/collect.go +++ b/internal/collect.go @@ -442,7 +442,6 @@ func CollectSystems(c *gofish.APIClient, q *QueryParams) ([]map[string]any, erro } var systems []map[string]any - for _, system := range rfSystems { eths, err := system.EthernetInterfaces() if err != nil { @@ -454,15 +453,21 @@ func CollectSystems(c *gofish.APIClient, q *QueryParams) ([]map[string]any, erro if q.Verbose { fmt.Printf("no system ethernet interfaces found...trying to get from managers interface\n") } - for _, managerLink := range system.ManagedBy { - // try getting ethernet interface from all managers until one is found - eths, err = redfish.ListReferencedEthernetInterfaces(c, managerLink+"/EthernetInterfaces") - if err != nil { - return nil, fmt.Errorf("failed to get system manager ethernet interfaces: %v", err) - } - if len(eths) > 0 { - break + + managedBy, err := system.ManagedBy() + if err == nil { + for _, managerLink := range system.ManagedBy { + // try getting ethernet interface from all managers until one is found + eths, err = redfish.ListReferencedEthernetInterfaces(c, managerLink+"/EthernetInterfaces") + if err != nil { + return nil, fmt.Errorf("failed to get system manager ethernet interfaces: %v", err) + } + if len(eths) > 0 { + break + } } + } else { + } } @@ -494,115 +499,6 @@ func CollectSystems(c *gofish.APIClient, q *QueryParams) ([]map[string]any, erro }) } - // do manual requests if systems is empty to only get necessary info as last resort - // /redfish/v1/Systems - - // /redfish/v1/Systems/Members - // /redfish/v1/Systems/ - // fmt.Printf("system count: %d\n", len(systems)) - // if len(systems) == 0 { - // url := baseRedfishUrl(q) + "/Systems" - // if q.Verbose { - // fmt.Printf("%s\n", url) - // } - // res, body, err := util.MakeRequest(nil, url, "GET", nil, nil) - // if err != nil { - // return nil, fmt.Errorf("failed to make request: %v", err) - // } else if res.StatusCode != http.StatusOK { - // return nil, fmt.Errorf("request returned status code %d", res.StatusCode) - // } - - // // sweet syntatic sugar type aliases - // type System = map[string]any - // type Member = map[string]string - - // // get all the systems - // var ( - // tempSystems System - // interfaces []*redfish.EthernetInterface - // errList []error - // ) - // err = json.Unmarshal(body, &tempSystems) - // if err != nil { - // return nil, fmt.Errorf("failed to unmarshal systems: %v", err) - // } - - // // then, get all the members within a system - // members, ok := tempSystems["Members"] - // if ok { - // for _, member := range members.([]Member) { - // id, ok := member["@odata.id"] - // if ok { - // // /redfish/v1/Systems/Self (or whatever) - // // memberEndpoint := fmt.Sprintf("%s%s", url, id) - // // res, body, err := util.MakeRequest(nil, baseRedfishUrl(q)+memberEndpoint, http.MethodGet, nil, nil) - // // if err != nil { - // // continue - // // } else if res.StatusCode != http.StatusOK { - // // continue - // // } - // // TODO: extract EthernetInterfaces from Systems then query - - // // get all of the ethernet interfaces in our systems - // ethernetInterface, err := redfish.ListReferencedEthernetInterfaces(c, id+"/EthernetInterfaces/") - // if err != nil { - // errList = append(errList, err) - // continue - // } - // interfaces = append(interfaces, ethernetInterface...) - // } else { - // return nil, fmt.Errorf("no ID found for member") - // } - // if util.HasErrors(errList) { - // return nil, util.FormatErrorList(errList) - // } - // } - // i, err := json.Marshal(interfaces) - // if err != nil { - // return nil, fmt.Errorf("failed to unmarshal interface: %v", err) - // } - // temp = append(temp, map[string]any{ - // "Data": nil, - // "EthernetInterfaces": string(i), - // }) - // } else { - // return nil, fmt.Errorf("no members found in systems") - // } - - // } else { - // b, err := json.Marshal(systems) - // if err != nil { - // fmt.Printf("failed to marshal systems: %v", err) - // } - // fmt.Printf("systems: %v\n", string(b)) - - // // query the system's ethernet interfaces - // // var temp []map[string]any - // var errList []error - // for _, system := range systems { - // interfaces, err := CollectEthernetInterfaces(c, q, system.ID) - // if err != nil { - // errList = append(errList, fmt.Errorf("failed to collect ethernet interface: %v", err)) - // continue - // } - // var i map[string]any - // err = json.Unmarshal(interfaces, &i) - // if err != nil { - // return nil, fmt.Errorf("failed to unmarshal interface: %v", err) - // } - // temp = append(temp, map[string]any{ - // "Data": system, - // "EthernetInterfaces": i["EthernetInterfaces"], - // }) - // } - // if util.HasErrors(errList) { - // err = util.FormatErrorList(errList) - // if err != nil { - // return nil, fmt.Errorf("multiple errors occurred: %v", err) - // } - // } - // } - return systems, nil } From a65c640aabef4302c4de84b7e763a32a3ea43829 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 24 Jul 2024 10:19:01 -0600 Subject: [PATCH 04/67] Removed unused functions in collect.go --- internal/collect.go | 137 -------------------------------------------- 1 file changed, 137 deletions(-) diff --git a/internal/collect.go b/internal/collect.go index 17f7815..5e3bb14 100644 --- a/internal/collect.go +++ b/internal/collect.go @@ -228,101 +228,6 @@ func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) err return nil } -// CollectInventory() fetches inventory data from all of the BMC hosts provided. -func CollectInventory(client *bmclib.Client, q *QueryParams) ([]byte, error) { - // open BMC session and update driver registry - ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(q.Timeout)) - client.Registry.FilterForCompatible(ctx) - err := client.PreferProvider(q.Preferred).Open(ctx) - if err != nil { - ctxCancel() - return nil, fmt.Errorf("failed to open client: %v", err) - } - - inventory, err := client.Inventory(ctx) - if err != nil { - ctxCancel() - return nil, fmt.Errorf("failed to get inventory: %v", err) - } - - // retrieve inventory data - data := map[string]any{"Inventory": inventory} - b, err := json.MarshalIndent(data, "", " ") - if err != nil { - ctxCancel() - return nil, fmt.Errorf("failed to marshal JSON: %v", err) - } - - ctxCancel() - return b, nil -} - -// TODO: DELETE ME!!! -func CollectPowerState(client *bmclib.Client, q *QueryParams) ([]byte, error) { - ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(q.Timeout)) - client.Registry.FilterForCompatible(ctx) - err := client.PreferProvider(q.Preferred).Open(ctx) - if err != nil { - ctxCancel() - return nil, fmt.Errorf("failed to open client: %v", err) - } - - powerState, err := client.GetPowerState(ctx) - if err != nil { - ctxCancel() - return nil, fmt.Errorf("failed to get inventory: %v", err) - } - - // retrieve inventory data - data := map[string]any{"PowerState": powerState} - b, err := json.MarshalIndent(data, "", " ") - if err != nil { - ctxCancel() - return nil, fmt.Errorf("failed to marshal JSON: %v", err) - } - - ctxCancel() - return b, nil - -} - -// TODO: DELETE ME!!! -func CollectUsers(client *bmclib.Client, q *QueryParams) ([]byte, error) { - // open BMC session and update driver registry - ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(q.Timeout)) - client.Registry.FilterForCompatible(ctx) - err := client.Open(ctx) - if err != nil { - ctxCancel() - return nil, fmt.Errorf("failed to connect to bmc: %v", err) - } - - defer client.Close(ctx) - - users, err := client.ReadUsers(ctx) - if err != nil { - ctxCancel() - return nil, fmt.Errorf("failed to get users: %v", err) - } - - // retrieve inventory data - data := map[string]any{"Users": users} - b, err := json.MarshalIndent(data, "", " ") - if err != nil { - ctxCancel() - return nil, fmt.Errorf("failed to marshal JSON: %v", err) - } - - ctxCancel() - return b, nil -} - -// TODO: DELETE ME!!! -func CollectBios(client *bmclib.Client, q *QueryParams) ([]byte, error) { - b, err := makeRequest(client, client.GetBiosConfiguration, q.Timeout) - return b, err -} - // CollectEthernetInterfaces() collects all of the ethernet interfaces found // from all systems from under the "/redfish/v1/Systems" endpoint. // @@ -398,32 +303,6 @@ func CollectChassis(c *gofish.APIClient, q *QueryParams) ([]map[string]any, erro return chassis, nil } -// TODO: DELETE ME!!! -func CollectStorage(c *gofish.APIClient, q *QueryParams) ([]byte, error) { - systems, err := c.Service.StorageSystems() - if err != nil { - return nil, fmt.Errorf("failed to query storage systems (%v:%v): %v", q.Host, q.Port, err) - } - - services, err := c.Service.StorageServices() - if err != nil { - return nil, fmt.Errorf("failed to query storage services (%v:%v): %v", q.Host, q.Port, err) - } - - data := map[string]any{ - "Storage": map[string]any{ - "Systems": systems, - "Services": services, - }, - } - b, err := json.MarshalIndent(data, "", " ") - if err != nil { - return nil, fmt.Errorf("failed to marshal JSON: %v", err) - } - - return b, nil -} - // CollectSystems pulls system information from each BMC node via Redfish using the // `gofish` library. // @@ -502,22 +381,6 @@ func CollectSystems(c *gofish.APIClient, q *QueryParams) ([]map[string]any, erro return systems, nil } -// TODO: DELETE ME!!! -func CollectRegisteries(c *gofish.APIClient, q *QueryParams) ([]byte, error) { - registries, err := c.Service.Registries() - if err != nil { - return nil, fmt.Errorf("failed to query storage systems (%v:%v): %v", q.Host, q.Port, err) - } - - data := map[string]any{"Registries": registries} - b, err := json.MarshalIndent(data, "", " ") - if err != nil { - return nil, fmt.Errorf("failed to marshal JSON: %v", err) - } - - return b, nil -} - // TODO: MAYBE DELETE??? func CollectProcessors(q *QueryParams) ([]byte, error) { url := baseRedfishUrl(q) + "/Systems" From f498d07aa20fbf4797db724646f53a54ef7ac4bc Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 24 Jul 2024 10:22:48 -0600 Subject: [PATCH 05/67] Fixed small issue with command string --- cmd/list.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/list.go b/cmd/list.go index 89cd847..ed02dfc 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -19,10 +19,10 @@ var listCmd = &cobra.Command{ Use: "list", Short: "List information stored in cache from a scan", Long: "Prints all of the host and associated data found from performing a scan.\n" + - "See the 'scan' command on how to perform a scan.\n\n" + - "Examples:\n" + - " magellan list\n" + - " magellan list " + "See the 'scan' command on how to perform a scan.\n\n" + + "Examples:\n" + + " magellan list\n" + + " magellan list --cache ./assets.db", Run: func(cmd *cobra.Command, args []string) { probeResults, err := sqlite.GetProbeResults(cachePath) if err != nil { From d73575ab05f73670ca0bfddb5c5ece43bd799cae Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 24 Jul 2024 10:24:12 -0600 Subject: [PATCH 06/67] Fixed imports and removed unused query params --- cmd/collect.go | 2 +- cmd/root.go | 2 +- cmd/update.go | 14 ++++++-------- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/cmd/collect.go b/cmd/collect.go index 3641a55..eb4574b 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -5,9 +5,9 @@ import ( "os/user" magellan "github.com/OpenCHAMI/magellan/internal" - "github.com/OpenCHAMI/magellan/internal/api/smd" "github.com/OpenCHAMI/magellan/internal/db/sqlite" "github.com/OpenCHAMI/magellan/internal/log" + "github.com/OpenCHAMI/magellan/pkg/smd" "github.com/cznic/mathutil" "github.com/sirupsen/logrus" "github.com/spf13/cobra" diff --git a/cmd/root.go b/cmd/root.go index 40f7407..6571d9d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -21,7 +21,7 @@ import ( "os/user" magellan "github.com/OpenCHAMI/magellan/internal" - "github.com/OpenCHAMI/magellan/internal/api/smd" + "github.com/OpenCHAMI/magellan/pkg/smd" "github.com/spf13/cobra" "github.com/spf13/viper" ) diff --git a/cmd/update.go b/cmd/update.go index 87f2282..9613f64 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -36,14 +36,12 @@ var updateCmd = &cobra.Command{ Component: component, TransferProtocol: transferProtocol, QueryParams: magellan.QueryParams{ - Drivers: []string{"redfish"}, - Preferred: "redfish", - Protocol: protocol, - Host: host, - Username: username, - Password: password, - Timeout: timeout, - Port: port, + Protocol: protocol, + Host: host, + Username: username, + Password: password, + Timeout: timeout, + Port: port, }, } From d877f9b18f9e4a39f6e8e37071dd63816fb3cb85 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 24 Jul 2024 10:28:23 -0600 Subject: [PATCH 07/67] Removed unused query params --- internal/collect.go | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/internal/collect.go b/internal/collect.go index 5e3bb14..24b5a76 100644 --- a/internal/collect.go +++ b/internal/collect.go @@ -37,21 +37,18 @@ const ( // takes an object as an argument. However, the implementation may not // use all of the properties within the object. type QueryParams struct { - Host string // set by the 'host' flag - Port int // set by the 'port' flag - Protocol string // set by the 'protocol' flag - Username string // set the BMC username with the 'username' flag - Password string // set the BMC password with the 'password' flag - Drivers []string // DEPRECATED: TO BE REMOVED!!! - Concurrency int // set the of concurrent jobs with the 'concurrency' flag - Preferred string // DEPRECATED: TO BE REMOVED!!! - Timeout int // set the timeout with the 'timeout' flag - CaCertPath string // set the cert path with the 'cacert' flag - Verbose bool // set whether to include verbose output with 'verbose' flag - IpmitoolPath string // DEPRECATED: TO BE REMOVE!!! - OutputPath string // set the path to save output with 'output' flag - ForceUpdate bool // set whether to force updating SMD with 'force-update' flag - AccessToken string // set the access token to include in request with 'access-token' flag + Host string // set by the 'host' flag + Port int // set by the 'port' flag + Protocol string // set by the 'protocol' flag + Username string // set the BMC username with the 'username' flag + Password string // set the BMC password with the 'password' flag + Concurrency int // set the of concurrent jobs with the 'concurrency' flag + Timeout int // set the timeout with the 'timeout' flag + CaCertPath string // set the cert path with the 'cacert' flag + Verbose bool // set whether to include verbose output with 'verbose' flag + OutputPath string // set the path to save output with 'output' flag + ForceUpdate bool // set whether to force updating SMD with 'force-update' flag + AccessToken string // set the access token to include in request with 'access-token' flag } // This is the main function used to collect information from the BMC nodes via Redfish. From fe3c3391955d30d59cc6fbf0b07cab990865ffa5 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 24 Jul 2024 10:34:28 -0600 Subject: [PATCH 08/67] Corrected the username/password flag names --- cmd/update.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/update.go b/cmd/update.go index 9613f64..489cd78 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -74,8 +74,8 @@ var updateCmd = &cobra.Command{ func init() { updateCmd.Flags().StringVar(&host, "bmc-host", "", "set the BMC host") updateCmd.Flags().IntVar(&port, "bmc-port", 443, "set the BMC port") - updateCmd.Flags().StringVar(&username, "user", "", "set the BMC user") - updateCmd.Flags().StringVar(&password, "pass", "", "set the BMC password") + updateCmd.Flags().StringVar(&username, "username", "", "set the BMC user") + updateCmd.Flags().StringVar(&password, "password", "", "set the BMC password") updateCmd.Flags().StringVar(&transferProtocol, "transfer-protocol", "HTTP", "set the transfer protocol") updateCmd.Flags().StringVar(&protocol, "protocol", "https", "set the Redfish protocol") updateCmd.Flags().StringVar(&firmwareUrl, "firmware-url", "", "set the path to the firmware") @@ -85,8 +85,8 @@ func init() { viper.BindPFlag("host", updateCmd.Flags().Lookup("host")) viper.BindPFlag("port", updateCmd.Flags().Lookup("port")) - viper.BindPFlag("username", updateCmd.Flags().Lookup("user")) - viper.BindPFlag("password", updateCmd.Flags().Lookup("pass")) + viper.BindPFlag("username", updateCmd.Flags().Lookup("username")) + viper.BindPFlag("password", updateCmd.Flags().Lookup("password")) viper.BindPFlag("transfer-protocol", updateCmd.Flags().Lookup("transfer-protocol")) viper.BindPFlag("protocol", updateCmd.Flags().Lookup("protocol")) viper.BindPFlag("firmware.url", updateCmd.Flags().Lookup("firmware.url")) From aefce13f577f5cd8f9402c9a2f807af3a35905d2 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 24 Jul 2024 11:52:12 -0600 Subject: [PATCH 09/67] Refactored/reorganized utils --- internal/util/auth.go | 37 ++++++++++ internal/util/error.go | 25 +++++++ internal/util/net.go | 64 +++++++++++++++++ internal/util/path.go | 70 +++++++++++++++++++ internal/util/util.go | 151 ----------------------------------------- 5 files changed, 196 insertions(+), 151 deletions(-) create mode 100644 internal/util/auth.go create mode 100644 internal/util/error.go create mode 100644 internal/util/net.go create mode 100644 internal/util/path.go delete mode 100644 internal/util/util.go diff --git a/internal/util/auth.go b/internal/util/auth.go new file mode 100644 index 0000000..98ef88c --- /dev/null +++ b/internal/util/auth.go @@ -0,0 +1,37 @@ +package util + +import ( + "fmt" + "os" + + "github.com/spf13/viper" +) + +// LoadAccessToken() tries to load a JWT string from an environment +// variable, file, or config in that order. If loading the token +// fails with one options, it will fallback to the next option until +// all options are exhausted. +// +// Returns a token as a string with no error if successful. +// Alternatively, returns an empty string with an error if a token is +// not able to be loaded. +func LoadAccessToken(path string) (string, error) { + // try to load token from env var + testToken := os.Getenv("ACCESS_TOKEN") + if testToken != "" { + return testToken, nil + } + + // try reading access token from a file + b, err := os.ReadFile(path) + if err == nil { + return string(b), nil + } + + // TODO: try to load token from config + testToken = viper.GetString("access_token") + if testToken != "" { + return testToken, nil + } + return "", fmt.Errorf("failed toload token from environment variable, file, or config") +} diff --git a/internal/util/error.go b/internal/util/error.go new file mode 100644 index 0000000..addca80 --- /dev/null +++ b/internal/util/error.go @@ -0,0 +1,25 @@ +package util + +import "fmt" + +// FormatErrorList() is a wrapper function that unifies error list formatting +// and makes printing error lists consistent. +// +// NOTE: The error returned IS NOT an error in itself and may be a bit misleading. +// Instead, it is a single condensed error composed of all of the errors included +// in the errList argument. +func FormatErrorList(errList []error) error { + var err error + for i, e := range errList { + err = fmt.Errorf("\t[%d] %v\n", i, e) + i += 1 + } + return err +} + +// HasErrors() is a simple wrapper function to check if an error list contains +// errors. Having a function that clearly states its purpose helps to improve +// readibility although it may seem pointless. +func HasErrors(errList []error) bool { + return len(errList) > 0 +} diff --git a/internal/util/net.go b/internal/util/net.go new file mode 100644 index 0000000..cdc18db --- /dev/null +++ b/internal/util/net.go @@ -0,0 +1,64 @@ +package util + +import ( + "bytes" + "crypto/tls" + "fmt" + "io" + "net" + "net/http" +) + +// GetNextIP() returns the next IP address, but does not account +// for net masks. +func GetNextIP(ip *net.IP, inc uint) *net.IP { + if ip == nil { + return &net.IP{} + } + i := ip.To4() + v := uint(i[0])<<24 + uint(i[1])<<16 + uint(i[2])<<8 + uint(i[3]) + v += inc + v3 := byte(v & 0xFF) + v2 := byte((v >> 8) & 0xFF) + v1 := byte((v >> 16) & 0xFF) + v0 := byte((v >> 24) & 0xFF) + // return &net.IP{[]byte{v0, v1, v2, v3}} + r := net.IPv4(v0, v1, v2, v3) + return &r +} + +// MakeRequest() is a wrapper function that condenses simple HTTP +// requests done to a single call. It expects an optional HTTP client, +// URL, HTTP method, request body, and request headers. This function +// is useful when making many requests where only these few arguments +// are changing. +// +// Returns a HTTP response object, response body as byte array, and any +// error that may have occurred with making the request. +func MakeRequest(client *http.Client, url string, httpMethod string, body []byte, headers map[string]string) (*http.Response, []byte, error) { + // use defaults if no client provided + if client == nil { + client = http.DefaultClient + client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + req, err := http.NewRequest(httpMethod, url, bytes.NewBuffer(body)) + if err != nil { + return nil, nil, fmt.Errorf("failed to create new HTTP request: %v", err) + } + req.Header.Add("User-Agent", "magellan") + for k, v := range headers { + req.Header.Add(k, v) + } + res, err := client.Do(req) + if err != nil { + return nil, nil, fmt.Errorf("failed to make request: %v", err) + } + b, err := io.ReadAll(res.Body) + res.Body.Close() + if err != nil { + return nil, nil, fmt.Errorf("failed to read response body: %v", err) + } + return res, b, err +} diff --git a/internal/util/path.go b/internal/util/path.go new file mode 100644 index 0000000..06611b6 --- /dev/null +++ b/internal/util/path.go @@ -0,0 +1,70 @@ +package util + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// PathExists() is a wrapper function that simplifies checking +// if a file or directory already exists at the provided path. +// +// Returns whether the path exists and no error if successful, +// otherwise, it returns false with an error. +func PathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +// SplitPathForViper() is an utility function to split a path into 3 parts: +// - directory +// - filename +// - extension +// The intent was to break a path into a format that's more easily consumable +// by spf13/viper's API. See the "LoadConfig()" function in internal/config.go +// for more details. +// +// TODO: Rename function to something more generalized. +func SplitPathForViper(path string) (string, string, string) { + filename := filepath.Base(path) + ext := filepath.Ext(filename) + return filepath.Dir(path), strings.TrimSuffix(filename, ext), strings.TrimPrefix(ext, ".") +} + +// MakeOutputDirectory() creates a new directory at the path argument if +// the path does not exist +// +// TODO: Refactor this function for hive partitioning or possibly move into +// the logging package. +// TODO: Add an option to force overwriting the path. +func MakeOutputDirectory(path string) (string, error) { + // get the current data + time using Go's stupid formatting + t := time.Now() + dirname := t.Format("2006-01-01 15:04:05") + final := path + "/" + dirname + + // check if path is valid and directory + pathExists, err := PathExists(final) + if err != nil { + return final, fmt.Errorf("failed to check for existing path: %v", err) + } + if pathExists { + // make sure it is directory with 0o644 permissions + return final, fmt.Errorf("found existing path: %v", final) + } + + // create directory with data + time + err = os.MkdirAll(final, 0766) + if err != nil { + return final, fmt.Errorf("failed to make directory: %v", err) + } + return final, nil +} diff --git a/internal/util/util.go b/internal/util/util.go deleted file mode 100644 index 6817f4a..0000000 --- a/internal/util/util.go +++ /dev/null @@ -1,151 +0,0 @@ -package util - -import ( - "bytes" - "crypto/tls" - "fmt" - "io" - "net" - "net/http" - "os" - "path/filepath" - "strings" - "time" -) - -// PathExists() is a wrapper function that simplifies checking -// if a file or directory already exists at the provided path. -// -// Returns whether the path exists and no error if successful, -// otherwise, it returns false with an error. -func PathExists(path string) (bool, error) { - _, err := os.Stat(path) - if err == nil { - return true, nil - } - if os.IsNotExist(err) { - return false, nil - } - return false, err -} - -// GetNextIP() returns the next IP address, but does not account -// for net masks. -func GetNextIP(ip *net.IP, inc uint) *net.IP { - if ip == nil { - return &net.IP{} - } - i := ip.To4() - v := uint(i[0])<<24 + uint(i[1])<<16 + uint(i[2])<<8 + uint(i[3]) - v += inc - v3 := byte(v & 0xFF) - v2 := byte((v >> 8) & 0xFF) - v1 := byte((v >> 16) & 0xFF) - v0 := byte((v >> 24) & 0xFF) - // return &net.IP{[]byte{v0, v1, v2, v3}} - r := net.IPv4(v0, v1, v2, v3) - return &r -} - -// MakeRequest() is a wrapper function that condenses simple HTTP -// requests done to a single call. It expects an optional HTTP client, -// URL, HTTP method, request body, and request headers. This function -// is useful when making many requests where only these few arguments -// are changing. -// -// Returns a HTTP response object, response body as byte array, and any -// error that may have occurred with making the request. -func MakeRequest(client *http.Client, url string, httpMethod string, body []byte, headers map[string]string) (*http.Response, []byte, error) { - // use defaults if no client provided - if client == nil { - client = http.DefaultClient - client.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - } - req, err := http.NewRequest(httpMethod, url, bytes.NewBuffer(body)) - if err != nil { - return nil, nil, fmt.Errorf("failed to create new HTTP request: %v", err) - } - req.Header.Add("User-Agent", "magellan") - for k, v := range headers { - req.Header.Add(k, v) - } - res, err := client.Do(req) - if err != nil { - return nil, nil, fmt.Errorf("failed to make request: %v", err) - } - b, err := io.ReadAll(res.Body) - res.Body.Close() - if err != nil { - return nil, nil, fmt.Errorf("failed to read response body: %v", err) - } - return res, b, err -} - -// MakeOutputDirectory() creates a new directory at the path argument if -// the path does not exist -// -// TODO: Refactor this function for hive partitioning or possibly move into -// the logging package. -// TODO: Add an option to force overwriting the path. -func MakeOutputDirectory(path string) (string, error) { - // get the current data + time using Go's stupid formatting - t := time.Now() - dirname := t.Format("2006-01-01 15:04:05") - final := path + "/" + dirname - - // check if path is valid and directory - pathExists, err := PathExists(final) - if err != nil { - return final, fmt.Errorf("failed to check for existing path: %v", err) - } - if pathExists { - // make sure it is directory with 0o644 permissions - return final, fmt.Errorf("found existing path: %v", final) - } - - // create directory with data + time - err = os.MkdirAll(final, 0766) - if err != nil { - return final, fmt.Errorf("failed to make directory: %v", err) - } - return final, nil -} - -// SplitPathForViper() is an utility function to split a path into 3 parts: -// - directory -// - filename -// - extension -// The intent was to break a path into a format that's more easily consumable -// by spf13/viper's API. See the "LoadConfig()" function in internal/config.go -// for more details. -// -// TODO: Rename function to something more generalized. -func SplitPathForViper(path string) (string, string, string) { - filename := filepath.Base(path) - ext := filepath.Ext(filename) - return filepath.Dir(path), strings.TrimSuffix(filename, ext), strings.TrimPrefix(ext, ".") -} - -// FormatErrorList() is a wrapper function that unifies error list formatting -// and makes printing error lists consistent. -// -// NOTE: The error returned IS NOT an error in itself and may be a bit misleading. -// Instead, it is a single condensed error composed of all of the errors included -// in the errList argument. -func FormatErrorList(errList []error) error { - var err error - for i, e := range errList { - err = fmt.Errorf("\t[%d] %v\n", i, e) - i += 1 - } - return err -} - -// HasErrors() is a simple wrapper function to check if an error list contains -// errors. Having a function that clearly states its purpose helps to improve -// readibility although it may seem pointless. -func HasErrors(errList []error) bool { - return len(errList) > 0 -} From 606a7b47ccc1c963ee30959c3b902f968a3a350e Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 30 Jul 2024 14:00:23 -0600 Subject: [PATCH 10/67] Removed unused code, rename vars, and changed output to use hive partitioning strategy --- internal/collect.go | 431 +++++++------------------------------------- 1 file changed, 64 insertions(+), 367 deletions(-) diff --git a/internal/collect.go b/internal/collect.go index 24b5a76..2666c50 100644 --- a/internal/collect.go +++ b/internal/collect.go @@ -2,27 +2,22 @@ package magellan import ( - "context" - "crypto/tls" "encoding/json" "fmt" - "net/http" "os" "path" "sync" "time" - "github.com/OpenCHAMI/magellan/internal/log" + "github.com/OpenCHAMI/magellan/pkg/client" + "github.com/OpenCHAMI/magellan/pkg/crawler" - "github.com/OpenCHAMI/magellan/internal/api/smd" "github.com/OpenCHAMI/magellan/internal/util" + "github.com/rs/zerolog/log" "github.com/Cray-HPE/hms-xname/xnames" - bmclib "github.com/bmc-toolbox/bmclib/v2" _ "github.com/mattn/go-sqlite3" - "github.com/stmcginnis/gofish" _ "github.com/stmcginnis/gofish" - "github.com/stmcginnis/gofish/redfish" "golang.org/x/exp/slices" ) @@ -39,7 +34,6 @@ const ( type QueryParams struct { Host string // set by the 'host' flag Port int // set by the 'port' flag - Protocol string // set by the 'protocol' flag Username string // set the BMC username with the 'username' flag Password string // set the BMC password with the 'password' flag Concurrency int // set the of concurrent jobs with the 'concurrency' flag @@ -56,44 +50,38 @@ type QueryParams struct { // // Requests can be made to several of the nodes using a goroutine by setting the q.Concurrency // property value between 1 and 255. -func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) error { +func CollectInventory(scannedResults *[]ScannedResult, params *QueryParams) error { // check for available probe states - if probeStates == nil { + if scannedResults == nil { return fmt.Errorf("no probe states found") } - if len(*probeStates) <= 0 { + if len(*scannedResults) <= 0 { return fmt.Errorf("no probe states found") } - // make the output directory to store files - outputPath := path.Clean(q.OutputPath) - outputPath, err := util.MakeOutputDirectory(outputPath) - if err != nil { - l.Log.Errorf("failed to make output directory: %v", err) - } - // collect bmc information asynchronously var ( - offset = 0 - wg sync.WaitGroup - found = make([]string, 0, len(*probeStates)) - done = make(chan struct{}, q.Concurrency+1) - chanProbeState = make(chan ScannedResult, q.Concurrency+1) - client = smd.NewClient( - smd.WithSecureTLS(q.CaCertPath), + offset = 0 + wg sync.WaitGroup + found = make([]string, 0, len(*scannedResults)) + done = make(chan struct{}, params.Concurrency+1) + chanScannedResult = make(chan ScannedResult, params.Concurrency+1) + outputPath = path.Clean(params.OutputPath) + smdClient = client.NewClient( + client.WithSecureTLS(params.CaCertPath), ) ) - wg.Add(q.Concurrency) - for i := 0; i < q.Concurrency; i++ { + wg.Add(params.Concurrency) + for i := 0; i < params.Concurrency; i++ { go func() { for { - ps, ok := <-chanProbeState + sr, ok := <-chanScannedResult if !ok { wg.Done() return } - q.Host = ps.Host - q.Port = ps.Port + params.Host = sr.Host + params.Port = sr.Port // generate custom xnames for bmcs node := xnames.Node{ @@ -104,107 +92,91 @@ func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) err } offset += 1 - gofishClient, err := connectGofish(q) + // TODO: use pkg/crawler to request inventory data via Redfish + systems, err := crawler.CrawlBMC(crawler.CrawlerConfig{ + URI: fmt.Sprintf("https://%s:%d", sr.Host, sr.Port), + Username: params.Username, + Password: params.Password, + Insecure: true, + }) if err != nil { - l.Log.Errorf("failed to connect to BMC (%v:%v): %v", q.Host, q.Port, err) + log.Error().Err(err).Msgf("failed to crawl BMC") } - defer gofishClient.Logout() // data to be sent to smd data := map[string]any{ - "ID": fmt.Sprintf("%v", node.String()[:len(node.String())-2]), - "Type": "", - "Name": "", - "FQDN": ps.Host, - "User": q.Username, - // "Password": q.Pass, + "ID": fmt.Sprintf("%v", node.String()[:len(node.String())-2]), + "Type": "", + "Name": "", + "FQDN": sr.Host, + "User": params.Username, "MACRequired": true, "RediscoverOnUpdate": false, + "Systems": systems, } - // chassis - if gofishClient != nil { - chassis, err := CollectChassis(gofishClient, q) - if err != nil { - l.Log.Errorf("failed to collect chassis: %v", err) - continue - } - data["Chassis"] = chassis - - // systems - systems, err := CollectSystems(gofishClient, q) - if err != nil { - l.Log.Errorf("failed to collect systems: %v", err) - } - data["Systems"] = systems - - // add other fields from systems - if len(systems) > 0 { - system := systems[0]["Data"].(*redfish.ComputerSystem) - if system == nil { - l.Log.Errorf("invalid system data (data is nil)") - } else { - data["Name"] = system.Name - } - } - } else { - l.Log.Errorf("invalid client (client is nil)") - continue - } - - headers := make(map[string]string) - headers["Content-Type"] = "application/json" - - // use access token in authorization header if we have it - if q.AccessToken != "" { - headers["Authorization"] = "Bearer " + q.AccessToken - } + // create and set headers for request + headers := util.HTTPHeader{} + headers.Authorization(params.AccessToken) + headers.ContentType("application/json") body, err := json.MarshalIndent(data, "", " ") if err != nil { - l.Log.Errorf("failed to marshal output to JSON: %v", err) + log.Error().Err(err).Msgf("failed to marshal output to JSON") } - if q.Verbose { + if params.Verbose { fmt.Printf("%v\n", string(body)) } - // write JSON data to file if output path is set + // write JSON data to file if output path is set using hive partitioning strategy if outputPath != "" { - err = os.WriteFile(path.Clean(outputPath+"/"+q.Host+".json"), body, os.ModePerm) + err = os.MkdirAll(outputPath, os.ModeDir) if err != nil { - l.Log.Errorf("failed to write data to file: %v", err) + log.Error().Err(err).Msg("failed to make directory for output") + } else { + // make the output directory to store files + outputPath, err := util.MakeOutputDirectory(outputPath, false) + if err != nil { + log.Error().Msgf("failed to make output directory: %v", err) + } else { + // write the output to the final path + err = os.WriteFile(path.Clean(fmt.Sprintf("%s/%s/%d.json", params.Host, outputPath, time.Now().Unix())), body, os.ModePerm) + if err != nil { + log.Error().Err(err).Msgf("failed to write data to file") + } + } } } // add all endpoints to smd - err = client.AddRedfishEndpoint(body, headers) + err = smdClient.AddRedfishEndpoint(data, headers) if err != nil { - l.Log.Error(err) + log.Error().Err(err).Msgf("failed to add Redfish endpoint") // try updating instead - if q.ForceUpdate { - err = client.UpdateRedfishEndpoint(data["ID"].(string), body, headers) + if params.ForceUpdate { + err = smdClient.UpdateRedfishEndpoint(data["ID"].(string), body, headers) if err != nil { - l.Log.Error(err) + log.Error().Err(err).Msgf("failed to update Redfish endpoint") } } } // got host information, so add to list of already probed hosts - found = append(found, ps.Host) + found = append(found, sr.Host) } }() } // use the found results to query bmc information - for _, ps := range *probeStates { + for _, ps := range *scannedResults { // skip if found info from host foundHost := slices.Index(found, ps.Host) if !ps.State || foundHost >= 0 { continue } - chanProbeState <- ps + chanScannedResult <- ps } // handle goroutine paths @@ -218,288 +190,13 @@ func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) err } }() - close(chanProbeState) + close(chanScannedResult) wg.Wait() close(done) return nil } -// CollectEthernetInterfaces() collects all of the ethernet interfaces found -// from all systems from under the "/redfish/v1/Systems" endpoint. -// -// TODO: This function needs to be refactored entirely...if not deleted -// in favor of using crawler.CrawlBM() instead. -func CollectEthernetInterfaces(c *gofish.APIClient, q *QueryParams, systemID string) ([]byte, error) { - // TODO: add more endpoints to test for ethernet interfaces - // /redfish/v1/Chassis/{ChassisID}/NetworkAdapters/{NetworkAdapterId}/NetworkDeviceFunctions/{NetworkDeviceFunctionId}/EthernetInterfaces/{EthernetInterfaceId} - // /redfish/v1/Managers/{ManagerId}/EthernetInterfaces/{EthernetInterfaceId} - // /redfish/v1/Systems/{ComputerSystemId}/EthernetInterfaces/{EthernetInterfaceId} - // /redfish/v1/Systems/{ComputerSystemId}/OperatingSystem/Containers/EthernetInterfaces/{EthernetInterfaceId} - systems, err := c.Service.Systems() - if err != nil { - return nil, fmt.Errorf("failed to get systems: (%v:%v): %v", q.Host, q.Port, err) - } - - var ( - interfaces []*redfish.EthernetInterface - errList []error - ) - - // get all of the ethernet interfaces in our systems - for _, system := range systems { - system.EthernetInterfaces() - eth, err := redfish.ListReferencedEthernetInterfaces(c, "/redfish/v1/Systems/"+system.ID+"/EthernetInterfaces") - if err != nil { - errList = append(errList, err) - } - - interfaces = append(interfaces, eth...) - } - - // print any report errors - err = util.FormatErrorList(errList) - if util.HasErrors(errList) { - return nil, fmt.Errorf("failed to get ethernet interfaces with %d error(s): \n%v", len(errList), err) - } - - data := map[string]any{"EthernetInterfaces": interfaces} - b, err := json.MarshalIndent(data, "", " ") - if err != nil { - return nil, fmt.Errorf("failed to marshal JSON: %v", err) - } - - return b, nil -} - -// CollectChassis() fetches all chassis related information from each node specified -// via the Redfish API. Like the other collect functions, this function uses the gofish -// library to make requests to each node. Additionally, all of the network adapters found -// are added to the output as well. -// -// Returns a map that represents a Chassis object with NetworkAdapters. -func CollectChassis(c *gofish.APIClient, q *QueryParams) ([]map[string]any, error) { - rfChassis, err := c.Service.Chassis() - if err != nil { - return nil, fmt.Errorf("failed to query chassis (%v:%v): %v", q.Host, q.Port, err) - } - - var chassis []map[string]any - for _, ch := range rfChassis { - networkAdapters, err := ch.NetworkAdapters() - if err != nil { - return nil, fmt.Errorf("failed to get network adapters: %v", err) - } - - chassis = append(chassis, map[string]any{ - "Data": ch, - "NetworkAdapters": networkAdapters, - }) - } - - return chassis, nil -} - -// CollectSystems pulls system information from each BMC node via Redfish using the -// `gofish` library. -// -// The process of collecting this info is as follows: -// 1. check if system has ethernet interfaces -// 1.a. if yes, create system data and ethernet interfaces JSON -// 1.b. if no, try to get data using manager instead -// 2. check if manager has "ManagerForServices" and "EthernetInterfaces" properties -// 2.a. if yes, query both properties to use in next step -// 2.b. for each service, query its data and add the ethernet interfaces -// 2.c. add the system to list of systems to marshal and return -func CollectSystems(c *gofish.APIClient, q *QueryParams) ([]map[string]any, error) { - rfSystems, err := c.Service.Systems() - if err != nil { - return nil, fmt.Errorf("failed to get systems (%v:%v): %v", q.Host, q.Port, err) - } - - var systems []map[string]any - for _, system := range rfSystems { - eths, err := system.EthernetInterfaces() - if err != nil { - return nil, fmt.Errorf("failed to get system ethernet interfaces: %v", err) - } - - // try and get ethernet interfaces through manager if empty - if len(eths) <= 0 { - if q.Verbose { - fmt.Printf("no system ethernet interfaces found...trying to get from managers interface\n") - } - - managedBy, err := system.ManagedBy() - if err == nil { - for _, managerLink := range system.ManagedBy { - // try getting ethernet interface from all managers until one is found - eths, err = redfish.ListReferencedEthernetInterfaces(c, managerLink+"/EthernetInterfaces") - if err != nil { - return nil, fmt.Errorf("failed to get system manager ethernet interfaces: %v", err) - } - if len(eths) > 0 { - break - } - } - } else { - - } - } - - // add network interfaces to system - rfNetworkInterfaces, err := system.NetworkInterfaces() - if err != nil { - return nil, fmt.Errorf("failed to get system network interfaces: %v", err) - } - - // get the network adapter ID for each network interface - var networkInterfaces []map[string]any - for _, rfNetworkInterface := range rfNetworkInterfaces { - networkAdapter, err := rfNetworkInterface.NetworkAdapter() - if err != nil { - return nil, fmt.Errorf("failed to get network adapter: %v", err) - } - - networkInterfaces = append(networkInterfaces, map[string]any{ - "Data": rfNetworkInterface, - "NetworkAdapterId": networkAdapter.ID, - }) - } - - // add system to collection of systems - systems = append(systems, map[string]any{ - "Data": system, - "EthernetInterfaces": eths, - "NetworkInterfaces": networkInterfaces, - }) - } - - return systems, nil -} - -// TODO: MAYBE DELETE??? -func CollectProcessors(q *QueryParams) ([]byte, error) { - url := baseRedfishUrl(q) + "/Systems" - res, body, err := util.MakeRequest(nil, url, "GET", nil, nil) - if err != nil { - return nil, fmt.Errorf("something went wrong: %v", err) - } else if res == nil { - return nil, fmt.Errorf("no response returned (url: %s)", url) - } else if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("returned status code %d", res.StatusCode) - } - - // convert to not get base64 string - var procs map[string]json.RawMessage - var members []map[string]any - err = json.Unmarshal(body, &procs) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal processors: %v", err) - } - err = json.Unmarshal(procs["Members"], &members) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal processor members: %v", err) - } - - // request data about each processor member on node - for _, member := range members { - var oid = member["@odata.id"].(string) - var infoUrl = url + oid - res, _, err := util.MakeRequest(nil, infoUrl, "GET", nil, nil) - if err != nil { - return nil, fmt.Errorf("something went wrong: %v", err) - } else if res == nil { - return nil, fmt.Errorf("no response returned (url: %s)", url) - } else if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("returned status code %d", res.StatusCode) - } - } - - data := map[string]any{"Processors": procs} - b, err := json.MarshalIndent(data, "", " ") - if err != nil { - return nil, fmt.Errorf("failed to marshal JSON: %v", err) - } - - return b, nil -} - -func connectGofish(q *QueryParams) (*gofish.APIClient, error) { - config, err := makeGofishConfig(q) - if err != nil { - return nil, fmt.Errorf("failed to make gofish config: %v", err) - } - c, err := gofish.Connect(config) - if err != nil { - return nil, fmt.Errorf("failed to connect to redfish endpoint: %v", err) - } - if c != nil { - c.Service.ProtocolFeaturesSupported = gofish.ProtocolFeaturesSupported{ - ExpandQuery: gofish.Expand{ - ExpandAll: true, - Links: true, - }, - } - } - return c, err -} - -func makeGofishConfig(q *QueryParams) (gofish.ClientConfig, error) { - var ( - client = &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, - } - url = baseRedfishUrl(q) - ) - return gofish.ClientConfig{ - Endpoint: url, - Username: q.Username, - Password: q.Password, - Insecure: true, - TLSHandshakeTimeout: q.Timeout, - HTTPClient: client, - // MaxConcurrentRequests: int64(q.Threads), // NOTE: this was added in latest version of gofish - }, nil -} - -func makeRequest[T any](client *bmclib.Client, fn func(context.Context) (T, error), timeout int) ([]byte, error) { - ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(timeout)) - client.Registry.FilterForCompatible(ctx) - err := client.Open(ctx) - if err != nil { - ctxCancel() - return nil, fmt.Errorf("failed to open client: %v", err) - } - - defer client.Close(ctx) - - response, err := fn(ctx) - if err != nil { - ctxCancel() - return nil, fmt.Errorf("failed to get response: %v", err) - } - - ctxCancel() - return makeJson(response) -} - -func makeJson(object any) ([]byte, error) { - b, err := json.MarshalIndent(object, "", " ") - if err != nil { - return nil, fmt.Errorf("failed to marshal JSON: %v", err) - } - return []byte(b), nil -} - func baseRedfishUrl(q *QueryParams) string { - url := fmt.Sprintf("%s://", q.Protocol) - if q.Username != "" && q.Password != "" { - url += fmt.Sprintf("%s:%s@", q.Username, q.Password) - } - return fmt.Sprintf("%s%s:%d", url, q.Host, q.Port) + return fmt.Sprintf("%s:%d", q.Host, q.Port) } From b27e6b6a736296f138a73342d5437eb136b53d25 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 30 Jul 2024 14:02:12 -0600 Subject: [PATCH 11/67] Renamed vars and switched to use zerolog --- cmd/collect.go | 36 +++++++++++++++------------------- cmd/crawl.go | 5 +++-- cmd/list.go | 10 +++++++--- cmd/login.go | 29 +++++++++++++++------------- cmd/root.go | 35 +++------------------------------ cmd/scan.go | 4 +--- cmd/update.go | 52 ++++++++++++++++++++++---------------------------- 7 files changed, 68 insertions(+), 103 deletions(-) diff --git a/cmd/collect.go b/cmd/collect.go index eb4574b..670dca2 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -6,10 +6,10 @@ import ( magellan "github.com/OpenCHAMI/magellan/internal" "github.com/OpenCHAMI/magellan/internal/db/sqlite" - "github.com/OpenCHAMI/magellan/internal/log" - "github.com/OpenCHAMI/magellan/pkg/smd" + "github.com/OpenCHAMI/magellan/internal/util" + "github.com/OpenCHAMI/magellan/pkg/client" "github.com/cznic/mathutil" - "github.com/sirupsen/logrus" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -30,36 +30,32 @@ var collectCmd = &cobra.Command{ " magellan collect --cache ./assets.db --output ./logs --timeout 30 --cacert cecert.pem\n" + " magellan collect --host smd.example.com --port 27779 --username username --password password", Run: func(cmd *cobra.Command, args []string) { - // make application logger - l := log.NewLogger(logrus.New(), logrus.DebugLevel) - // get probe states stored in db from scan - probeStates, err := sqlite.GetProbeResults(cachePath) + scannedResults, err := sqlite.GetScannedResults(cachePath) if err != nil { - l.Log.Errorf("failed toget states: %v", err) + log.Error().Err(err).Msgf("failed to get scanned results from cache") } // try to load access token either from env var, file, or config if var not set if accessToken == "" { var err error - accessToken, err = LoadAccessToken() + accessToken, err = util.LoadAccessToken(tokenPath) if err != nil { - l.Log.Errorf("failed to load access token: %v", err) + log.Error().Err(err).Msgf("failed to load access token") } } if verbose { - fmt.Printf("access token: %v\n", accessToken) + log.Debug().Str("Access Token", accessToken) } // if concurrency <= 0 { - concurrency = mathutil.Clamp(len(probeStates), 1, 255) + concurrency = mathutil.Clamp(len(scannedResults), 1, 255) } q := &magellan.QueryParams{ Username: username, Password: password, - Protocol: protocol, Timeout: timeout, Concurrency: concurrency, Verbose: verbose, @@ -68,23 +64,21 @@ var collectCmd = &cobra.Command{ ForceUpdate: forceUpdate, AccessToken: accessToken, } - err = magellan.CollectAll(&probeStates, l, q) + err = magellan.CollectInventory(&scannedResults, q) if err != nil { - l.Log.Errorf("failed to collect data: %v", err) + log.Error().Err(err).Msgf("failed to collect data") } // add necessary headers for final request (like token) - headers := make(map[string]string) - if q.AccessToken != "" { - headers["Authorization"] = "Bearer " + q.AccessToken - } + header := util.HTTPHeader{} + header.Authorization(q.AccessToken) }, } func init() { currentUser, _ = user.Current() - collectCmd.PersistentFlags().StringVar(&smd.Host, "host", smd.Host, "set the host to the SMD API") - collectCmd.PersistentFlags().IntVarP(&smd.Port, "port", "p", smd.Port, "set the port to the SMD API") + collectCmd.PersistentFlags().StringVar(&client.Host, "host", client.Host, "set the host to the SMD API") + collectCmd.PersistentFlags().IntVarP(&client.Port, "port", "p", client.Port, "set the port to the SMD API") collectCmd.PersistentFlags().StringVar(&username, "username", "", "set the BMC user") collectCmd.PersistentFlags().StringVar(&password, "password", "", "set the BMC password") collectCmd.PersistentFlags().StringVar(&protocol, "protocol", "https", "set the protocol used to query") diff --git a/cmd/crawl.go b/cmd/crawl.go index a4956bb..ba94636 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -19,8 +19,9 @@ var crawlCmd = &cobra.Command{ Short: "Crawl a single BMC for inventory information", Long: "Crawl a single BMC for inventory information\n" + "\n" + - "Example:\n" + - " magellan crawl https://bmc.example.com", + "Examples:\n" + + " magellan crawl https://bmc.example.com\n" + + " magellan crawl https://bmc.example.com -i -u username -p password", Args: func(cmd *cobra.Command, args []string) error { // Validate that the only argument is a valid URI if err := cobra.ExactArgs(1)(cmd, args); err != nil { diff --git a/cmd/list.go b/cmd/list.go index ed02dfc..e1b254d 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -7,6 +7,7 @@ import ( "time" "github.com/OpenCHAMI/magellan/internal/db/sqlite" + "github.com/rs/zerolog/log" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -24,16 +25,19 @@ var listCmd = &cobra.Command{ " magellan list\n" + " magellan list --cache ./assets.db", Run: func(cmd *cobra.Command, args []string) { - probeResults, err := sqlite.GetProbeResults(cachePath) + scannedResults, err := sqlite.GetScannedResults(cachePath) if err != nil { logrus.Errorf("failed toget probe results: %v\n", err) } format = strings.ToLower(format) if format == "json" { - b, _ := json.Marshal(probeResults) + b, err := json.Marshal(scannedResults) + if err != nil { + log.Error().Err(err).Msgf("failed to unmarshal scanned results") + } fmt.Printf("%s\n", string(b)) } else { - for _, r := range probeResults { + for _, r := range scannedResults { fmt.Printf("%s:%d (%s) @ %s\n", r.Host, r.Port, r.Protocol, r.Timestamp.Format(time.UnixDate)) } } diff --git a/cmd/login.go b/cmd/login.go index bd23e29..b0c0308 100644 --- a/cmd/login.go +++ b/cmd/login.go @@ -7,9 +7,9 @@ import ( "os" magellan "github.com/OpenCHAMI/magellan/internal" - "github.com/OpenCHAMI/magellan/internal/log" + "github.com/OpenCHAMI/magellan/internal/util" "github.com/lestrrat-go/jwx/jwt" - "github.com/sirupsen/logrus" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) @@ -27,47 +27,50 @@ var loginCmd = &cobra.Command{ Short: "Log in with identity provider for access token", Long: "", Run: func(cmd *cobra.Command, args []string) { - // make application logger - l := log.NewLogger(logrus.New(), logrus.DebugLevel) - // check if we have a valid JWT before starting login if !forceLogin { // try getting the access token from env var - testToken, err := LoadAccessToken() + testToken, err := util.LoadAccessToken(tokenPath) if err != nil { - l.Log.Errorf("failed to load access token: %v", err) + log.Error().Err(err).Msgf("failed to load access token") } // parse into jwt.Token to validate token, err := jwt.Parse([]byte(testToken)) if err != nil { - fmt.Printf("failed to parse access token contents: %v\n", err) + log.Error().Err(err).Msgf("failed to parse access token contents") return } // check if the token is invalid and we need a new one err = jwt.Validate(token) if err != nil { - fmt.Printf("failed to validate access token...fetching a new one") + log.Error().Err(err).Msgf("failed to validate access token...fetching a new one") } else { - fmt.Printf("found a valid token...skipping login (use the '-f/--force' flag to login anyway)") + log.Printf("found a valid token...skipping login (use the '-f/--force' flag to login anyway)") return } } + if verbose { + log.Printf("Listening for token on %s:%d", targetHost, targetPort) + } + // start the login flow var err error accessToken, err = magellan.Login(loginUrl, targetHost, targetPort) if errors.Is(err, http.ErrServerClosed) { - fmt.Printf("\n=========================================\nServer closed.\n=========================================\n\n") + if verbose { + fmt.Printf("\n=========================================\nServer closed.\n=========================================\n\n") + } } else if err != nil { - fmt.Printf("failed to start server: %v\n", err) + log.Error().Err(err).Msgf("failed to start server") } // if we got a new token successfully, save it to the token path if accessToken != "" && tokenPath != "" { err := os.WriteFile(tokenPath, []byte(accessToken), os.ModePerm) if err != nil { - fmt.Printf("failed to write access token to file: %v\n", err) + log.Error().Err(err).Msgf("failed to write access token to file") } } }, diff --git a/cmd/root.go b/cmd/root.go index 6571d9d..75f77aa 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -21,7 +21,7 @@ import ( "os/user" magellan "github.com/OpenCHAMI/magellan/internal" - "github.com/OpenCHAMI/magellan/pkg/smd" + "github.com/OpenCHAMI/magellan/pkg/client" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -66,35 +66,6 @@ func Execute() { } } -// LoadAccessToken() tries to load a JWT string from an environment -// variable, file, or config in that order. If loading the token -// fails with one options, it will fallback to the next option until -// all options are exhausted. -// -// Returns a token as a string with no error if successful. -// Alternatively, returns an empty string with an error if a token is -// not able to be loaded. -func LoadAccessToken() (string, error) { - // try to load token from env var - testToken := os.Getenv("ACCESS_TOKEN") - if testToken != "" { - return testToken, nil - } - - // try reading access token from a file - b, err := os.ReadFile(tokenPath) - if err == nil { - return string(b), nil - } - - // TODO: try to load token from config - testToken = viper.GetString("access_token") - if testToken != "" { - return testToken, nil - } - return "", fmt.Errorf("failed toload token from environment variable, file, or config") -} - func init() { currentUser, _ = user.Current() cobra.OnInitialize(InitializeConfig) @@ -140,8 +111,8 @@ func SetDefaults() { viper.SetDefault("scan.subnet-masks", []net.IP{}) viper.SetDefault("scan.disable-probing", false) viper.SetDefault("collect.driver", []string{"redfish"}) - viper.SetDefault("collect.host", smd.Host) - viper.SetDefault("collect.port", smd.Port) + viper.SetDefault("collect.host", client.Host) + viper.SetDefault("collect.port", client.Port) viper.SetDefault("collect.user", "") viper.SetDefault("collect.pass", "") viper.SetDefault("collect.protocol", "https") diff --git a/cmd/scan.go b/cmd/scan.go index 9fafdfe..e5866bf 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -18,8 +18,6 @@ import ( ) var ( - begin uint8 - end uint8 subnets []string subnetMasks []net.IP disableProbing bool @@ -38,7 +36,7 @@ var scanCmd = &cobra.Command{ "If the '--disable-probe` flag is used, the tool will not send another request to probe for available " + "Redfish services.\n\n" + "Example:\n" + - " magellan scan --subnet 172.16.0.0/24 --add-host 10.0.0.101\n" + + " magellan scan --subnet 172.16.0.0/24 --host 10.0.0.101\n" + " magellan scan --subnet 172.16.0.0 --subnet-mask 255.255.255.0 --cache ./assets.db", Run: func(cmd *cobra.Command, args []string) { var ( diff --git a/cmd/update.go b/cmd/update.go index 489cd78..592cd0f 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -2,8 +2,7 @@ package cmd import ( magellan "github.com/OpenCHAMI/magellan/internal" - "github.com/OpenCHAMI/magellan/internal/log" - "github.com/sirupsen/logrus" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -26,17 +25,16 @@ var updateCmd = &cobra.Command{ Short: "Update BMC node firmware", Long: "Perform an firmware update using Redfish by providing a remote firmware URL and component.\n" + "Examples:\n" + - " magellan update --host 172.16.0.108 --port 443 --username bmc_username --password bmc_password --firmware-url http://172.16.0.200:8005/firmware/bios/image.RBU --component BIOS\n" + - " magellan update --status --host 172.16.0.108 --port 443 --username bmc_username --password bmc_password", + " magellan update --bmc.host 172.16.0.108 --bmc.port 443 --username bmc_username --password bmc_password --firmware-url http://172.16.0.200:8005/firmware/bios/image.RBU --component BIOS\n" + + " magellan update --status --bmc.host 172.16.0.108 --bmc.port 443 --username bmc_username --password bmc_password", Run: func(cmd *cobra.Command, args []string) { - l := log.NewLogger(logrus.New(), logrus.DebugLevel) + // set up update parameters q := &magellan.UpdateParams{ FirmwarePath: firmwareUrl, FirmwareVersion: firmwareVersion, Component: component, TransferProtocol: transferProtocol, QueryParams: magellan.QueryParams{ - Protocol: protocol, Host: host, Username: username, Password: password, @@ -47,53 +45,49 @@ var updateCmd = &cobra.Command{ // check if required params are set if host == "" || username == "" || password == "" { - l.Log.Fatal("requires host, user, and pass to be set") + log.Error().Msg("requires host, user, and pass to be set") } // get status if flag is set and exit if status { err := magellan.GetUpdateStatus(q) if err != nil { - l.Log.Errorf("failed toget update status: %v", err) + log.Error().Err(err).Msgf("failed to get update status") } return } - // client, err := magellan.NewClient(l, &q.QueryParams) - // if err != nil { - // l.Log.Errorf("failed tomake client: %v", err) - // } - // err = magellan.UpdateFirmware(client, l, q) + // initiate a remote update err := magellan.UpdateFirmwareRemote(q) if err != nil { - l.Log.Errorf("failed toupdate firmware: %v", err) + log.Error().Err(err).Msgf("failed to update firmware") } }, } func init() { - updateCmd.Flags().StringVar(&host, "bmc-host", "", "set the BMC host") - updateCmd.Flags().IntVar(&port, "bmc-port", 443, "set the BMC port") + updateCmd.Flags().StringVar(&host, "bmc.host", "", "set the BMC host") + updateCmd.Flags().IntVar(&port, "bmc.port", 443, "set the BMC port") updateCmd.Flags().StringVar(&username, "username", "", "set the BMC user") updateCmd.Flags().StringVar(&password, "password", "", "set the BMC password") updateCmd.Flags().StringVar(&transferProtocol, "transfer-protocol", "HTTP", "set the transfer protocol") updateCmd.Flags().StringVar(&protocol, "protocol", "https", "set the Redfish protocol") - updateCmd.Flags().StringVar(&firmwareUrl, "firmware-url", "", "set the path to the firmware") - updateCmd.Flags().StringVar(&firmwareVersion, "firmware-version", "", "set the version of firmware to be installed") + updateCmd.Flags().StringVar(&firmwareUrl, "firmware.url", "", "set the path to the firmware") + updateCmd.Flags().StringVar(&firmwareVersion, "firmware.version", "", "set the version of firmware to be installed") updateCmd.Flags().StringVar(&component, "component", "", "set the component to upgrade") updateCmd.Flags().BoolVar(&status, "status", false, "get the status of the update") - viper.BindPFlag("host", updateCmd.Flags().Lookup("host")) - viper.BindPFlag("port", updateCmd.Flags().Lookup("port")) - viper.BindPFlag("username", updateCmd.Flags().Lookup("username")) - viper.BindPFlag("password", updateCmd.Flags().Lookup("password")) - viper.BindPFlag("transfer-protocol", updateCmd.Flags().Lookup("transfer-protocol")) - viper.BindPFlag("protocol", updateCmd.Flags().Lookup("protocol")) - viper.BindPFlag("firmware.url", updateCmd.Flags().Lookup("firmware.url")) - viper.BindPFlag("firmware.version", updateCmd.Flags().Lookup("firmware.version")) - viper.BindPFlag("component", updateCmd.Flags().Lookup("component")) - viper.BindPFlag("secure-tls", updateCmd.Flags().Lookup("secure-tls")) - viper.BindPFlag("status", updateCmd.Flags().Lookup("status")) + viper.BindPFlag("update.bmc.host", updateCmd.Flags().Lookup("bmc.host")) + viper.BindPFlag("update.bmc.port", updateCmd.Flags().Lookup("bmc.port")) + viper.BindPFlag("update.username", updateCmd.Flags().Lookup("username")) + viper.BindPFlag("update.password", updateCmd.Flags().Lookup("password")) + viper.BindPFlag("update.transfer-protocol", updateCmd.Flags().Lookup("transfer-protocol")) + viper.BindPFlag("update.protocol", updateCmd.Flags().Lookup("protocol")) + viper.BindPFlag("update.firmware.url", updateCmd.Flags().Lookup("firmware.url")) + viper.BindPFlag("update.firmware.version", updateCmd.Flags().Lookup("firmware.version")) + viper.BindPFlag("update.component", updateCmd.Flags().Lookup("component")) + viper.BindPFlag("update.secure-tls", updateCmd.Flags().Lookup("secure-tls")) + viper.BindPFlag("update.status", updateCmd.Flags().Lookup("status")) rootCmd.AddCommand(updateCmd) } From e19d0b2d7cdc17dc0cc112654ceba446c55c26a6 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 30 Jul 2024 14:02:27 -0600 Subject: [PATCH 12/67] Updated go dependencies --- go.mod | 28 ++++++++------------------- go.sum | 61 ++++++++++++++-------------------------------------------- 2 files changed, 22 insertions(+), 67 deletions(-) diff --git a/go.mod b/go.mod index ed40a3c..bc7015d 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/Cray-HPE/hms-xname v1.3.0 - github.com/bmc-toolbox/bmclib/v2 v2.2.3 + github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18 github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 github.com/go-chi/chi/v5 v5.1.0 github.com/jmoiron/sqlx v1.4.0 @@ -15,34 +15,24 @@ require ( github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 github.com/stmcginnis/gofish v0.19.0 - golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa + golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 ) require ( - dario.cat/mergo v1.0.0 // indirect - github.com/Jeffail/gabs/v2 v2.7.0 // indirect - github.com/ghodss/yaml v1.0.0 // indirect - github.com/jacobweinstock/registrar v0.4.7 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/lestrrat-go/httprc v1.0.4 // indirect + github.com/lestrrat-go/jwx/v2 v2.0.20 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - go.opentelemetry.io/otel v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + github.com/segmentio/asm v1.2.0 // indirect ) require ( - github.com/VictorLowther/simplexml v0.0.0-20180716164440-0bff93621230 // indirect - github.com/VictorLowther/soap v0.0.0-20150314151524-8e36fca84b22 // indirect - github.com/bmc-toolbox/common v0.0.0-20230717121556-5eb9915a8a5a // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/jacobweinstock/iamt v0.0.0-20230502042727-d7cdbe67d9ef // indirect github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect github.com/lestrrat-go/blackmagic v1.0.2 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect @@ -56,7 +46,6 @@ require ( github.com/rs/zerolog v1.33.0 github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect - github.com/satori/go.uuid v1.2.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect @@ -64,9 +53,8 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.18.0 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index b481397..63539ac 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,9 @@ -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/Cray-HPE/hms-xname v1.3.0 h1:DQmetMniubqcaL6Cxarz9+7KFfWGSEizIhfPHIgC3Gw= github.com/Cray-HPE/hms-xname v1.3.0/go.mod h1:XKdjQSzoTps5KDOE8yWojBTAWASGaS6LfRrVDxwTQO8= -github.com/Jeffail/gabs/v2 v2.7.0 h1:Y2edYaTcE8ZpRsR2AtmPu5xQdFDIthFG0jYhu5PY8kg= -github.com/Jeffail/gabs/v2 v2.7.0/go.mod h1:dp5ocw1FvBBQYssgHsG7I1WYsiLRtkUaB1FEtSwvNUw= -github.com/VictorLowther/simplexml v0.0.0-20180716164440-0bff93621230 h1:t95Grn2mOPfb3+kPDWsNnj4dlNcxnvuR72IjY8eYjfQ= -github.com/VictorLowther/simplexml v0.0.0-20180716164440-0bff93621230/go.mod h1:t2EzW1qybnPDQ3LR/GgeF0GOzHUXT5IVMLP2gkW1cmc= -github.com/VictorLowther/soap v0.0.0-20150314151524-8e36fca84b22 h1:a0MBqYm44o0NcthLKCljZHe1mxlN6oahCQHHThnSwB4= -github.com/VictorLowther/soap v0.0.0-20150314151524-8e36fca84b22/go.mod h1:/B7V22rcz4860iDqstGvia/2+IYWXf3/JdQCVd/1D2A= -github.com/bmc-toolbox/bmclib/v2 v2.2.3 h1:8IqAPtGXY7vfmSJm0ZYlQ4IOP9hKb33iTyQUbW1XyaE= -github.com/bmc-toolbox/bmclib/v2 v2.2.3/go.mod h1:gFF4iD468hbW1JUdJJx3mbhNGzoLsG47epbMa++grp8= -github.com/bmc-toolbox/common v0.0.0-20230717121556-5eb9915a8a5a h1:SjtoU9dE3bYfYnPXODCunMztjoDgnE3DVJCPLBqwz6Q= -github.com/bmc-toolbox/common v0.0.0-20230717121556-5eb9915a8a5a/go.mod h1:SY//n1PJjZfbFbmAsB6GvEKbc7UXz3d30s3kWxfJQ/c= -github.com/bombsimon/logrusr/v2 v2.0.1 h1:1VgxVNQMCvjirZIYaT9JYn6sAVGVEcNtRE0y4mvaOAM= -github.com/bombsimon/logrusr/v2 v2.0.1/go.mod h1:ByVAX+vHdLGAfdroiMg6q0zgq2FODY2lc5YJvzmOJio= +github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18 h1:oBPtXp9RVm9lk5zTmDLf+Vh21yDHpulBxUqGJQjwQCk= +github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18/go.mod h1:ggNHWgLfW/WRXcE8ZZC4S7UwHif16HVmyowOCWdNSN8= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= @@ -31,14 +19,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/zerologr v1.2.3 h1:up5N9vcH9Xck3jJkXzgyOxozT14R47IyDODz8LM1KSs= -github.com/go-logr/zerologr v1.2.3/go.mod h1:BxwGo7y5zgSHYR1BjbnHPyF/5ZjVKfKxAZANVu6E8Ho= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= @@ -46,19 +28,10 @@ github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jacobweinstock/iamt v0.0.0-20230502042727-d7cdbe67d9ef h1:G4k02HGmBUfJFSNu3gfKJ+ki+B3qutKsYzYndkqqKc4= -github.com/jacobweinstock/iamt v0.0.0-20230502042727-d7cdbe67d9ef/go.mod h1:FgmiLTU6cJewV4Xgrq6m5o8CUlTQOJtqzaFLGA0mG+E= -github.com/jacobweinstock/registrar v0.4.7 h1:s4dOExccgD+Pc7rJC+f3Mc3D+NXHcXUaOibtcEsPxOc= -github.com/jacobweinstock/registrar v0.4.7/go.mod h1:PWmkdGFG5/ZdCqgMo7pvB3pXABOLHc5l8oQ0sgmBNDU= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -74,10 +47,14 @@ github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= +github.com/lestrrat-go/httprc v1.0.4 h1:bAZymwoZQb+Oq8MEbyipag7iSq6YIga8Wj6GOiJGdI8= +github.com/lestrrat-go/httprc v1.0.4/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= github.com/lestrrat-go/jwx v1.2.29 h1:QT0utmUJ4/12rmsVQrJ3u55bycPkKqGYuGT4tyRhxSQ= github.com/lestrrat-go/jwx v1.2.29/go.mod h1:hU8k2l6WF0ncx20uQdOmik/Gjg6E3/wIRtXSNFeZuB8= +github.com/lestrrat-go/jwx/v2 v2.0.20 h1:sAgXuWS/t8ykxS9Bi2Qtn5Qhpakw1wrcjxChudjolCc= +github.com/lestrrat-go/jwx/v2 v2.0.20/go.mod h1:UlCSmKqw+agm5BsOBfEAbTvKsEApaGNqHAEUTv5PJC4= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= @@ -116,8 +93,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= +github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= @@ -149,23 +126,18 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= -golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 h1:ESSUROHIBHg7USnszlcdmjBEwdMj9VUvU+OPk4yl2mc= +golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -174,8 +146,6 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -192,8 +162,9 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -215,12 +186,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 386e9f2777c8936bd88a3cadf64e6b96f1cbdfac Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 30 Jul 2024 14:03:34 -0600 Subject: [PATCH 13/67] Renamed smd package to client --- pkg/client/client.go | 87 ++++++++++++++++++++++++++++++++++++++ pkg/{smd => client}/smd.go | 75 ++++---------------------------- 2 files changed, 95 insertions(+), 67 deletions(-) create mode 100644 pkg/client/client.go rename pkg/{smd => client}/smd.go (50%) diff --git a/pkg/client/client.go b/pkg/client/client.go new file mode 100644 index 0000000..229471d --- /dev/null +++ b/pkg/client/client.go @@ -0,0 +1,87 @@ +package client + +import ( + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "net" + "net/http" + "os" + "time" + + "github.com/OpenCHAMI/magellan/internal/util" +) + +type Option func(*Client) + +// The 'Client' struct is a wrapper around the default http.Client +// that provides an extended API to work with functional options. +// It also provides functions that work with `collect` data. +type Client struct { + *http.Client +} + +// NewClient() creates a new client +func NewClient(opts ...Option) *Client { + client := &Client{ + Client: http.DefaultClient, + } + for _, opt := range opts { + opt(client) + } + return client +} + +func WithHttpClient(httpClient *http.Client) Option { + return func(c *Client) { + c.Client = httpClient + } +} + +func WithCertPool(certPool *x509.CertPool) Option { + if certPool == nil { + return func(c *Client) {} + } + return func(c *Client) { + c.Client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + InsecureSkipVerify: true, + }, + DisableKeepAlives: true, + Dial: (&net.Dialer{ + Timeout: 120 * time.Second, + KeepAlive: 120 * time.Second, + }).Dial, + TLSHandshakeTimeout: 120 * time.Second, + ResponseHeaderTimeout: 120 * time.Second, + } + } +} + +func WithSecureTLS(certPath string) Option { + cacert, err := os.ReadFile(certPath) + if err != nil { + return func(c *Client) {} + } + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(cacert) + return WithCertPool(certPool) +} + +// Post() is a simplified wrapper function that packages all of the +// that marshals a mapper into a JSON-formatted byte array, and then performs +// a request to the specified URL. +func (c *Client) Post(url string, data map[string]any, header util.HTTPHeader) (*http.Response, util.HTTPBody, error) { + // serialize data into byte array + body, err := json.Marshal(data) + if err != nil { + return nil, nil, fmt.Errorf("failed to marshal data for request: %v", err) + } + return util.MakeRequest(c.Client, url, http.MethodPost, body, header) +} + +func (c *Client) MakeRequest(url string, method string, body util.HTTPBody, header util.HTTPHeader) (*http.Response, util.HTTPBody, error) { + return util.MakeRequest(c.Client, url, method, body, header) +} diff --git a/pkg/smd/smd.go b/pkg/client/smd.go similarity index 50% rename from pkg/smd/smd.go rename to pkg/client/smd.go index 370841d..726c5f6 100644 --- a/pkg/smd/smd.go +++ b/pkg/client/smd.go @@ -1,16 +1,11 @@ -package smd +package client // See ref for API docs: // https://github.com/OpenCHAMI/hms-smd/blob/master/docs/examples.adoc // https://github.com/OpenCHAMI/hms-smd import ( - "crypto/tls" - "crypto/x509" "fmt" - "net" "net/http" - "os" - "time" "github.com/OpenCHAMI/magellan/internal/util" ) @@ -21,65 +16,11 @@ var ( Port = 27779 ) -type Option func(*Client) - -type Client struct { - *http.Client - CACertPool *x509.CertPool -} - -func NewClient(opts ...Option) *Client { - client := &Client{ - Client: http.DefaultClient, - } - for _, opt := range opts { - opt(client) - } - return client -} - -func WithHttpClient(httpClient *http.Client) Option { - return func(c *Client) { - c.Client = httpClient - } -} - -// This MakeRequest function is a wrapper around the util.MakeRequest function -// with a couple of niceties with using a smd.Client -func (c *Client) MakeRequest(url string, method string, body []byte, headers map[string]string) (*http.Response, []byte, error) { - return util.MakeRequest(c.Client, url, method, body, headers) -} - -func WithCertPool(certPool *x509.CertPool) Option { - return func(c *Client) { - c.Client.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{ - RootCAs: certPool, - InsecureSkipVerify: true, - }, - DisableKeepAlives: true, - Dial: (&net.Dialer{ - Timeout: 120 * time.Second, - KeepAlive: 120 * time.Second, - }).Dial, - TLSHandshakeTimeout: 120 * time.Second, - ResponseHeaderTimeout: 120 * time.Second, - } - } -} - -func WithSecureTLS(certPath string) Option { - cacert, _ := os.ReadFile(certPath) - certPool := x509.NewCertPool() - certPool.AppendCertsFromPEM(cacert) - return WithCertPool(certPool) -} - -func (c *Client) GetRedfishEndpoints(headers map[string]string, opts ...Option) error { +func (c *Client) GetRedfishEndpoints(header util.HTTPHeader) error { url := makeEndpointUrl("/Inventory/RedfishEndpoints") - _, body, err := c.MakeRequest(url, "GET", nil, headers) + _, body, err := util.MakeRequest(c.Client, url, http.MethodGet, nil, header) if err != nil { - return fmt.Errorf("failed toget endpoint: %v", err) + return fmt.Errorf("failed to get endpoint: %v", err) } // fmt.Println(res) fmt.Println(string(body)) @@ -90,21 +31,21 @@ func (c *Client) GetComponentEndpoint(xname string) error { url := makeEndpointUrl("/Inventory/ComponentsEndpoints/" + xname) res, body, err := c.MakeRequest(url, "GET", nil, nil) if err != nil { - return fmt.Errorf("failed toget endpoint: %v", err) + return fmt.Errorf("failed to get endpoint: %v", err) } fmt.Println(res) fmt.Println(string(body)) return nil } -func (c *Client) AddRedfishEndpoint(data []byte, headers map[string]string) error { +func (c *Client) AddRedfishEndpoint(data map[string]any, headers util.HTTPHeader) error { if data == nil { - return fmt.Errorf("failed toadd redfish endpoint: no data found") + return fmt.Errorf("failed to add redfish endpoint: no data found") } // Add redfish endpoint via POST `/hsm/v2/Inventory/RedfishEndpoints` endpoint url := makeEndpointUrl("/Inventory/RedfishEndpoints") - res, body, err := c.MakeRequest(url, "POST", data, headers) + res, body, err := c.Post(url, data, headers) if res != nil { statusOk := res.StatusCode >= 200 && res.StatusCode < 300 if !statusOk { From 9b3c21a20a0e414761c67befdeae837cc9e09c47 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 30 Jul 2024 14:04:34 -0600 Subject: [PATCH 14/67] Removed magellan's internal logger for zerolog --- internal/log/logger.go | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 internal/log/logger.go diff --git a/internal/log/logger.go b/internal/log/logger.go deleted file mode 100644 index 43e2db2..0000000 --- a/internal/log/logger.go +++ /dev/null @@ -1,22 +0,0 @@ -package log - -import ( - "github.com/sirupsen/logrus" -) - -type Logger struct { - Log *logrus.Logger - Path string -} - -func NewLogger(l *logrus.Logger, level logrus.Level) *Logger { - l.SetLevel(level) - return &Logger{ - Log: logrus.New(), - Path: "", - } -} - -func (l *Logger) WriteFile(path string) { - -} From 0922bbf5f9feb65bb3b053dd7f0a97b18864b760 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 30 Jul 2024 14:05:55 -0600 Subject: [PATCH 15/67] Removed unused updating code and bmclib dependency and other minor changes --- internal/db/sqlite/sqlite.go | 2 +- internal/update.go | 139 +---------------------------------- internal/util/auth.go | 4 +- internal/util/net.go | 20 ++++- internal/util/path.go | 17 +++-- 5 files changed, 33 insertions(+), 149 deletions(-) diff --git a/internal/db/sqlite/sqlite.go b/internal/db/sqlite/sqlite.go index 7eb75a8..9a60240 100644 --- a/internal/db/sqlite/sqlite.go +++ b/internal/db/sqlite/sqlite.go @@ -80,7 +80,7 @@ func DeleteProbeResults(path string, results *[]magellan.ScannedResult) error { return nil } -func GetProbeResults(path string) ([]magellan.ScannedResult, error) { +func GetScannedResults(path string) ([]magellan.ScannedResult, error) { db, err := sqlx.Open("sqlite3", path) if err != nil { return nil, fmt.Errorf("failed toopen database: %v", err) diff --git a/internal/update.go b/internal/update.go index 35ed4c4..8c07dce 100644 --- a/internal/update.go +++ b/internal/update.go @@ -1,21 +1,11 @@ package magellan import ( - "context" "encoding/json" - "errors" "fmt" "net/http" - "os" - "strings" - "time" - "github.com/OpenCHAMI/magellan/internal/log" "github.com/OpenCHAMI/magellan/internal/util" - bmclib "github.com/bmc-toolbox/bmclib/v2" - "github.com/bmc-toolbox/bmclib/v2/constants" - bmclibErrs "github.com/bmc-toolbox/bmclib/v2/errors" - "github.com/sirupsen/logrus" ) type UpdateParams struct { @@ -26,110 +16,9 @@ type UpdateParams struct { TransferProtocol string } -// UpdateFirmware() uses 'bmc-toolbox/bmclib' to update the firmware of a BMC node. +// UpdateFirmwareRemote() uses 'gofish' to update the firmware of a BMC node. // The function expects the firmware URL, firmware version, and component flags to be // set from the CLI to perform a firmware update. -// -// NOTE: Multipart HTTP updating may not work since older verions of OpenBMC, which bmclib -// uses underneath, did not support support multipart updates. This was changed with the -// inclusion of support for MultipartHttpPushUri in OpenBMC (https://gerrit.openbmc.org/c/openbmc/bmcweb/+/32174). -// Also, related to bmclib: https://github.com/bmc-toolbox/bmclib/issues/341 -func UpdateFirmware(client *bmclib.Client, l *log.Logger, q *UpdateParams) error { - if q.Component == "" { - return fmt.Errorf("component is required") - } - - // open BMC session and update driver registry - ctx, ctxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(q.Timeout)) - client.Registry.FilterForCompatible(ctx) - err := client.Open(ctx) - if err != nil { - ctxCancel() - return fmt.Errorf("failed toconnect to bmc: %v", err) - } - - defer client.Close(ctx) - - file, err := os.Open(q.FirmwarePath) - if err != nil { - ctxCancel() - return fmt.Errorf("failed toopen firmware path: %v", err) - } - - defer file.Close() - - taskId, err := client.FirmwareInstall(ctx, q.Component, constants.FirmwareApplyOnReset, true, file) - if err != nil { - ctxCancel() - return fmt.Errorf("failed toinstall firmware: %v", err) - } - - for { - if ctx.Err() != nil { - ctxCancel() - return fmt.Errorf("context error: %v", ctx.Err()) - } - - state, err := client.FirmwareInstallStatus(ctx, q.FirmwareVersion, q.Component, taskId) - if err != nil { - // when its under update a connection refused is returned - if strings.Contains(err.Error(), "connection refused") || strings.Contains(err.Error(), "operation timed out") { - l.Log.Info("BMC refused connection, BMC most likely resetting...") - time.Sleep(2 * time.Second) - - continue - } - - if errors.Is(err, bmclibErrs.ErrSessionExpired) || strings.Contains(err.Error(), "session expired") { - err := client.Open(ctx) - if err != nil { - l.Log.Fatal(err, "bmc re-login failed") - } - - l.Log.WithFields(logrus.Fields{"state": state, "component": q.Component}).Info("BMC session expired, logging in...") - - continue - } - - l.Log.Fatal(err) - } - - switch state { - case constants.FirmwareInstallRunning, constants.FirmwareInstallInitializing: - l.Log.WithFields(logrus.Fields{"state": state, "component": q.Component}).Info("firmware install running") - - case constants.FirmwareInstallFailed: - ctxCancel() - l.Log.WithFields(logrus.Fields{"state": state, "component": q.Component}).Info("firmware install failed") - return fmt.Errorf("failed to install firmware") - - case constants.FirmwareInstallComplete: - ctxCancel() - l.Log.WithFields(logrus.Fields{"state": state, "component": q.Component}).Info("firmware install completed") - return nil - - case constants.FirmwareInstallPowerCyleHost: - l.Log.WithFields(logrus.Fields{"state": state, "component": q.Component}).Info("host powercycle required") - - if _, err := client.SetPowerState(ctx, "cycle"); err != nil { - ctxCancel() - l.Log.WithFields(logrus.Fields{"state": state, "component": q.Component}).Info("error power cycling host for install") - return fmt.Errorf("failed to install firmware") - } - - ctxCancel() - l.Log.WithFields(logrus.Fields{"state": state, "component": q.Component}).Info("host power cycled, all done!") - return nil - default: - l.Log.WithFields(logrus.Fields{"state": state, "component": q.Component}).Info("unknown state returned") - } - - time.Sleep(2 * time.Second) - } - - return nil -} - func UpdateFirmwareRemote(q *UpdateParams) error { url := baseRedfishUrl(&q.QueryParams) + "/redfish/v1/UpdateService/Actions/SimpleUpdate" headers := map[string]string{ @@ -143,7 +32,7 @@ func UpdateFirmwareRemote(q *UpdateParams) error { } data, err := json.Marshal(b) if err != nil { - return fmt.Errorf("failed tomarshal data: %v", err) + return fmt.Errorf("failed to marshal data: %v", err) } res, body, err := util.MakeRequest(nil, url, "POST", data, headers) if err != nil { @@ -172,27 +61,3 @@ func GetUpdateStatus(q *UpdateParams) error { } return nil } - -// func UpdateFirmwareLocal(q *UpdateParams) error { -// fwUrl := baseUrl(&q.QueryParams) + "" -// url := baseUrl(&q.QueryParams) + "UpdateService/Actions/" -// headers := map[string]string { - -// } - -// // get etag from FW inventory -// response, err := util.MakeRequest() - -// // load file from disk -// file, err := os.ReadFile(q.FirmwarePath) -// if err != nil { -// return fmt.Errorf("failed toread file: %v", err) -// } - -// switch q.TransferProtocol { -// case "HTTP": -// default: -// return fmt.Errorf("transfer protocol not supported") -// } -// return nil -// } diff --git a/internal/util/auth.go b/internal/util/auth.go index 98ef88c..9df0a11 100644 --- a/internal/util/auth.go +++ b/internal/util/auth.go @@ -29,9 +29,9 @@ func LoadAccessToken(path string) (string, error) { } // TODO: try to load token from config - testToken = viper.GetString("access_token") + testToken = viper.GetString("access-token") if testToken != "" { return testToken, nil } - return "", fmt.Errorf("failed toload token from environment variable, file, or config") + return "", fmt.Errorf("failed to load token from environment variable, file, or config") } diff --git a/internal/util/net.go b/internal/util/net.go index cdc18db..f0528f1 100644 --- a/internal/util/net.go +++ b/internal/util/net.go @@ -9,6 +9,22 @@ import ( "net/http" ) +// HTTP aliases for readibility +type HTTPHeader map[string]string +type HTTPBody []byte + +func (h HTTPHeader) Authorization(accessToken string) HTTPHeader { + if accessToken != "" { + h["Authorization"] = fmt.Sprintf("Bearer %s", accessToken) + } + return h +} + +func (h HTTPHeader) ContentType(contentType string) HTTPHeader { + h["Content-Type"] = contentType + return h +} + // GetNextIP() returns the next IP address, but does not account // for net masks. func GetNextIP(ip *net.IP, inc uint) *net.IP { @@ -35,7 +51,7 @@ func GetNextIP(ip *net.IP, inc uint) *net.IP { // // Returns a HTTP response object, response body as byte array, and any // error that may have occurred with making the request. -func MakeRequest(client *http.Client, url string, httpMethod string, body []byte, headers map[string]string) (*http.Response, []byte, error) { +func MakeRequest(client *http.Client, url string, httpMethod string, body HTTPBody, header HTTPHeader) (*http.Response, HTTPBody, error) { // use defaults if no client provided if client == nil { client = http.DefaultClient @@ -48,7 +64,7 @@ func MakeRequest(client *http.Client, url string, httpMethod string, body []byte return nil, nil, fmt.Errorf("failed to create new HTTP request: %v", err) } req.Header.Add("User-Agent", "magellan") - for k, v := range headers { + for k, v := range header { req.Header.Add(k, v) } res, err := client.Do(req) diff --git a/internal/util/path.go b/internal/util/path.go index 06611b6..19f6afe 100644 --- a/internal/util/path.go +++ b/internal/util/path.go @@ -40,31 +40,34 @@ func SplitPathForViper(path string) (string, string, string) { } // MakeOutputDirectory() creates a new directory at the path argument if -// the path does not exist +// the path does not exist. +// +// Returns the final path that was created if no errors occurred. Otherwise, +// it returns an empty string with an error. // // TODO: Refactor this function for hive partitioning or possibly move into // the logging package. // TODO: Add an option to force overwriting the path. -func MakeOutputDirectory(path string) (string, error) { +func MakeOutputDirectory(path string, overwrite bool) (string, error) { // get the current data + time using Go's stupid formatting t := time.Now() - dirname := t.Format("2006-01-01 15:04:05") + dirname := t.Format("2006-01-01") final := path + "/" + dirname // check if path is valid and directory pathExists, err := PathExists(final) if err != nil { - return final, fmt.Errorf("failed to check for existing path: %v", err) + return "", fmt.Errorf("failed to check for existing path: %v", err) } - if pathExists { + if pathExists && !overwrite { // make sure it is directory with 0o644 permissions - return final, fmt.Errorf("found existing path: %v", final) + return "", fmt.Errorf("found existing path: %v", final) } // create directory with data + time err = os.MkdirAll(final, 0766) if err != nil { - return final, fmt.Errorf("failed to make directory: %v", err) + return "", fmt.Errorf("failed to make directory: %v", err) } return final, nil } From 8ff71f6cef8d8d7cae70a8b721445504879dd0b4 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 30 Jul 2024 14:06:31 -0600 Subject: [PATCH 16/67] Minor changes to tests --- tests/api_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/api_test.go b/tests/api_test.go index 558e688..60c1886 100644 --- a/tests/api_test.go +++ b/tests/api_test.go @@ -11,15 +11,12 @@ import ( "testing" magellan "github.com/OpenCHAMI/magellan/internal" - "github.com/OpenCHAMI/magellan/internal/log" - "github.com/sirupsen/logrus" ) func TestScanAndCollect(t *testing.T) { var ( hosts = []string{"http://127.0.0.1"} ports = []int{5000} - l = log.NewLogger(logrus.New(), logrus.DebugLevel) ) // do a scan on the emulator cluster with probing disabled and check results results := magellan.ScanForAssets(hosts, ports, 1, 30, true, false) @@ -33,7 +30,7 @@ func TestScanAndCollect(t *testing.T) { } // do a collect on the emulator cluster to collect Redfish info - magellan.CollectAll(results) + magellan.CollectInventory(&results, &magellan.QueryParams{}) } func TestCrawlCommand(t *testing.T) { From 1ea779e802ce8a9fae9826fc7c32a61d04a97fc9 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 30 Jul 2024 14:23:53 -0600 Subject: [PATCH 17/67] Changed short help message for root command --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 75f77aa..3c00d91 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -48,7 +48,7 @@ var ( // a help message and then exits. var rootCmd = &cobra.Command{ Use: "magellan", - Short: "Tool for BMC discovery", + Short: "Redfish-based BMC discovery tool", Long: "", Run: func(cmd *cobra.Command, args []string) { if len(args) == 0 { From 2d4293359aaab0d131d3c4fbd53ba94d022ae8a4 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 30 Jul 2024 14:25:38 -0600 Subject: [PATCH 18/67] Updated README.md with features section --- README.md | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 52db387..cb53d7c 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,18 @@ The `magellan` CLI tool is a Redfish-based, board management controller (BMC) discovery tool designed to scan networks and is written in Go. The tool collects information from BMC nodes using the provided Redfish RESTful API with [`gofish`](https://github.com/stmcginnis/gofish) and loads the queried data into an [SMD](https://github.com/OpenCHAMI/smd/tree/master) instance. The tool strives to be more flexible by implementing multiple methods of discovery to work for a wider range of systems (WIP) and is capable of using independently of other tools or services. +## Main Features + +The `magellan` tool comes packed with a handleful of features for doing discovery, such as: + +- Simple network scanning +- Redfish-based inventory collection +- Redfish-based firmware updating +- Integration with OpenCHAMI SMD +- Write inventory data to JSON + +See the [TODO](#todo) section for a list of soon-ish goals planned. + ## Getting Started [Build](#building) and [run on bare metal](#running-the-tool) or run and test with Docker using the [latest prebuilt image](#running-with-docker). For quick testing, the repository integrates a Redfish emulator that can be ran by executing the `emulator/setup.sh` script or running `make emulator`. @@ -52,10 +64,6 @@ docker pull ghcr.io/openchami/magellan:latest See the ["Running with Docker"](#running-with-docker) section below about running with the Docker container. - - - - ## Usage The sections below assume that the BMC nodes have an IP address available to query Redfish. Currently, `magellan` does not support discovery with MAC addresses although that may change in the future. @@ -89,7 +97,9 @@ This should return a JSON response with general information. The output below ha } ``` -To see all of the available commands, run `magellan` with the `help` subcommand: +### Running the Tool + +There are three main commands to use with the tool: `scan`, `list`, and `collect`. To see all of the available commands, run `magellan` with the `help` subcommand: ```bash ./magellan help @@ -120,9 +130,7 @@ Flags: Use "magellan [command] --help" for more information about a command. ``` -### Running the Tool - -There are three main commands to use with the tool: `scan`, `list`, and `collect`. To start a network scan for BMC nodes, use the `scan` command. If the port is not specified, `magellan` will probe ports 623 and 443 by default: +To start a network scan for BMC nodes, use the `scan` command. If the port is not specified, `magellan` will probe ports 623 and 443 by default: ```bash ./magellan scan \ @@ -234,13 +242,13 @@ See the [issue list](https://github.com/OpenCHAMI/magellan/issues) for plans for * [X] Confirm loading different components into SMD * [X] Add ability to set subnet mask for scanning -* [ ] Add ability to scan with other protocols like LLDP -* [ ] Add more debugging messages with the `-v/--verbose` flag +* [ ] Add ability to scan with other protocols like LLDP and SSDP +* [X] Add more debugging messages with the `-v/--verbose` flag * [ ] Separate `collect` subcommand with making request to endpoint * [X] Support logging in with `opaal` to get access token * [X] Support using CA certificates with HTTP requests to SMD -* [ ] Add unit tests for `scan`, `list`, and `collect` commands -* [ ] Clean up, remove unused, and tidy code +* [ ] Add tests for the regressions and compatibility +* [X] Clean up, remove unused, and tidy code (first round) ## Copyright From d5eacf02640ef80c2edf465dce583cb7cbadc430 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 31 Jul 2024 08:45:42 -0600 Subject: [PATCH 19/67] Updated Makefile --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index c21ed53..0da4f4c 100644 --- a/Makefile +++ b/Makefile @@ -88,6 +88,11 @@ docs: ## go docs go doc github.com/OpenCHAMI/magellan/internal go doc github.com/OpenCHAMI/magellan/pkg/crawler +.PHONY: emulator +emulator: + $(call print-target) + ./emulator/setup.sh + define print-target @printf "Executing target: \033[36m$@\033[0m\n" endef From 7fc913eb1f4abf9f4f9031f10cf1f122664f452f Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 31 Jul 2024 08:45:54 -0600 Subject: [PATCH 20/67] Updated .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 4385204..fa5476a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ magellan +emulator/rf-emulator **/*.db dist/* From 2c841906b29e38ee37619a5f90dd4cdbc881f96f Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 7 Aug 2024 10:58:03 -0600 Subject: [PATCH 21/67] Updated 'cmd' package --- cmd/collect.go | 18 ++--- cmd/crawl.go | 4 +- cmd/list.go | 6 +- cmd/root.go | 13 ++-- cmd/scan.go | 197 +++++++++++++++++++++++++++++++++++-------------- cmd/update.go | 44 ++++++----- 6 files changed, 187 insertions(+), 95 deletions(-) diff --git a/cmd/collect.go b/cmd/collect.go index 670dca2..b2cd151 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -5,7 +5,7 @@ import ( "os/user" magellan "github.com/OpenCHAMI/magellan/internal" - "github.com/OpenCHAMI/magellan/internal/db/sqlite" + "github.com/OpenCHAMI/magellan/internal/cache/sqlite" "github.com/OpenCHAMI/magellan/internal/util" "github.com/OpenCHAMI/magellan/pkg/client" "github.com/cznic/mathutil" @@ -31,7 +31,7 @@ var collectCmd = &cobra.Command{ " magellan collect --host smd.example.com --port 27779 --username username --password password", Run: func(cmd *cobra.Command, args []string) { // get probe states stored in db from scan - scannedResults, err := sqlite.GetScannedResults(cachePath) + scannedResults, err := sqlite.GetScannedAssets(cachePath) if err != nil { log.Error().Err(err).Msgf("failed to get scanned results from cache") } @@ -53,7 +53,7 @@ var collectCmd = &cobra.Command{ if concurrency <= 0 { concurrency = mathutil.Clamp(len(scannedResults), 1, 255) } - q := &magellan.QueryParams{ + err = magellan.CollectInventory(&scannedResults, &magellan.CollectParams{ Username: username, Password: password, Timeout: timeout, @@ -63,15 +63,10 @@ var collectCmd = &cobra.Command{ OutputPath: outputPath, ForceUpdate: forceUpdate, AccessToken: accessToken, - } - err = magellan.CollectInventory(&scannedResults, q) + }) if err != nil { log.Error().Err(err).Msgf("failed to collect data") } - - // add necessary headers for final request (like token) - header := util.HTTPHeader{} - header.Authorization(q.AccessToken) }, } @@ -81,8 +76,9 @@ func init() { collectCmd.PersistentFlags().IntVarP(&client.Port, "port", "p", client.Port, "set the port to the SMD API") collectCmd.PersistentFlags().StringVar(&username, "username", "", "set the BMC user") collectCmd.PersistentFlags().StringVar(&password, "password", "", "set the BMC password") - collectCmd.PersistentFlags().StringVar(&protocol, "protocol", "https", "set the protocol used to query") - collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/data/", currentUser.Username+"/"), "set the path to store collection data") + collectCmd.PersistentFlags().StringVar(&scheme, "scheme", "https", "set the scheme used to query") + collectCmd.PersistentFlags().StringVar(&protocol, "protocol", "tcp", "set the protocol used to query") + collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/inventory/", currentUser.Username+"/"), "set the path to store collection data") collectCmd.PersistentFlags().BoolVar(&forceUpdate, "force-update", false, "set flag to force update data sent to SMD") collectCmd.PersistentFlags().StringVar(&cacertPath, "cacert", "", "path to CA cert. (defaults to system CAs)") diff --git a/cmd/crawl.go b/cmd/crawl.go index ba94636..2df487b 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -17,7 +17,9 @@ import ( var crawlCmd = &cobra.Command{ Use: "crawl [uri]", Short: "Crawl a single BMC for inventory information", - Long: "Crawl a single BMC for inventory information\n" + + Long: "Crawl a single BMC for inventory information. This command does NOT store information" + + "store information about the scan into cache after completion. To do so, use the 'collect'" + + "command instead\n" + "\n" + "Examples:\n" + " magellan crawl https://bmc.example.com\n" + diff --git a/cmd/list.go b/cmd/list.go index e1b254d..8d788d2 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -6,7 +6,7 @@ import ( "strings" "time" - "github.com/OpenCHAMI/magellan/internal/db/sqlite" + "github.com/OpenCHAMI/magellan/internal/cache/sqlite" "github.com/rs/zerolog/log" "github.com/sirupsen/logrus" @@ -25,9 +25,9 @@ var listCmd = &cobra.Command{ " magellan list\n" + " magellan list --cache ./assets.db", Run: func(cmd *cobra.Command, args []string) { - scannedResults, err := sqlite.GetScannedResults(cachePath) + scannedResults, err := sqlite.GetScannedAssets(cachePath) if err != nil { - logrus.Errorf("failed toget probe results: %v\n", err) + logrus.Errorf("failed to get scanned assets: %v\n", err) } format = strings.ToLower(format) if format == "json" { diff --git a/cmd/root.go b/cmd/root.go index 3c00d91..09a38c1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -42,6 +42,7 @@ var ( outputPath string configPath string verbose bool + debug bool ) // The `root` command doesn't do anything on it's own except display @@ -70,9 +71,10 @@ func init() { currentUser, _ = user.Current() cobra.OnInitialize(InitializeConfig) rootCmd.PersistentFlags().IntVar(&concurrency, "concurrency", -1, "set the number of concurrent processes") - rootCmd.PersistentFlags().IntVar(&timeout, "timeout", 30, "set the timeout") + rootCmd.PersistentFlags().IntVar(&timeout, "timeout", 5, "set the timeout") rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", "", "set the config file path") - rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "set output verbosity") + rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "set to enable/disable verbose output") + rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "set to enable/disable debug messages") rootCmd.PersistentFlags().StringVar(&accessToken, "access-token", "", "set the access token") rootCmd.PersistentFlags().StringVar(&cachePath, "cache", fmt.Sprintf("/tmp/%smagellan/magellan.db", currentUser.Username+"/"), "set the scanning result cache path") @@ -101,9 +103,10 @@ func InitializeConfig() { // instead of in this file. func SetDefaults() { viper.SetDefault("threads", 1) - viper.SetDefault("timeout", 30) + viper.SetDefault("timeout", 5) viper.SetDefault("config", "") viper.SetDefault("verbose", false) + viper.SetDefault("debug", false) viper.SetDefault("cache", "/tmp/magellan/magellan.db") viper.SetDefault("scan.hosts", []string{}) viper.SetDefault("scan.ports", []int{}) @@ -115,7 +118,7 @@ func SetDefaults() { viper.SetDefault("collect.port", client.Port) viper.SetDefault("collect.user", "") viper.SetDefault("collect.pass", "") - viper.SetDefault("collect.protocol", "https") + viper.SetDefault("collect.protocol", "tcp") viper.SetDefault("collect.output", "/tmp/magellan/data/") viper.SetDefault("collect.force-update", false) viper.SetDefault("collect.ca-cert", "") @@ -124,7 +127,7 @@ func SetDefaults() { viper.SetDefault("user", "") viper.SetDefault("pass", "") viper.SetDefault("transfer-protocol", "HTTP") - viper.SetDefault("protocol", "https") + viper.SetDefault("protocol", "tcp") viper.SetDefault("firmware-url", "") viper.SetDefault("firmware-version", "") viper.SetDefault("component", "") diff --git a/cmd/scan.go b/cmd/scan.go index e5866bf..65baa16 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -6,11 +6,11 @@ import ( "net" "os" "path" - "strings" - "time" magellan "github.com/OpenCHAMI/magellan/internal" - "github.com/OpenCHAMI/magellan/internal/db/sqlite" + "github.com/OpenCHAMI/magellan/internal/cache/sqlite" + "github.com/OpenCHAMI/magellan/internal/util" + "github.com/rs/zerolog/log" "github.com/cznic/mathutil" "github.com/spf13/cobra" @@ -18,9 +18,12 @@ import ( ) var ( + scheme string subnets []string - subnetMasks []net.IP + subnetMask net.IPMask + targetHosts [][]string disableProbing bool + disableCache bool ) // The `scan` command is usually the first step to using the CLI tool. @@ -30,85 +33,165 @@ var ( // See the `ScanForAssets()` function in 'internal/scan.go' for details // related to the implementation. var scanCmd = &cobra.Command{ - Use: "scan", - Short: "Scan for BMC nodes on a network", + Use: "scan urls...", + Short: "Scan to discover BMC nodes on a network", Long: "Perform a net scan by attempting to connect to each host and port specified and getting a response. " + - "If the '--disable-probe` flag is used, the tool will not send another request to probe for available " + - "Redfish services.\n\n" + - "Example:\n" + - " magellan scan --subnet 172.16.0.0/24 --host 10.0.0.101\n" + - " magellan scan --subnet 172.16.0.0 --subnet-mask 255.255.255.0 --cache ./assets.db", + "Each host is passed *with a full URL* including the protocol and port. Additional subnets can be added " + + "by using the '--subnet' flag and providing an IP address on the subnet as well as a CIDR. If no CIDR is " + + "provided, then the subnet mask specified with the '--subnet-mask' flag will be used instead (will use " + + "default mask if not set).\n" + + "Similarly, any host provided with no port with use either the ports specified" + + "with `--port` or the default port used with each specified protocol. The default protocol is 'tcp' unless " + + "specified. The `--scheme` flag works similarly and the default value is 'https' in the host URL or with the " + + "'--protocol' flag.\n" + + "If the '--disable-probe` flag is used, the tool will not send another request to probe for available. " + + "Redfish services. This is not recommended, since the extra request makes the scan a bit more reliable " + + "for determining which hosts to collect inventory data.\n\n" + + "Examples:\n" + + // assumes host https://10.0.0.101:443 + " magellan scan 10.0.0.101\n" + + // assumes subnet using HTTPS and port 443 except for specified host + " magellan scan http://10.0.0.101:80 https://user:password@10.0.0.102:443 http://172.16.0.105:8080 --subnet 172.16.0.0/24\n" + + // assumes hosts http://10.0.0.101:8080 and http://10.0.0.102:8080 + " magellan scan 10.0.0.101 10.0.0.102 https://172.16.0.10:443 --port 8080 --protocol tcp" + + // assumes subnet using default unspecified subnet-masks + " magellan scan --subnet 10.0.0.0" + + // assumes subnet using HTTPS and port 443 with specified CIDR + " magellan scan --subnet 10.0.0.0/16" + + // assumes subnet using HTTP and port 5000 similar to 192.168.0.0/16 + " magellan scan --subnet 192.168.0.0 --protocol tcp --scheme https --port 5000 --subnet-mask 255.255.0.0" + + // assumes subnet without CIDR has a subnet-mask of 255.255.0.0 + " magellan scan --subnet 10.0.0.0/24 --subnet 172.16.0.0 --subnet-mask 255.255.0.0 --cache ./assets.db", Run: func(cmd *cobra.Command, args []string) { - var ( - hostsToScan []string - portsToScan []int - ) - - // start by adding `--host` supplied to scan - if len(hosts) > 0 { - hostsToScan = hosts - } + // format and combine flag and positional args + targetHosts = append(targetHosts, util.FormatHostUrls(args, ports, scheme, verbose)...) + targetHosts = append(targetHosts, util.FormatHostUrls(hosts, ports, scheme, verbose)...) - // add hosts from `--subnets` and `--subnet-mask` - for i, subnet := range subnets { + // add more hosts specified with `--subnet` flag + if debug { + log.Debug().Msg("adding hosts from subnets") + } + for _, subnet := range subnets { // subnet string is empty so nothing to do here if subnet == "" { continue } - // NOTE: should we check if subnet is valid here or is it done elsewhere (maybe in GenerateHosts)? + // generate a slice of all hosts to scan from subnets + targetHosts = append(targetHosts, magellan.GenerateHostsWithSubnet(subnet, &subnetMask, ports, scheme)...) + } - // no subnet masks supplied so add a default one for class C private networks - if len(subnetMasks) < i+1 { - subnetMasks = append(subnetMasks, net.IP{255, 255, 255, 0}) + // convert everything into full addresses for scanning + for _, host := range hosts { + var targets []string + for _, port := range ports { + _ = port + targets = append(targets, host) } + targetHosts = append(targetHosts, targets) + } - // generate a slice of all hosts to scan from subnets - hostsToScan = append(hostsToScan, magellan.GenerateHosts(subnet, &subnetMasks[i])...) + // if there are no target hosts, then there's nothing to do + if len(targetHosts) <= 0 { + log.Warn().Msg("nothing to do (no target hosts)") + return + } else { + if len(targetHosts[0]) <= 0 { + log.Warn().Msg("nothing to do (no target hosts)") + return + } + } + + // add default ports for hosts if none are specified with flag + if len(ports) == 0 { + if debug { + log.Debug().Msg("adding default ports") + } + ports = magellan.GetDefaultPorts() } - // add ports to use for scanning - if len(ports) > 0 { - portsToScan = ports + // show the parameters going into the scan + if debug { + combinedTargetHosts := []string{} + for _, targetHost := range targetHosts { + combinedTargetHosts = append(combinedTargetHosts, targetHost...) + } + c := map[string]any{ + "hosts": combinedTargetHosts, + "cache": cachePath, + "concurrency": concurrency, + "protocol": protocol, + "subnets": subnets, + "subnet-masks": subnetMask, + "cert": cacertPath, + "disable-probing": disableProbing, + "disable-caching": disableCache, + } + b, _ := json.MarshalIndent(c, "", " ") + fmt.Printf("%s", string(b)) + } + + // set the number of concurrent requests (1 request per BMC node) + // + // NOTE: The number of concurrent job is equal to the number of hosts by default. + // The max concurrent jobs cannot be greater than the number of hosts. + if concurrency <= 0 { + concurrency = len(targetHosts) } else { - // no ports supplied so only use defaults - portsToScan = magellan.GetDefaultPorts() + concurrency = mathutil.Clamp(len(targetHosts), 1, len(targetHosts)) } // scan and store scanned data in cache - if concurrency <= 0 { - concurrency = mathutil.Clamp(len(hostsToScan), 1, 255) + foundAssets := magellan.ScanForAssets(&magellan.ScanParams{ + TargetHosts: targetHosts, + Scheme: scheme, + Protocol: protocol, + Concurrency: concurrency, + Timeout: timeout, + DisableProbing: disableProbing, + Verbose: verbose, + Debug: debug, + }) + + if len(foundAssets) > 0 && verbose { + log.Info().Any("assets", foundAssets).Msgf("found assets from scan") } - probeStates := magellan.ScanForAssets(hostsToScan, portsToScan, concurrency, timeout, disableProbing, verbose) - if verbose { - format = strings.ToLower(format) - if format == "json" { - b, _ := json.Marshal(probeStates) - fmt.Printf("%s\n", string(b)) - } else { - for _, r := range probeStates { - fmt.Printf("%s:%d (%s) @ %s\n", r.Host, r.Port, r.Protocol, r.Timestamp.Format(time.UnixDate)) - } + + if !disableCache && cachePath != "" { + // make the cache directory path if needed + err := os.MkdirAll(path.Dir(cachePath), 0755) + if err != nil { + log.Printf("failed to make cache directory: %v", err) } - } - // make the dbpath dir if needed - err := os.MkdirAll(path.Dir(cachePath), 0766) - if err != nil { - fmt.Printf("failed tomake database directory: %v", err) + // TODO: change this to use an extensible plugin system for storage solutions + // (i.e. something like cache.InsertScannedAssets(path, assets) which implements a Cache interface) + if len(foundAssets) > 0 { + err = sqlite.InsertScannedAssets(cachePath, foundAssets...) + if err != nil { + log.Error().Err(err).Msg("failed to write scanned assets to cache") + } + if verbose { + log.Info().Msgf("saved assets to cache: %s", cachePath) + } + } else { + log.Warn().Msg("no assets found to save") + } } - sqlite.InsertProbeResults(cachePath, &probeStates) }, } func init() { - scanCmd.Flags().StringSliceVar(&hosts, "host", []string{}, "set additional hosts to scan") - scanCmd.Flags().IntSliceVar(&ports, "port", []int{}, "set the ports to scan") - scanCmd.Flags().StringVar(&format, "format", "", "set the output format") - scanCmd.Flags().StringSliceVar(&subnets, "subnet", []string{}, "set additional subnets") - scanCmd.Flags().IPSliceVar(&subnetMasks, "subnet-mask", []net.IP{}, "set the subnet masks to use for network (must match number of subnets)") - scanCmd.Flags().BoolVar(&disableProbing, "disable-probing", false, "disable probing scanned results for BMC nodes") + // scanCmd.Flags().StringSliceVar(&hosts, "host", []string{}, "set additional hosts to scan") + scanCmd.Flags().StringSliceVar(&hosts, "host", nil, "Add individual hosts to scan. (example: https://my.bmc.com:5000; same as using positional args)") + scanCmd.Flags().IntSliceVar(&ports, "port", nil, "Adds additional ports to scan for each host with unspecified ports.") + scanCmd.Flags().StringVar(&scheme, "scheme", "https", "Set the default scheme to use if not specified in host URI. (default is 'https')") + scanCmd.Flags().StringVar(&protocol, "protocol", "tcp", "Set the default protocol to use in scan. (default is 'tcp')") + scanCmd.Flags().StringSliceVar(&subnets, "subnet", nil, "Add additional hosts from specified subnets to scan.") + scanCmd.Flags().IPMaskVar(&subnetMask, "subnet-mask", net.IPv4Mask(255, 255, 255, 0), "Set the default subnet mask to use for with all subnets not using CIDR notation.") + scanCmd.Flags().BoolVar(&disableProbing, "disable-probing", false, "disable probing found assets for Redfish service(s) running on BMC nodes") + scanCmd.Flags().BoolVar(&disableCache, "disable-cache", false, "disable saving found assets to a cache database specified with 'cache' flag") viper.BindPFlag("scan.hosts", scanCmd.Flags().Lookup("host")) viper.BindPFlag("scan.ports", scanCmd.Flags().Lookup("port")) diff --git a/cmd/update.go b/cmd/update.go index 592cd0f..67bba16 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -28,21 +28,6 @@ var updateCmd = &cobra.Command{ " magellan update --bmc.host 172.16.0.108 --bmc.port 443 --username bmc_username --password bmc_password --firmware-url http://172.16.0.200:8005/firmware/bios/image.RBU --component BIOS\n" + " magellan update --status --bmc.host 172.16.0.108 --bmc.port 443 --username bmc_username --password bmc_password", Run: func(cmd *cobra.Command, args []string) { - // set up update parameters - q := &magellan.UpdateParams{ - FirmwarePath: firmwareUrl, - FirmwareVersion: firmwareVersion, - Component: component, - TransferProtocol: transferProtocol, - QueryParams: magellan.QueryParams{ - Host: host, - Username: username, - Password: password, - Timeout: timeout, - Port: port, - }, - } - // check if required params are set if host == "" || username == "" || password == "" { log.Error().Msg("requires host, user, and pass to be set") @@ -50,7 +35,19 @@ var updateCmd = &cobra.Command{ // get status if flag is set and exit if status { - err := magellan.GetUpdateStatus(q) + err := magellan.GetUpdateStatus(&magellan.UpdateParams{ + FirmwarePath: firmwareUrl, + FirmwareVersion: firmwareVersion, + Component: component, + TransferProtocol: transferProtocol, + CollectParams: magellan.CollectParams{ + Host: host, + Username: username, + Password: password, + Timeout: timeout, + Port: port, + }, + }) if err != nil { log.Error().Err(err).Msgf("failed to get update status") } @@ -58,7 +55,19 @@ var updateCmd = &cobra.Command{ } // initiate a remote update - err := magellan.UpdateFirmwareRemote(q) + err := magellan.UpdateFirmwareRemote(&magellan.UpdateParams{ + FirmwarePath: firmwareUrl, + FirmwareVersion: firmwareVersion, + Component: component, + TransferProtocol: transferProtocol, + CollectParams: magellan.CollectParams{ + Host: host, + Username: username, + Password: password, + Timeout: timeout, + Port: port, + }, + }) if err != nil { log.Error().Err(err).Msgf("failed to update firmware") } @@ -71,7 +80,6 @@ func init() { updateCmd.Flags().StringVar(&username, "username", "", "set the BMC user") updateCmd.Flags().StringVar(&password, "password", "", "set the BMC password") updateCmd.Flags().StringVar(&transferProtocol, "transfer-protocol", "HTTP", "set the transfer protocol") - updateCmd.Flags().StringVar(&protocol, "protocol", "https", "set the Redfish protocol") updateCmd.Flags().StringVar(&firmwareUrl, "firmware.url", "", "set the path to the firmware") updateCmd.Flags().StringVar(&firmwareVersion, "firmware.version", "", "set the version of firmware to be installed") updateCmd.Flags().StringVar(&component, "component", "", "set the component to upgrade") From 6d1dae25ecc9d7ff57038e134db2e94ebc1ffc0e Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 7 Aug 2024 10:59:10 -0600 Subject: [PATCH 22/67] Major 'internal' package refactor --- internal/cache/cache.go | 10 + .../{db => cache}/postgresql/postgresql.go | 0 internal/cache/sqlite/sqlite.go | 97 ++++++++ internal/collect.go | 20 +- internal/db/sqlite/sqlite.go | 95 -------- internal/scan.go | 214 +++++++++++------- internal/update.go | 6 +- internal/util/net.go | 102 +++++++++ internal/util/path.go | 4 - 9 files changed, 358 insertions(+), 190 deletions(-) create mode 100644 internal/cache/cache.go rename internal/{db => cache}/postgresql/postgresql.go (100%) create mode 100644 internal/cache/sqlite/sqlite.go delete mode 100644 internal/db/sqlite/sqlite.go diff --git a/internal/cache/cache.go b/internal/cache/cache.go new file mode 100644 index 0000000..d4dabed --- /dev/null +++ b/internal/cache/cache.go @@ -0,0 +1,10 @@ +package cache + +import "database/sql/driver" + +type Cache[T any] interface { + CreateIfNotExists(path string) (driver.Connector, error) + Insert(path string, data ...T) error + Delete(path string, data ...T) error + Get(path string) ([]T, error) +} diff --git a/internal/db/postgresql/postgresql.go b/internal/cache/postgresql/postgresql.go similarity index 100% rename from internal/db/postgresql/postgresql.go rename to internal/cache/postgresql/postgresql.go diff --git a/internal/cache/sqlite/sqlite.go b/internal/cache/sqlite/sqlite.go new file mode 100644 index 0000000..7f0bbf5 --- /dev/null +++ b/internal/cache/sqlite/sqlite.go @@ -0,0 +1,97 @@ +package sqlite + +import ( + "fmt" + + magellan "github.com/OpenCHAMI/magellan/internal" + + "github.com/jmoiron/sqlx" +) + +const TABLE_NAME = "magellan_scanned_assets" + +func CreateScannedAssetIfNotExists(path string) (*sqlx.DB, error) { + schema := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + host TEXT NOT NULL, + port INTEGER NOT NULL, + protocol TEXT, + state INTEGER, + timestamp TIMESTAMP, + PRIMARY KEY (host, port) + ); + `, TABLE_NAME) + // TODO: it may help with debugging to check for file permissions here first + db, err := sqlx.Open("sqlite3", path) + if err != nil { + return nil, fmt.Errorf("failed to open database: %v", err) + } + db.MustExec(schema) + return db, nil +} + +func InsertScannedAssets(path string, assets ...magellan.ScannedAsset) error { + if assets == nil { + return fmt.Errorf("states == nil") + } + + // create database if it doesn't already exist + db, err := CreateScannedAssetIfNotExists(path) + if err != nil { + return err + } + + // insert all probe states into db + tx := db.MustBegin() + for _, state := range assets { + sql := fmt.Sprintf(`INSERT OR REPLACE INTO %s (host, port, protocol, state, timestamp) + VALUES (:host, :port, :protocol, :state, :timestamp);`, TABLE_NAME) + _, err := tx.NamedExec(sql, &state) + if err != nil { + fmt.Printf("failed to execute transaction: %v\n", err) + } + } + err = tx.Commit() + if err != nil { + return fmt.Errorf("failed to commit transaction: %v", err) + } + return nil +} + +func DeleteScannedAssets(path string, results ...magellan.ScannedAsset) error { + if results == nil { + return fmt.Errorf("no assets found") + } + db, err := sqlx.Open("sqlite3", path) + if err != nil { + return fmt.Errorf("failed to open database: %v", err) + } + tx := db.MustBegin() + for _, state := range results { + sql := fmt.Sprintf(`DELETE FROM %s WHERE host = :host, port = :port;`, TABLE_NAME) + _, err := tx.NamedExec(sql, &state) + if err != nil { + fmt.Printf("failed to execute transaction: %v\n", err) + } + } + + err = tx.Commit() + if err != nil { + return fmt.Errorf("failed to commit transaction: %v", err) + } + return nil +} + +func GetScannedAssets(path string) ([]magellan.ScannedAsset, error) { + db, err := sqlx.Open("sqlite3", path) + if err != nil { + return nil, fmt.Errorf("failed to open database: %v", err) + } + + results := []magellan.ScannedAsset{} + err = db.Select(&results, fmt.Sprintf("SELECT * FROM %s ORDER BY host ASC, port ASC;", TABLE_NAME)) + if err != nil { + return nil, fmt.Errorf("failed to retrieve assets: %v", err) + } + return results, nil +} diff --git a/internal/collect.go b/internal/collect.go index 2666c50..6705a69 100644 --- a/internal/collect.go +++ b/internal/collect.go @@ -27,11 +27,9 @@ const ( HTTPS_PORT = 443 ) -// QueryParams is a collections of common parameters passed to the CLI. -// Each CLI subcommand has a corresponding implementation function that -// takes an object as an argument. However, the implementation may not -// use all of the properties within the object. -type QueryParams struct { +// CollectParams is a collection of common parameters passed to the CLI +// for the 'collect' subcommand. +type CollectParams struct { Host string // set by the 'host' flag Port int // set by the 'port' flag Username string // set the BMC username with the 'username' flag @@ -50,7 +48,7 @@ type QueryParams struct { // // Requests can be made to several of the nodes using a goroutine by setting the q.Concurrency // property value between 1 and 255. -func CollectInventory(scannedResults *[]ScannedResult, params *QueryParams) error { +func CollectInventory(scannedResults *[]ScannedAsset, params *CollectParams) error { // check for available probe states if scannedResults == nil { return fmt.Errorf("no probe states found") @@ -65,7 +63,7 @@ func CollectInventory(scannedResults *[]ScannedResult, params *QueryParams) erro wg sync.WaitGroup found = make([]string, 0, len(*scannedResults)) done = make(chan struct{}, params.Concurrency+1) - chanScannedResult = make(chan ScannedResult, params.Concurrency+1) + chanScannedResult = make(chan ScannedAsset, params.Concurrency+1) outputPath = path.Clean(params.OutputPath) smdClient = client.NewClient( client.WithSecureTLS(params.CaCertPath), @@ -94,7 +92,7 @@ func CollectInventory(scannedResults *[]ScannedResult, params *QueryParams) erro // TODO: use pkg/crawler to request inventory data via Redfish systems, err := crawler.CrawlBMC(crawler.CrawlerConfig{ - URI: fmt.Sprintf("https://%s:%d", sr.Host, sr.Port), + URI: fmt.Sprintf("%s:%d", sr.Host, sr.Port), Username: params.Username, Password: params.Password, Insecure: true, @@ -131,14 +129,14 @@ func CollectInventory(scannedResults *[]ScannedResult, params *QueryParams) erro // write JSON data to file if output path is set using hive partitioning strategy if outputPath != "" { - err = os.MkdirAll(outputPath, os.ModeDir) + err = os.MkdirAll(outputPath, 0o644) if err != nil { log.Error().Err(err).Msg("failed to make directory for output") } else { // make the output directory to store files outputPath, err := util.MakeOutputDirectory(outputPath, false) if err != nil { - log.Error().Msgf("failed to make output directory: %v", err) + log.Error().Err(err).Msg("failed to make output directory") } else { // write the output to the final path err = os.WriteFile(path.Clean(fmt.Sprintf("%s/%s/%d.json", params.Host, outputPath, time.Now().Unix())), body, os.ModePerm) @@ -197,6 +195,6 @@ func CollectInventory(scannedResults *[]ScannedResult, params *QueryParams) erro return nil } -func baseRedfishUrl(q *QueryParams) string { +func baseRedfishUrl(q *CollectParams) string { return fmt.Sprintf("%s:%d", q.Host, q.Port) } diff --git a/internal/db/sqlite/sqlite.go b/internal/db/sqlite/sqlite.go deleted file mode 100644 index 9a60240..0000000 --- a/internal/db/sqlite/sqlite.go +++ /dev/null @@ -1,95 +0,0 @@ -package sqlite - -import ( - "fmt" - - magellan "github.com/OpenCHAMI/magellan/internal" - - "github.com/jmoiron/sqlx" -) - -func CreateProbeResultsIfNotExists(path string) (*sqlx.DB, error) { - schema := ` - CREATE TABLE IF NOT EXISTS magellan_scanned_ports ( - host TEXT NOT NULL, - port INTEGER NOT NULL, - protocol TEXT, - state INTEGER, - timestamp TIMESTAMP, - PRIMARY KEY (host, port) - ); - ` - // TODO: it may help with debugging to check for file permissions here first - db, err := sqlx.Open("sqlite3", path) - if err != nil { - return nil, fmt.Errorf("failed toopen database: %v", err) - } - db.MustExec(schema) - return db, nil -} - -func InsertProbeResults(path string, states *[]magellan.ScannedResult) error { - if states == nil { - return fmt.Errorf("states == nil") - } - - // create database if it doesn't already exist - db, err := CreateProbeResultsIfNotExists(path) - if err != nil { - return err - } - - // insert all probe states into db - tx := db.MustBegin() - for _, state := range *states { - sql := `INSERT OR REPLACE INTO magellan_scanned_ports (host, port, protocol, state, timestamp) - VALUES (:host, :port, :protocol, :state, :timestamp);` - _, err := tx.NamedExec(sql, &state) - if err != nil { - fmt.Printf("failed toexecute transaction: %v\n", err) - } - } - err = tx.Commit() - if err != nil { - return fmt.Errorf("failed tocommit transaction: %v", err) - } - return nil -} - -func DeleteProbeResults(path string, results *[]magellan.ScannedResult) error { - if results == nil { - return fmt.Errorf("no probe results found") - } - db, err := sqlx.Open("sqlite3", path) - if err != nil { - return fmt.Errorf("failed toopen database: %v", err) - } - tx := db.MustBegin() - for _, state := range *results { - sql := `DELETE FROM magellan_scanned_ports WHERE host = :host, port = :port;` - _, err := tx.NamedExec(sql, &state) - if err != nil { - fmt.Printf("failed toexecute transaction: %v\n", err) - } - } - - err = tx.Commit() - if err != nil { - return fmt.Errorf("failed tocommit transaction: %v", err) - } - return nil -} - -func GetScannedResults(path string) ([]magellan.ScannedResult, error) { - db, err := sqlx.Open("sqlite3", path) - if err != nil { - return nil, fmt.Errorf("failed toopen database: %v", err) - } - - results := []magellan.ScannedResult{} - err = db.Select(&results, "SELECT * FROM magellan_scanned_ports ORDER BY host ASC, port ASC;") - if err != nil { - return nil, fmt.Errorf("failed toretrieve probes: %v", err) - } - return results, nil -} diff --git a/internal/scan.go b/internal/scan.go index 660aede..ebab103 100644 --- a/internal/scan.go +++ b/internal/scan.go @@ -5,13 +5,16 @@ import ( "math" "net" "net/http" + "net/url" + "strconv" "sync" "time" "github.com/OpenCHAMI/magellan/internal/util" + "github.com/rs/zerolog/log" ) -type ScannedResult struct { +type ScannedAsset struct { Host string `json:"host"` Port int `json:"port"` Protocol string `json:"protocol"` @@ -19,9 +22,21 @@ type ScannedResult struct { Timestamp time.Time `json:"timestamp"` } +// ScanParams is a collection of commom parameters passed to the CLI +type ScanParams struct { + TargetHosts [][]string + Scheme string + Protocol string + Concurrency int + Timeout int + DisableProbing bool + Verbose bool + Debug bool +} + // ScanForAssets() performs a net scan on a network to find available services -// running. The function expects a list of hosts and ports to make requests. -// Note that each all ports will be used per host. +// running. The function expects a list of targets (as [][]string) to make requests. +// The 2D list is to permit one goroutine per BMC node when making each request. // // This function runs in a goroutine with the "concurrency" flag setting the // number of concurrent requests. Only one request is made to each BMC node @@ -34,54 +49,67 @@ type ScannedResult struct { // remove the service from being stored in the list of scanned results. // // Returns a list of scanned results to be stored in cache (but isn't doing here). -func ScanForAssets(hosts []string, ports []int, concurrency int, timeout int, disableProbing bool, verbose bool) []ScannedResult { +func ScanForAssets(params *ScanParams) []ScannedAsset { var ( - results = make([]ScannedResult, 0, len(hosts)) - done = make(chan struct{}, concurrency+1) - chanHost = make(chan string, concurrency+1) + results = make([]ScannedAsset, 0, len(params.TargetHosts)) + done = make(chan struct{}, params.Concurrency+1) + chanHosts = make(chan []string, params.Concurrency+1) ) + if params.Verbose { + log.Info().Msg("starting scan...") + } + var wg sync.WaitGroup - wg.Add(concurrency) - for i := 0; i < concurrency; i++ { + wg.Add(params.Concurrency) + for i := 0; i < params.Concurrency; i++ { go func() { for { - host, ok := <-chanHost + hosts, ok := <-chanHosts if !ok { wg.Done() return } - scannedResults := rawConnect(host, ports, timeout, true) - if !disableProbing { - probeResults := []ScannedResult{} - for _, result := range scannedResults { - url := fmt.Sprintf("https://%s:%d/redfish/v1/", result.Host, result.Port) - res, _, err := util.MakeRequest(nil, url, "GET", nil, nil) - if err != nil || res == nil { - if verbose { - fmt.Printf("failed to make request: %v\n", err) - } - continue - } else if res.StatusCode != http.StatusOK { - if verbose { - fmt.Printf("request returned code: %v\n", res.StatusCode) + for _, host := range hosts { + foundAssets, err := rawConnect(host, params.Protocol, params.Timeout, true) + // if we failed to connect, exit from the function + if err != nil { + if params.Verbose { + log.Debug().Err(err).Msgf("failed to connect to host (%s)", host) + } + wg.Done() + return + } + if !params.DisableProbing { + assetsToAdd := []ScannedAsset{} + for _, foundAsset := range foundAssets { + url := fmt.Sprintf("%s://%s/redfish/v1/", params.Scheme, foundAsset.Host) + res, _, err := util.MakeRequest(nil, url, http.MethodGet, nil, nil) + if err != nil || res == nil { + if params.Verbose { + log.Printf("failed to make request: %v\n", err) + } + continue + } else if res.StatusCode != http.StatusOK { + if params.Verbose { + log.Printf("request returned code: %v\n", res.StatusCode) + } + continue + } else { + assetsToAdd = append(assetsToAdd, foundAsset) } - continue - } else { - probeResults = append(probeResults, result) } + results = append(results, assetsToAdd...) + } else { + results = append(results, foundAssets...) } - results = append(results, probeResults...) - } else { - results = append(results, scannedResults...) } - } }() } - for _, host := range hosts { - chanHost <- host + for _, hosts := range params.TargetHosts { + chanHosts <- hosts } go func() { select { @@ -92,13 +120,17 @@ func ScanForAssets(hosts []string, ports []int, concurrency int, timeout int, di time.Sleep(1000) } }() - close(chanHost) + close(chanHosts) wg.Wait() close(done) + + if params.Verbose { + log.Info().Msg("scan complete") + } return results } -// GenerateHosts() builds a list of hosts to scan using the "subnet" +// GenerateHostsWithSubnet() builds a list of hosts to scan using the "subnet" // and "subnetMask" arguments passed. The function is capable of // distinguishing between IP formats: a subnet with just an IP address (172.16.0.0) and // a subnet with IP address and CIDR (172.16.0.0/24). @@ -106,83 +138,111 @@ func ScanForAssets(hosts []string, ports []int, concurrency int, timeout int, di // NOTE: If a IP address is provided with CIDR, then the "subnetMask" // parameter will be ignored. If neither is provided, then the default // subnet mask will be used instead. -func GenerateHosts(subnet string, subnetMask *net.IP) []string { +func GenerateHostsWithSubnet(subnet string, subnetMask *net.IPMask, additionalPorts []int, defaultScheme string) [][]string { if subnet == "" || subnetMask == nil { return nil } - // convert subnets from string to net.IP + // convert subnets from string to net.IP to test if CIDR is included subnetIp := net.ParseIP(subnet) if subnetIp == nil { - // try parse CIDR instead + // not a valid IP so try again with CIDR ip, network, err := net.ParseCIDR(subnet) if err != nil { return nil } subnetIp = ip - if network != nil { - t := net.IP(network.Mask) - subnetMask = &t + if network == nil { + // use the default subnet mask if a valid one is not provided + network = &net.IPNet{ + IP: subnetIp, + Mask: net.IPv4Mask(255, 255, 255, 0), + } } + subnetMask = &network.Mask } - mask := net.IPMask(subnetMask.To4()) - - // if no subnet mask, use a default 24-bit mask (for now) - return generateHosts(&subnetIp, &mask) + // generate new IPs from subnet and format to full URL + subnetIps := generateIPsWithSubnet(&subnetIp, subnetMask) + return util.FormatIPUrls(subnetIps, additionalPorts, defaultScheme, false) } +// GetDefaultPorts() returns a list of default ports. The only reason to have +// this function is to add/remove ports without affecting usage. func GetDefaultPorts() []int { return []int{HTTPS_PORT} } -func rawConnect(host string, ports []int, timeout int, keepOpenOnly bool) []ScannedResult { - results := []ScannedResult{} - for _, p := range ports { - result := ScannedResult{ - Host: host, - Port: p, - Protocol: "tcp", +// rawConnect() tries to connect to the host using DialTimeout() and waits +// until a response is receive or if the timeout (in seconds) expires. This +// function expects a full URL such as https://my.bmc.host:443/ to make the +// connection. +func rawConnect(address string, protocol string, timeoutSeconds int, keepOpenOnly bool) ([]ScannedAsset, error) { + uri, err := url.ParseRequestURI(address) + if err != nil { + return nil, fmt.Errorf("failed to split host/port: %w", err) + } + + // convert port to its "proper" type + port, err := strconv.Atoi(uri.Port()) + if err != nil { + return nil, fmt.Errorf("failed to convert port to integer type: %w", err) + } + + var ( + timeoutDuration = time.Second * time.Duration(timeoutSeconds) + assets []ScannedAsset + asset = ScannedAsset{ + Host: uri.Host, + Port: port, + Protocol: protocol, State: false, Timestamp: time.Now(), } - t := time.Second * time.Duration(timeout) - port := fmt.Sprint(p) - conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, port), t) - if err != nil { - result.State = false - // fmt.Println("Connecting error:", err) - } - if conn != nil { - result.State = true - defer conn.Close() - // fmt.Println("Opened", net.JoinHostPort(host, port)) - } - if keepOpenOnly { - if result.State { - results = append(results, result) - } - } else { - results = append(results, result) + ) + + // try to conntect to host (expects host in format [10.0.0.0]:443) + target := fmt.Sprintf("[%s]:%s", uri.Hostname(), uri.Port()) + conn, err := net.DialTimeout(protocol, target, timeoutDuration) + if err != nil { + asset.State = false + return nil, fmt.Errorf("failed to dial host: %w", err) + } + if conn != nil { + asset.State = true + defer conn.Close() + } + if keepOpenOnly { + if asset.State { + assets = append(assets, asset) } + } else { + assets = append(assets, asset) } - return results + return assets, nil } -func generateHosts(ip *net.IP, mask *net.IPMask) []string { +// generateIPsWithSubnet() returns a collection of host IP strings with a +// provided subnet mask. +// +// TODO: add a way for filtering/exclude specific IPs and IP ranges. +func generateIPsWithSubnet(ip *net.IP, mask *net.IPMask) []string { + // check if subnet IP and mask are valid + if ip == nil || mask == nil { + log.Error().Msg("invalid subnet IP or mask (ip == nil or mask == nil)") + return nil + } // get all IP addresses in network - ones, _ := mask.Size() + ones, bits := mask.Size() hosts := []string{} - end := int(math.Pow(2, float64((32-ones)))) - 1 + end := int(math.Pow(2, float64((bits-ones)))) - 1 for i := 0; i < end; i++ { - // ip[3] = byte(i) ip = util.GetNextIP(ip, 1) if ip == nil { continue } - // host := fmt.Sprintf("%v.%v.%v.%v", (*ip)[0], (*ip)[1], (*ip)[2], (*ip)[3]) - // fmt.Printf("host: %v\n", ip.String()) + hosts = append(hosts, ip.String()) } return hosts diff --git a/internal/update.go b/internal/update.go index 8c07dce..8fca958 100644 --- a/internal/update.go +++ b/internal/update.go @@ -9,7 +9,7 @@ import ( ) type UpdateParams struct { - QueryParams + CollectParams FirmwarePath string FirmwareVersion string Component string @@ -20,7 +20,7 @@ type UpdateParams struct { // The function expects the firmware URL, firmware version, and component flags to be // set from the CLI to perform a firmware update. func UpdateFirmwareRemote(q *UpdateParams) error { - url := baseRedfishUrl(&q.QueryParams) + "/redfish/v1/UpdateService/Actions/SimpleUpdate" + url := baseRedfishUrl(&q.CollectParams) + "/redfish/v1/UpdateService/Actions/SimpleUpdate" headers := map[string]string{ "Content-Type": "application/json", "cache-control": "no-cache", @@ -47,7 +47,7 @@ func UpdateFirmwareRemote(q *UpdateParams) error { } func GetUpdateStatus(q *UpdateParams) error { - url := baseRedfishUrl(&q.QueryParams) + "/redfish/v1/UpdateService" + url := baseRedfishUrl(&q.CollectParams) + "/redfish/v1/UpdateService" res, body, err := util.MakeRequest(nil, url, "GET", nil, nil) if err != nil { return fmt.Errorf("something went wrong: %v", err) diff --git a/internal/util/net.go b/internal/util/net.go index f0528f1..5999ac7 100644 --- a/internal/util/net.go +++ b/internal/util/net.go @@ -7,6 +7,10 @@ import ( "io" "net" "net/http" + "net/url" + "strings" + + "github.com/rs/zerolog/log" ) // HTTP aliases for readibility @@ -78,3 +82,101 @@ func MakeRequest(client *http.Client, url string, httpMethod string, body HTTPBo } return res, b, err } + +// FormatHostUrls() takes a list of hosts and ports and builds full URLs in the +// form of scheme://host:port. If no scheme is provided, it will use "https" by +// default. +// +// Returns a 2D string slice where each slice contains URL host strings for each +// port. The intention is to have all of the URLs for a single host combined into +// a single slice to initiate one goroutine per host, but making request to multiple +// ports. +func FormatHostUrls(hosts []string, ports []int, scheme string, verbose bool) [][]string { + // format each positional arg as a complete URL + var formattedHosts [][]string + for _, host := range hosts { + uri, err := url.ParseRequestURI(host) + if err != nil { + if verbose { + log.Warn().Msgf("invalid URI parsed: %s", host) + } + continue + } + + // check if scheme is set, if not set it with flag or default value ('https' if flag is not set) + if uri.Scheme == "" { + if scheme != "" { + uri.Scheme = scheme + } else { + // hardcoded assumption + uri.Scheme = "https" + } + } + + // tidy up slashes and update arg with new value + uri.Path = strings.TrimSuffix(uri.Path, "/") + uri.Path = strings.ReplaceAll(uri.Path, "//", "/") + + // for hosts with unspecified ports, add ports to scan from flag + if uri.Port() == "" { + var tmp []string + for _, port := range ports { + uri.Host += fmt.Sprintf(":%d", port) + tmp = append(tmp, uri.String()) + } + formattedHosts = append(formattedHosts, tmp) + } else { + formattedHosts = append(formattedHosts, []string{uri.String()}) + } + + } + return formattedHosts +} + +// FormatIPUrls() takes a list of IP addresses and ports and builds full URLs in the +// form of scheme://host:port. If no scheme is provided, it will use "https" by +// default. +// +// Returns a 2D string slice where each slice contains URL host strings for each +// port. The intention is to have all of the URLs for a single host combined into +// a single slice to initiate one goroutine per host, but making request to multiple +// ports. +func FormatIPUrls(ips []string, ports []int, scheme string, verbose bool) [][]string { + // format each positional arg as a complete URL + var formattedHosts [][]string + for _, ip := range ips { + // if parsing completely fails, try to build new URL object + uri := &url.URL{ + Scheme: scheme, + Host: ip, + } + + // check if scheme is set, if not set it with flag or default value ('https' if flag is not set) + if uri.Scheme == "" { + if scheme != "" { + uri.Scheme = scheme + } else { + // hardcoded assumption + uri.Scheme = "https" + } + } + + // tidy up slashes and update arg with new value + uri.Path = strings.TrimSuffix(uri.Path, "/") + uri.Path = strings.ReplaceAll(uri.Path, "//", "/") + + // for hosts with unspecified ports, add ports to scan from flag + if uri.Port() == "" { + var tmp []string + for _, port := range ports { + uri.Host = fmt.Sprintf("%s:%d", ip, port) + tmp = append(tmp, uri.String()) + } + formattedHosts = append(formattedHosts, tmp) + } else { + formattedHosts = append(formattedHosts, []string{uri.String()}) + } + + } + return formattedHosts +} diff --git a/internal/util/path.go b/internal/util/path.go index 19f6afe..c2e3e58 100644 --- a/internal/util/path.go +++ b/internal/util/path.go @@ -44,10 +44,6 @@ func SplitPathForViper(path string) (string, string, string) { // // Returns the final path that was created if no errors occurred. Otherwise, // it returns an empty string with an error. -// -// TODO: Refactor this function for hive partitioning or possibly move into -// the logging package. -// TODO: Add an option to force overwriting the path. func MakeOutputDirectory(path string, overwrite bool) (string, error) { // get the current data + time using Go's stupid formatting t := time.Now() From 22b88a908a18a2f58076dd947832cb53a2fc741f Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 7 Aug 2024 10:59:46 -0600 Subject: [PATCH 23/67] Updated example config --- config.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/config.yaml b/config.yaml index 30f244c..11b0c28 100644 --- a/config.yaml +++ b/config.yaml @@ -5,15 +5,18 @@ scan: - "172.16.0.0" - "172.16.0.0/24" subnet-masks: + - "255.255.255.0" ports: - - 433 + - 443 disable-probing: false + disable-caching: false + protocol: "tcp" + scheme: "https" collect: - # host: smd-host - # port: smd-port username: "admin" password: "password" - protocol: "https" + protocol: "tcp" + scheme: "https" output: "/tmp/magellan/data/" threads: 1 force-update: false @@ -23,8 +26,7 @@ update: port: 443 username: "admin" password: "password" - transfer-protocol: "HTTP" - protocol: "https" + transfer-protocol: "HTTPS" firmware: url: version: From dd829cd10e382723ce832d91bac2a2022df90421 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 7 Aug 2024 13:14:07 -0600 Subject: [PATCH 24/67] Fixed '--subnet' flag not adding hosts to scan --- cmd/scan.go | 17 ++++------------- internal/cache/cache.go | 4 +++- internal/cache/storage.go | 28 ++++++++++++++++++++++++++++ internal/util/net.go | 20 ++++++++------------ 4 files changed, 43 insertions(+), 26 deletions(-) create mode 100644 internal/cache/storage.go diff --git a/cmd/scan.go b/cmd/scan.go index 65baa16..e48173c 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -78,26 +78,17 @@ var scanCmd = &cobra.Command{ } // generate a slice of all hosts to scan from subnets - targetHosts = append(targetHosts, magellan.GenerateHostsWithSubnet(subnet, &subnetMask, ports, scheme)...) - } - - // convert everything into full addresses for scanning - for _, host := range hosts { - var targets []string - for _, port := range ports { - _ = port - targets = append(targets, host) - } - targetHosts = append(targetHosts, targets) + subnetHosts := magellan.GenerateHostsWithSubnet(subnet, &subnetMask, ports, scheme) + targetHosts = append(targetHosts, subnetHosts...) } // if there are no target hosts, then there's nothing to do if len(targetHosts) <= 0 { - log.Warn().Msg("nothing to do (no target hosts)") + log.Warn().Msg("nothing to do (no valid target hosts)") return } else { if len(targetHosts[0]) <= 0 { - log.Warn().Msg("nothing to do (no target hosts)") + log.Warn().Msg("nothing to do (no valid target hosts)") return } } diff --git a/internal/cache/cache.go b/internal/cache/cache.go index d4dabed..96513de 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -1,6 +1,8 @@ package cache -import "database/sql/driver" +import ( + "database/sql/driver" +) type Cache[T any] interface { CreateIfNotExists(path string) (driver.Connector, error) diff --git a/internal/cache/storage.go b/internal/cache/storage.go new file mode 100644 index 0000000..06b42a9 --- /dev/null +++ b/internal/cache/storage.go @@ -0,0 +1,28 @@ +package cache + +import "github.com/google/uuid" + +type Storage[T any] interface { + Save(id uuid.UUID, val T, varargs ...T) error + Get(id uuid.UUID) (T, error) + Update(id uuid.UUID, val T) error + Delete(id uuid.UUID) error +} + +type Compute struct{} +type BMC struct{} + +type Node[T any] struct { +} + +type NodeStorage struct { + Storage[Node[Compute]] +} + +type BMCStorage struct { + Storage[Node[BMC]] +} + +func (ns *NodeStorage) Save(id uuid.UUID, val Node[Compute], varargs ...Node[Compute]) { + +} diff --git a/internal/util/net.go b/internal/util/net.go index 5999ac7..50bbbfa 100644 --- a/internal/util/net.go +++ b/internal/util/net.go @@ -145,31 +145,27 @@ func FormatIPUrls(ips []string, ports []int, scheme string, verbose bool) [][]st // format each positional arg as a complete URL var formattedHosts [][]string for _, ip := range ips { - // if parsing completely fails, try to build new URL object + if scheme == "" { + scheme = "https" + } + // make an entirely new object since we're expecting just IPs uri := &url.URL{ Scheme: scheme, Host: ip, } - // check if scheme is set, if not set it with flag or default value ('https' if flag is not set) - if uri.Scheme == "" { - if scheme != "" { - uri.Scheme = scheme - } else { - // hardcoded assumption - uri.Scheme = "https" - } - } - // tidy up slashes and update arg with new value uri.Path = strings.TrimSuffix(uri.Path, "/") uri.Path = strings.ReplaceAll(uri.Path, "//", "/") // for hosts with unspecified ports, add ports to scan from flag if uri.Port() == "" { + if len(ports) == 0 { + ports = append(ports, 443) + } var tmp []string for _, port := range ports { - uri.Host = fmt.Sprintf("%s:%d", ip, port) + uri.Host += fmt.Sprintf(":%d", port) tmp = append(tmp, uri.String()) } formattedHosts = append(formattedHosts, tmp) From 890e3392ed443edba3a515bd6903a0e0b1c9c3f1 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 7 Aug 2024 13:15:09 -0600 Subject: [PATCH 25/67] Added TODO comments to tests and other minor change --- tests/api_test.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/api_test.go b/tests/api_test.go index 60c1886..05f7cd2 100644 --- a/tests/api_test.go +++ b/tests/api_test.go @@ -30,21 +30,27 @@ func TestScanAndCollect(t *testing.T) { } // do a collect on the emulator cluster to collect Redfish info - magellan.CollectInventory(&results, &magellan.QueryParams{}) + magellan.CollectInventory(&results, &magellan.CollectParams{}) } func TestCrawlCommand(t *testing.T) { - + // TODO: add test to check the crawl command's behavior } func TestListCommand(t *testing.T) { - + // TODO: add test to check the list command's output } func TestUpdateCommand(t *testing.T) { - + // TODO: add test that does a Redfish simple update checking it success and + // failure points } func TestGofishFunctions(t *testing.T) { + // TODO: add test that checks certain gofish function output to make sure + // gofish's output isn't changing spontaneously and remains predictable +} +func TestGenerateHosts(t *testing.T) { + // TODO: add test to generate hosts using a collection of subnets/masks } From 88fb71b43627bbd9c7f408e901d9ed5e8447a9fb Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 7 Aug 2024 19:03:27 -0600 Subject: [PATCH 26/67] Removed storage file --- internal/cache/storage.go | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 internal/cache/storage.go diff --git a/internal/cache/storage.go b/internal/cache/storage.go deleted file mode 100644 index 06b42a9..0000000 --- a/internal/cache/storage.go +++ /dev/null @@ -1,28 +0,0 @@ -package cache - -import "github.com/google/uuid" - -type Storage[T any] interface { - Save(id uuid.UUID, val T, varargs ...T) error - Get(id uuid.UUID) (T, error) - Update(id uuid.UUID, val T) error - Delete(id uuid.UUID) error -} - -type Compute struct{} -type BMC struct{} - -type Node[T any] struct { -} - -type NodeStorage struct { - Storage[Node[Compute]] -} - -type BMCStorage struct { - Storage[Node[BMC]] -} - -func (ns *NodeStorage) Save(id uuid.UUID, val Node[Compute], varargs ...Node[Compute]) { - -} From 39ad0d75c674c2ec91fddd22efbac10e7440a6fb Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 7 Aug 2024 19:16:43 -0600 Subject: [PATCH 27/67] Changed host to hostname being stored in cache --- internal/scan.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/scan.go b/internal/scan.go index ebab103..0accae9 100644 --- a/internal/scan.go +++ b/internal/scan.go @@ -132,8 +132,8 @@ func ScanForAssets(params *ScanParams) []ScannedAsset { // GenerateHostsWithSubnet() builds a list of hosts to scan using the "subnet" // and "subnetMask" arguments passed. The function is capable of -// distinguishing between IP formats: a subnet with just an IP address (172.16.0.0) and -// a subnet with IP address and CIDR (172.16.0.0/24). +// distinguishing between IP formats: a subnet with just an IP address (172.16.0.0) +// and a subnet with IP address and CIDR (172.16.0.0/24). // // NOTE: If a IP address is provided with CIDR, then the "subnetMask" // parameter will be ignored. If neither is provided, then the default @@ -193,7 +193,7 @@ func rawConnect(address string, protocol string, timeoutSeconds int, keepOpenOnl timeoutDuration = time.Second * time.Duration(timeoutSeconds) assets []ScannedAsset asset = ScannedAsset{ - Host: uri.Host, + Host: uri.Hostname(), Port: port, Protocol: protocol, State: false, From 3d6daa757a3a1487c6ea59421d09f8900923e5e9 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 7 Aug 2024 19:20:11 -0600 Subject: [PATCH 28/67] Fixed error message format for list command --- cmd/list.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/list.go b/cmd/list.go index 8d788d2..26cc1fc 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -9,7 +9,6 @@ import ( "github.com/OpenCHAMI/magellan/internal/cache/sqlite" "github.com/rs/zerolog/log" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -27,7 +26,7 @@ var listCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { scannedResults, err := sqlite.GetScannedAssets(cachePath) if err != nil { - logrus.Errorf("failed to get scanned assets: %v\n", err) + log.Error().Err(err).Msg("failed to get scanned assets") } format = strings.ToLower(format) if format == "json" { @@ -45,6 +44,6 @@ var listCmd = &cobra.Command{ } func init() { - listCmd.Flags().StringVar(&format, "format", "", "set the output format") + listCmd.Flags().StringVar(&format, "format", "", "set the output format (json|default)") rootCmd.AddCommand(listCmd) } From 4cc3f7f7acb8d3280e4e3b5c305da478b355e75d Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Wed, 7 Aug 2024 19:57:37 -0600 Subject: [PATCH 29/67] Changed order of adding default ports to add host correctly --- cmd/scan.go | 17 +++++++++-------- internal/util/net.go | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/cmd/scan.go b/cmd/scan.go index e48173c..3e9a9d0 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -63,6 +63,14 @@ var scanCmd = &cobra.Command{ // assumes subnet without CIDR has a subnet-mask of 255.255.0.0 " magellan scan --subnet 10.0.0.0/24 --subnet 172.16.0.0 --subnet-mask 255.255.0.0 --cache ./assets.db", Run: func(cmd *cobra.Command, args []string) { + // add default ports for hosts if none are specified with flag + if len(ports) == 0 { + if debug { + log.Debug().Msg("adding default ports") + } + ports = magellan.GetDefaultPorts() + } + // format and combine flag and positional args targetHosts = append(targetHosts, util.FormatHostUrls(args, ports, scheme, verbose)...) targetHosts = append(targetHosts, util.FormatHostUrls(hosts, ports, scheme, verbose)...) @@ -93,14 +101,6 @@ var scanCmd = &cobra.Command{ } } - // add default ports for hosts if none are specified with flag - if len(ports) == 0 { - if debug { - log.Debug().Msg("adding default ports") - } - ports = magellan.GetDefaultPorts() - } - // show the parameters going into the scan if debug { combinedTargetHosts := []string{} @@ -133,6 +133,7 @@ var scanCmd = &cobra.Command{ } // scan and store scanned data in cache + fmt.Printf("targets: %v\n", targetHosts) foundAssets := magellan.ScanForAssets(&magellan.ScanParams{ TargetHosts: targetHosts, Scheme: scheme, diff --git a/internal/util/net.go b/internal/util/net.go index 50bbbfa..76749aa 100644 --- a/internal/util/net.go +++ b/internal/util/net.go @@ -155,8 +155,8 @@ func FormatIPUrls(ips []string, ports []int, scheme string, verbose bool) [][]st } // tidy up slashes and update arg with new value - uri.Path = strings.TrimSuffix(uri.Path, "/") uri.Path = strings.ReplaceAll(uri.Path, "//", "/") + uri.Path = strings.TrimSuffix(uri.Path, "/") // for hosts with unspecified ports, add ports to scan from flag if uri.Port() == "" { From 77b6a9ba7be894d970a7201bc5c73c6e996ef76d Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 8 Aug 2024 12:21:49 -0600 Subject: [PATCH 30/67] Added check for output directory for collect --- internal/collect.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/internal/collect.go b/internal/collect.go index 6705a69..88cde9c 100644 --- a/internal/collect.go +++ b/internal/collect.go @@ -129,19 +129,23 @@ func CollectInventory(scannedResults *[]ScannedAsset, params *CollectParams) err // write JSON data to file if output path is set using hive partitioning strategy if outputPath != "" { - err = os.MkdirAll(outputPath, 0o644) - if err != nil { - log.Error().Err(err).Msg("failed to make directory for output") - } else { - // make the output directory to store files - outputPath, err := util.MakeOutputDirectory(outputPath, false) + // make directory if it does exists + exists, err := util.PathExists(outputPath) + if err == nil && !exists { + err = os.MkdirAll(outputPath, 0o644) if err != nil { - log.Error().Err(err).Msg("failed to make output directory") + log.Error().Err(err).Msg("failed to make directory for output") } else { - // write the output to the final path - err = os.WriteFile(path.Clean(fmt.Sprintf("%s/%s/%d.json", params.Host, outputPath, time.Now().Unix())), body, os.ModePerm) + // make the output directory to store files + outputPath, err := util.MakeOutputDirectory(outputPath, false) if err != nil { - log.Error().Err(err).Msgf("failed to write data to file") + log.Error().Err(err).Msg("failed to make output directory") + } else { + // write the output to the final path + err = os.WriteFile(path.Clean(fmt.Sprintf("%s/%s/%d.json", params.Host, outputPath, time.Now().Unix())), body, os.ModePerm) + if err != nil { + log.Error().Err(err).Msgf("failed to write data to file") + } } } } From 46fc35d4adbf876a54e9cb10d01dd0781932e711 Mon Sep 17 00:00:00 2001 From: David Allen Date: Thu, 8 Aug 2024 12:23:10 -0600 Subject: [PATCH 31/67] Removed extra print statement --- cmd/scan.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/scan.go b/cmd/scan.go index 3e9a9d0..aa5d8ab 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -133,7 +133,6 @@ var scanCmd = &cobra.Command{ } // scan and store scanned data in cache - fmt.Printf("targets: %v\n", targetHosts) foundAssets := magellan.ScanForAssets(&magellan.ScanParams{ TargetHosts: targetHosts, Scheme: scheme, From 8a2541717ddc1c013945bf3ae0c64a491a141f40 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 9 Aug 2024 07:42:28 -0600 Subject: [PATCH 32/67] Added flag to show cache info with list command and other minor changes --- cmd/collect.go | 7 +++---- cmd/list.go | 14 +++++++++++++- cmd/root.go | 12 ++++++++---- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/cmd/collect.go b/cmd/collect.go index b2cd151..af4df5e 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -40,8 +40,8 @@ var collectCmd = &cobra.Command{ if accessToken == "" { var err error accessToken, err = util.LoadAccessToken(tokenPath) - if err != nil { - log.Error().Err(err).Msgf("failed to load access token") + if err != nil && verbose { + log.Warn().Err(err).Msgf("could not load access token") } } @@ -72,8 +72,7 @@ var collectCmd = &cobra.Command{ func init() { currentUser, _ = user.Current() - collectCmd.PersistentFlags().StringVar(&client.Host, "host", client.Host, "set the host to the SMD API") - collectCmd.PersistentFlags().IntVarP(&client.Port, "port", "p", client.Port, "set the port to the SMD API") + collectCmd.PersistentFlags().StringVar(&client.Host, "host", "", "set the host:port to the SMD API") collectCmd.PersistentFlags().StringVar(&username, "username", "", "set the BMC user") collectCmd.PersistentFlags().StringVar(&password, "password", "", "set the BMC password") collectCmd.PersistentFlags().StringVar(&scheme, "scheme", "https", "set the scheme used to query") diff --git a/cmd/list.go b/cmd/list.go index 26cc1fc..826f1e3 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -12,6 +12,10 @@ import ( "github.com/spf13/cobra" ) +var ( + showCache bool +) + // The `list` command provides an easy way to show what was found // and stored in a cache database from a scan. The data that's stored // is what is consumed by the `collect` command with the --cache flag. @@ -24,6 +28,13 @@ var listCmd = &cobra.Command{ " magellan list\n" + " magellan list --cache ./assets.db", Run: func(cmd *cobra.Command, args []string) { + // check if we just want to show cache-related info and exit + if showCache { + fmt.Printf("cache: %s\n", cachePath) + return + } + + // load the assets found from scan scannedResults, err := sqlite.GetScannedAssets(cachePath) if err != nil { log.Error().Err(err).Msg("failed to get scanned assets") @@ -37,7 +48,7 @@ var listCmd = &cobra.Command{ fmt.Printf("%s\n", string(b)) } else { for _, r := range scannedResults { - fmt.Printf("%s:%d (%s) @ %s\n", r.Host, r.Port, r.Protocol, r.Timestamp.Format(time.UnixDate)) + fmt.Printf("%s:%d (%s) @%s\n", r.Host, r.Port, r.Protocol, r.Timestamp.Format(time.UnixDate)) } } }, @@ -45,5 +56,6 @@ var listCmd = &cobra.Command{ func init() { listCmd.Flags().StringVar(&format, "format", "", "set the output format (json|default)") + listCmd.Flags().BoolVar(&showCache, "cache-info", false, "show cache information and exit") rootCmd.AddCommand(listCmd) } diff --git a/cmd/root.go b/cmd/root.go index 09a38c1..55365cd 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -22,6 +22,7 @@ import ( magellan "github.com/OpenCHAMI/magellan/internal" "github.com/OpenCHAMI/magellan/pkg/client" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -76,7 +77,7 @@ func init() { rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "set to enable/disable verbose output") rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "set to enable/disable debug messages") rootCmd.PersistentFlags().StringVar(&accessToken, "access-token", "", "set the access token") - rootCmd.PersistentFlags().StringVar(&cachePath, "cache", fmt.Sprintf("/tmp/%smagellan/magellan.db", currentUser.Username+"/"), "set the scanning result cache path") + rootCmd.PersistentFlags().StringVar(&cachePath, "cache", fmt.Sprintf("/tmp/%s/magellan/assets.db", currentUser.Username), "set the scanning result cache path") // bind viper config flags with cobra viper.BindPFlag("concurrency", rootCmd.Flags().Lookup("concurrency")) @@ -92,7 +93,10 @@ func init() { // See the 'LoadConfig' function in 'internal/config' for details. func InitializeConfig() { if configPath != "" { - magellan.LoadConfig(configPath) + err := magellan.LoadConfig(configPath) + if err != nil { + log.Error().Err(err).Msg("failed to load config") + } } } @@ -102,12 +106,13 @@ func InitializeConfig() { // TODO: This function should probably be moved to 'internal/config.go' // instead of in this file. func SetDefaults() { + currentUser, _ = user.Current() viper.SetDefault("threads", 1) viper.SetDefault("timeout", 5) viper.SetDefault("config", "") viper.SetDefault("verbose", false) viper.SetDefault("debug", false) - viper.SetDefault("cache", "/tmp/magellan/magellan.db") + viper.SetDefault("cache", fmt.Sprintf("/tmp/%s/magellan/magellan.db", currentUser.Username)) viper.SetDefault("scan.hosts", []string{}) viper.SetDefault("scan.ports", []int{}) viper.SetDefault("scan.subnets", []string{}) @@ -115,7 +120,6 @@ func SetDefaults() { viper.SetDefault("scan.disable-probing", false) viper.SetDefault("collect.driver", []string{"redfish"}) viper.SetDefault("collect.host", client.Host) - viper.SetDefault("collect.port", client.Port) viper.SetDefault("collect.user", "") viper.SetDefault("collect.pass", "") viper.SetDefault("collect.protocol", "tcp") From 8e59885f556800f335bdea54d95674c44a3b54ed Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 9 Aug 2024 07:58:42 -0600 Subject: [PATCH 33/67] Minor changes and improvements --- internal/cache/sqlite/sqlite.go | 10 ++++++++++ internal/collect.go | 9 +++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/internal/cache/sqlite/sqlite.go b/internal/cache/sqlite/sqlite.go index 7f0bbf5..74abe88 100644 --- a/internal/cache/sqlite/sqlite.go +++ b/internal/cache/sqlite/sqlite.go @@ -4,6 +4,7 @@ import ( "fmt" magellan "github.com/OpenCHAMI/magellan/internal" + "github.com/OpenCHAMI/magellan/internal/util" "github.com/jmoiron/sqlx" ) @@ -83,6 +84,15 @@ func DeleteScannedAssets(path string, results ...magellan.ScannedAsset) error { } func GetScannedAssets(path string) ([]magellan.ScannedAsset, error) { + // check if path exists first to prevent creating the database + exists, err := util.PathExists(path) + if !exists { + return nil, fmt.Errorf("no file found") + } else if err != nil { + return nil, err + } + + // now check if the file is the SQLite database db, err := sqlx.Open("sqlite3", path) if err != nil { return nil, fmt.Errorf("failed to open database: %v", err) diff --git a/internal/collect.go b/internal/collect.go index 88cde9c..8809591 100644 --- a/internal/collect.go +++ b/internal/collect.go @@ -65,8 +65,8 @@ func CollectInventory(scannedResults *[]ScannedAsset, params *CollectParams) err done = make(chan struct{}, params.Concurrency+1) chanScannedResult = make(chan ScannedAsset, params.Concurrency+1) outputPath = path.Clean(params.OutputPath) - smdClient = client.NewClient( - client.WithSecureTLS(params.CaCertPath), + smdClient = client.NewClient[client.SmdClient]( + client.WithSecureTLS[client.SmdClient](params.CaCertPath), ) ) wg.Add(params.Concurrency) @@ -152,13 +152,14 @@ func CollectInventory(scannedResults *[]ScannedAsset, params *CollectParams) err } // add all endpoints to smd - err = smdClient.AddRedfishEndpoint(data, headers) + err = smdClient.Add(body, headers) if err != nil { log.Error().Err(err).Msgf("failed to add Redfish endpoint") // try updating instead if params.ForceUpdate { - err = smdClient.UpdateRedfishEndpoint(data["ID"].(string), body, headers) + smdClient.Xname = data["ID"].(string) + err = smdClient.Update(body, headers) if err != nil { log.Error().Err(err).Msgf("failed to update Redfish endpoint") } From c5a348562b105b65343060e3cfd01f82c402e065 Mon Sep 17 00:00:00 2001 From: David Allen Date: Fri, 9 Aug 2024 07:59:28 -0600 Subject: [PATCH 34/67] Refactored how clients work to reduce hard-coded dependencies --- pkg/client/client.go | 48 +++++++++++++++++-------------------- pkg/client/default.go | 55 +++++++++++++++++++++++++++++++++++++++++++ pkg/client/smd.go | 54 +++++++++++++++++++----------------------- 3 files changed, 100 insertions(+), 57 deletions(-) create mode 100644 pkg/client/default.go diff --git a/pkg/client/client.go b/pkg/client/client.go index 229471d..739332a 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -13,38 +13,36 @@ import ( "github.com/OpenCHAMI/magellan/internal/util" ) -type Option func(*Client) +type Option[T Client] func(client T) // The 'Client' struct is a wrapper around the default http.Client // that provides an extended API to work with functional options. // It also provides functions that work with `collect` data. -type Client struct { - *http.Client +type Client interface { + Name() string + GetClient() *http.Client + RootEndpoint(endpoint string) string + + // functions needed to make request + Add(data util.HTTPBody, headers util.HTTPHeader) error + Update(data util.HTTPBody, headers util.HTTPHeader) error } // NewClient() creates a new client -func NewClient(opts ...Option) *Client { - client := &Client{ - Client: http.DefaultClient, - } +func NewClient[T Client](opts ...func(T)) T { + client := new(T) for _, opt := range opts { - opt(client) + opt(*client) } - return client + return *client } -func WithHttpClient(httpClient *http.Client) Option { - return func(c *Client) { - c.Client = httpClient - } -} - -func WithCertPool(certPool *x509.CertPool) Option { +func WithCertPool[T Client](certPool *x509.CertPool) func(T) { if certPool == nil { - return func(c *Client) {} + return func(client T) {} } - return func(c *Client) { - c.Client.Transport = &http.Transport{ + return func(client T) { + client.GetClient().Transport = &http.Transport{ TLSClientConfig: &tls.Config{ RootCAs: certPool, InsecureSkipVerify: true, @@ -60,20 +58,20 @@ func WithCertPool(certPool *x509.CertPool) Option { } } -func WithSecureTLS(certPath string) Option { +func WithSecureTLS[T Client](certPath string) func(T) { cacert, err := os.ReadFile(certPath) if err != nil { - return func(c *Client) {} + return func(client T) {} } certPool := x509.NewCertPool() certPool.AppendCertsFromPEM(cacert) - return WithCertPool(certPool) + return WithCertPool[T](certPool) } // Post() is a simplified wrapper function that packages all of the // that marshals a mapper into a JSON-formatted byte array, and then performs // a request to the specified URL. -func (c *Client) Post(url string, data map[string]any, header util.HTTPHeader) (*http.Response, util.HTTPBody, error) { +func (c *MagellanClient) Post(url string, data map[string]any, header util.HTTPHeader) (*http.Response, util.HTTPBody, error) { // serialize data into byte array body, err := json.Marshal(data) if err != nil { @@ -81,7 +79,3 @@ func (c *Client) Post(url string, data map[string]any, header util.HTTPHeader) ( } return util.MakeRequest(c.Client, url, http.MethodPost, body, header) } - -func (c *Client) MakeRequest(url string, method string, body util.HTTPBody, header util.HTTPHeader) (*http.Response, util.HTTPBody, error) { - return util.MakeRequest(c.Client, url, method, body, header) -} diff --git a/pkg/client/default.go b/pkg/client/default.go new file mode 100644 index 0000000..e466125 --- /dev/null +++ b/pkg/client/default.go @@ -0,0 +1,55 @@ +package client + +import ( + "fmt" + "net/http" + + "github.com/OpenCHAMI/magellan/internal/util" +) + +type MagellanClient struct { + *http.Client +} + +func (c *MagellanClient) Name() string { + return "default" +} + +// Add() is the default function that is called with a client with no implementation. +// This function will simply make a HTTP request including all the data passed as +// the first argument with no data processing or manipulation. The function sends +// the data to a set callback URL (which may be changed to use a configurable value +// instead). +func (c *MagellanClient) Add(data util.HTTPBody, headers util.HTTPHeader) error { + if data == nil { + return fmt.Errorf("no data found") + } + + path := "/inventory/add" + res, body, err := util.MakeRequest(c.Client, path, http.MethodPost, data, headers) + if res != nil { + statusOk := res.StatusCode >= 200 && res.StatusCode < 300 + if !statusOk { + return fmt.Errorf("returned status code %d when POST'ing to endpoint", res.StatusCode) + } + fmt.Printf("%v (%v)\n%s\n", path, res.Status, string(body)) + } + return err +} + +func (c *MagellanClient) Update(data util.HTTPBody, headers util.HTTPHeader) error { + if data == nil { + return fmt.Errorf("no data found") + } + + path := "/inventory/update" + res, body, err := util.MakeRequest(c.Client, path, http.MethodPut, data, headers) + if res != nil { + statusOk := res.StatusCode >= 200 && res.StatusCode < 300 + if !statusOk { + return fmt.Errorf("returned status code %d when PUT'ing to endpoint", res.StatusCode) + } + fmt.Printf("%v (%v)\n%s\n", path, res.Status, string(body)) + } + return err +} diff --git a/pkg/client/smd.go b/pkg/client/smd.go index 726c5f6..2befba0 100644 --- a/pkg/client/smd.go +++ b/pkg/client/smd.go @@ -11,41 +11,39 @@ import ( ) var ( - Host = "http://localhost" + Host = "http://localhost:27779" BaseEndpoint = "/hsm/v2" - Port = 27779 ) -func (c *Client) GetRedfishEndpoints(header util.HTTPHeader) error { - url := makeEndpointUrl("/Inventory/RedfishEndpoints") - _, body, err := util.MakeRequest(c.Client, url, http.MethodGet, nil, header) - if err != nil { - return fmt.Errorf("failed to get endpoint: %v", err) - } - // fmt.Println(res) - fmt.Println(string(body)) - return nil +type SmdClient struct { + *http.Client + Host string + Xname string } -func (c *Client) GetComponentEndpoint(xname string) error { - url := makeEndpointUrl("/Inventory/ComponentsEndpoints/" + xname) - res, body, err := c.MakeRequest(url, "GET", nil, nil) - if err != nil { - return fmt.Errorf("failed to get endpoint: %v", err) - } - fmt.Println(res) - fmt.Println(string(body)) - return nil +func (c SmdClient) Name() string { + return "smd" +} + +func (c SmdClient) RootEndpoint(endpoint string) string { + return fmt.Sprintf("/hsm/v2/%s%s", Host, endpoint) } -func (c *Client) AddRedfishEndpoint(data map[string]any, headers util.HTTPHeader) error { +func (c SmdClient) GetClient() *http.Client { + return c.Client +} + +// Add() has a similar function definition to that of the default implementation, +// but also allows further customization and data/header manipulation that would +// be specific and/or unique to SMD's API. +func (c SmdClient) Add(data util.HTTPBody, headers util.HTTPHeader) error { if data == nil { return fmt.Errorf("failed to add redfish endpoint: no data found") } // Add redfish endpoint via POST `/hsm/v2/Inventory/RedfishEndpoints` endpoint - url := makeEndpointUrl("/Inventory/RedfishEndpoints") - res, body, err := c.Post(url, data, headers) + url := c.RootEndpoint("/Inventory/RedfishEndpoints") + res, body, err := util.MakeRequest(c.Client, url, http.MethodPost, data, headers) if res != nil { statusOk := res.StatusCode >= 200 && res.StatusCode < 300 if !statusOk { @@ -56,13 +54,13 @@ func (c *Client) AddRedfishEndpoint(data map[string]any, headers util.HTTPHeader return err } -func (c *Client) UpdateRedfishEndpoint(xname string, data []byte, headers map[string]string) error { +func (c SmdClient) Update(data util.HTTPBody, headers util.HTTPHeader) error { if data == nil { return fmt.Errorf("failed to add redfish endpoint: no data found") } // Update redfish endpoint via PUT `/hsm/v2/Inventory/RedfishEndpoints` endpoint - url := makeEndpointUrl("/Inventory/RedfishEndpoints/" + xname) - res, body, err := c.MakeRequest(url, "PUT", data, headers) + url := c.RootEndpoint("/Inventory/RedfishEndpoints/" + c.Xname) + res, body, err := util.MakeRequest(c.Client, url, http.MethodPut, data, headers) fmt.Printf("%v (%v)\n%s\n", url, res.Status, string(body)) if res != nil { statusOk := res.StatusCode >= 200 && res.StatusCode < 300 @@ -72,7 +70,3 @@ func (c *Client) UpdateRedfishEndpoint(xname string, data []byte, headers map[st } return err } - -func makeEndpointUrl(endpoint string) string { - return Host + ":" + fmt.Sprint(Port) + BaseEndpoint + endpoint -} From 24fba89a989a66f20f55a299c3cd0d2029f57961 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 9 Aug 2024 17:11:37 -0600 Subject: [PATCH 35/67] Reformatted scan help message --- cmd/scan.go | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/cmd/scan.go b/cmd/scan.go index aa5d8ab..acd29b6 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -35,17 +35,17 @@ var ( var scanCmd = &cobra.Command{ Use: "scan urls...", Short: "Scan to discover BMC nodes on a network", - Long: "Perform a net scan by attempting to connect to each host and port specified and getting a response. " + - "Each host is passed *with a full URL* including the protocol and port. Additional subnets can be added " + - "by using the '--subnet' flag and providing an IP address on the subnet as well as a CIDR. If no CIDR is " + - "provided, then the subnet mask specified with the '--subnet-mask' flag will be used instead (will use " + - "default mask if not set).\n" + - "Similarly, any host provided with no port with use either the ports specified" + - "with `--port` or the default port used with each specified protocol. The default protocol is 'tcp' unless " + - "specified. The `--scheme` flag works similarly and the default value is 'https' in the host URL or with the " + - "'--protocol' flag.\n" + - "If the '--disable-probe` flag is used, the tool will not send another request to probe for available. " + - "Redfish services. This is not recommended, since the extra request makes the scan a bit more reliable " + + Long: "Perform a net scan by attempting to connect to each host and port specified and getting a response.\n" + + "Each host is passed *with a full URL* including the protocol and port. Additional subnets can be added\n" + + "by using the '--subnet' flag and providing an IP address on the subnet as well as a CIDR. If no CIDR is\n" + + "provided, then the subnet mask specified with the '--subnet-mask' flag will be used instead (will use\n" + + "default mask if not set).\n\n" + + "Similarly, any host provided with no port with use either the ports specified\n" + + "with `--port` or the default port used with each specified protocol. The default protocol is 'tcp' unless\n" + + "specified. The `--scheme` flag works similarly and the default value is 'https' in the host URL or with the\n" + + "'--protocol' flag.\n\n" + + "If the '--disable-probe` flag is used, the tool will not send another request to probe for available.\n" + + "Redfish services. This is not recommended, since the extra request makes the scan a bit more reliable\n" + "for determining which hosts to collect inventory data.\n\n" + "Examples:\n" + // assumes host https://10.0.0.101:443 @@ -53,15 +53,15 @@ var scanCmd = &cobra.Command{ // assumes subnet using HTTPS and port 443 except for specified host " magellan scan http://10.0.0.101:80 https://user:password@10.0.0.102:443 http://172.16.0.105:8080 --subnet 172.16.0.0/24\n" + // assumes hosts http://10.0.0.101:8080 and http://10.0.0.102:8080 - " magellan scan 10.0.0.101 10.0.0.102 https://172.16.0.10:443 --port 8080 --protocol tcp" + + " magellan scan 10.0.0.101 10.0.0.102 https://172.16.0.10:443 --port 8080 --protocol tcp\n" + // assumes subnet using default unspecified subnet-masks - " magellan scan --subnet 10.0.0.0" + + " magellan scan --subnet 10.0.0.0\n" + // assumes subnet using HTTPS and port 443 with specified CIDR - " magellan scan --subnet 10.0.0.0/16" + + " magellan scan --subnet 10.0.0.0/16\n" + // assumes subnet using HTTP and port 5000 similar to 192.168.0.0/16 - " magellan scan --subnet 192.168.0.0 --protocol tcp --scheme https --port 5000 --subnet-mask 255.255.0.0" + + " magellan scan --subnet 192.168.0.0 --protocol tcp --scheme https --port 5000 --subnet-mask 255.255.0.0\n" + // assumes subnet without CIDR has a subnet-mask of 255.255.0.0 - " magellan scan --subnet 10.0.0.0/24 --subnet 172.16.0.0 --subnet-mask 255.255.0.0 --cache ./assets.db", + " magellan scan --subnet 10.0.0.0/24 --subnet 172.16.0.0 --subnet-mask 255.255.0.0 --cache ./assets.db\n", Run: func(cmd *cobra.Command, args []string) { // add default ports for hosts if none are specified with flag if len(ports) == 0 { @@ -181,8 +181,8 @@ func init() { scanCmd.Flags().StringVar(&protocol, "protocol", "tcp", "Set the default protocol to use in scan. (default is 'tcp')") scanCmd.Flags().StringSliceVar(&subnets, "subnet", nil, "Add additional hosts from specified subnets to scan.") scanCmd.Flags().IPMaskVar(&subnetMask, "subnet-mask", net.IPv4Mask(255, 255, 255, 0), "Set the default subnet mask to use for with all subnets not using CIDR notation.") - scanCmd.Flags().BoolVar(&disableProbing, "disable-probing", false, "disable probing found assets for Redfish service(s) running on BMC nodes") - scanCmd.Flags().BoolVar(&disableCache, "disable-cache", false, "disable saving found assets to a cache database specified with 'cache' flag") + scanCmd.Flags().BoolVar(&disableProbing, "disable-probing", false, "Disable probing found assets for Redfish service(s) running on BMC nodes") + scanCmd.Flags().BoolVar(&disableCache, "disable-cache", false, "Disable saving found assets to a cache database specified with 'cache' flag") viper.BindPFlag("scan.hosts", scanCmd.Flags().Lookup("host")) viper.BindPFlag("scan.ports", scanCmd.Flags().Lookup("port")) From fc6afc8559f4ec4d856a6be8c96f95a7895ad133 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 9 Aug 2024 17:35:24 -0600 Subject: [PATCH 36/67] Minor change --- cmd/scan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/scan.go b/cmd/scan.go index acd29b6..d246149 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -113,7 +113,7 @@ var scanCmd = &cobra.Command{ "concurrency": concurrency, "protocol": protocol, "subnets": subnets, - "subnet-masks": subnetMask, + "subnet-mask": subnetMask.String(), "cert": cacertPath, "disable-probing": disableProbing, "disable-caching": disableCache, From 7beb7a33fc6c47c1a6c60a4311e4a2cd9b931d77 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 9 Aug 2024 17:39:14 -0600 Subject: [PATCH 37/67] Renamed struct --- internal/cache/sqlite/sqlite.go | 8 ++++---- internal/collect.go | 4 ++-- internal/scan.go | 14 +++++++------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/internal/cache/sqlite/sqlite.go b/internal/cache/sqlite/sqlite.go index 74abe88..7a04978 100644 --- a/internal/cache/sqlite/sqlite.go +++ b/internal/cache/sqlite/sqlite.go @@ -31,7 +31,7 @@ func CreateScannedAssetIfNotExists(path string) (*sqlx.DB, error) { return db, nil } -func InsertScannedAssets(path string, assets ...magellan.ScannedAsset) error { +func InsertScannedAssets(path string, assets ...magellan.RemoteAsset) error { if assets == nil { return fmt.Errorf("states == nil") } @@ -59,7 +59,7 @@ func InsertScannedAssets(path string, assets ...magellan.ScannedAsset) error { return nil } -func DeleteScannedAssets(path string, results ...magellan.ScannedAsset) error { +func DeleteScannedAssets(path string, results ...magellan.RemoteAsset) error { if results == nil { return fmt.Errorf("no assets found") } @@ -83,7 +83,7 @@ func DeleteScannedAssets(path string, results ...magellan.ScannedAsset) error { return nil } -func GetScannedAssets(path string) ([]magellan.ScannedAsset, error) { +func GetScannedAssets(path string) ([]magellan.RemoteAsset, error) { // check if path exists first to prevent creating the database exists, err := util.PathExists(path) if !exists { @@ -98,7 +98,7 @@ func GetScannedAssets(path string) ([]magellan.ScannedAsset, error) { return nil, fmt.Errorf("failed to open database: %v", err) } - results := []magellan.ScannedAsset{} + results := []magellan.RemoteAsset{} err = db.Select(&results, fmt.Sprintf("SELECT * FROM %s ORDER BY host ASC, port ASC;", TABLE_NAME)) if err != nil { return nil, fmt.Errorf("failed to retrieve assets: %v", err) diff --git a/internal/collect.go b/internal/collect.go index 8809591..6f01199 100644 --- a/internal/collect.go +++ b/internal/collect.go @@ -48,7 +48,7 @@ type CollectParams struct { // // Requests can be made to several of the nodes using a goroutine by setting the q.Concurrency // property value between 1 and 255. -func CollectInventory(scannedResults *[]ScannedAsset, params *CollectParams) error { +func CollectInventory(scannedResults *[]RemoteAsset, params *CollectParams) error { // check for available probe states if scannedResults == nil { return fmt.Errorf("no probe states found") @@ -63,7 +63,7 @@ func CollectInventory(scannedResults *[]ScannedAsset, params *CollectParams) err wg sync.WaitGroup found = make([]string, 0, len(*scannedResults)) done = make(chan struct{}, params.Concurrency+1) - chanScannedResult = make(chan ScannedAsset, params.Concurrency+1) + chanScannedResult = make(chan RemoteAsset, params.Concurrency+1) outputPath = path.Clean(params.OutputPath) smdClient = client.NewClient[client.SmdClient]( client.WithSecureTLS[client.SmdClient](params.CaCertPath), diff --git a/internal/scan.go b/internal/scan.go index 0accae9..0eed0d7 100644 --- a/internal/scan.go +++ b/internal/scan.go @@ -14,7 +14,7 @@ import ( "github.com/rs/zerolog/log" ) -type ScannedAsset struct { +type RemoteAsset struct { Host string `json:"host"` Port int `json:"port"` Protocol string `json:"protocol"` @@ -49,9 +49,9 @@ type ScanParams struct { // remove the service from being stored in the list of scanned results. // // Returns a list of scanned results to be stored in cache (but isn't doing here). -func ScanForAssets(params *ScanParams) []ScannedAsset { +func ScanForAssets(params *ScanParams) []RemoteAsset { var ( - results = make([]ScannedAsset, 0, len(params.TargetHosts)) + results = make([]RemoteAsset, 0, len(params.TargetHosts)) done = make(chan struct{}, params.Concurrency+1) chanHosts = make(chan []string, params.Concurrency+1) ) @@ -81,7 +81,7 @@ func ScanForAssets(params *ScanParams) []ScannedAsset { return } if !params.DisableProbing { - assetsToAdd := []ScannedAsset{} + assetsToAdd := []RemoteAsset{} for _, foundAsset := range foundAssets { url := fmt.Sprintf("%s://%s/redfish/v1/", params.Scheme, foundAsset.Host) res, _, err := util.MakeRequest(nil, url, http.MethodGet, nil, nil) @@ -177,7 +177,7 @@ func GetDefaultPorts() []int { // until a response is receive or if the timeout (in seconds) expires. This // function expects a full URL such as https://my.bmc.host:443/ to make the // connection. -func rawConnect(address string, protocol string, timeoutSeconds int, keepOpenOnly bool) ([]ScannedAsset, error) { +func rawConnect(address string, protocol string, timeoutSeconds int, keepOpenOnly bool) ([]RemoteAsset, error) { uri, err := url.ParseRequestURI(address) if err != nil { return nil, fmt.Errorf("failed to split host/port: %w", err) @@ -191,8 +191,8 @@ func rawConnect(address string, protocol string, timeoutSeconds int, keepOpenOnl var ( timeoutDuration = time.Second * time.Duration(timeoutSeconds) - assets []ScannedAsset - asset = ScannedAsset{ + assets []RemoteAsset + asset = RemoteAsset{ Host: uri.Hostname(), Port: port, Protocol: protocol, From bc01412b080529c2d31ccfb4549d01bc82107103 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Fri, 9 Aug 2024 17:51:23 -0600 Subject: [PATCH 38/67] Fixed port not being added to probing request --- internal/scan.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/scan.go b/internal/scan.go index 0eed0d7..3ac6129 100644 --- a/internal/scan.go +++ b/internal/scan.go @@ -57,7 +57,7 @@ func ScanForAssets(params *ScanParams) []RemoteAsset { ) if params.Verbose { - log.Info().Msg("starting scan...") + log.Info().Any("args", params).Msg("starting scan...") } var wg sync.WaitGroup @@ -83,7 +83,7 @@ func ScanForAssets(params *ScanParams) []RemoteAsset { if !params.DisableProbing { assetsToAdd := []RemoteAsset{} for _, foundAsset := range foundAssets { - url := fmt.Sprintf("%s://%s/redfish/v1/", params.Scheme, foundAsset.Host) + url := fmt.Sprintf("%s://%s:%d/redfish/v1/", params.Scheme, foundAsset.Host, foundAsset.Port) res, _, err := util.MakeRequest(nil, url, http.MethodGet, nil, nil) if err != nil || res == nil { if params.Verbose { @@ -202,7 +202,7 @@ func rawConnect(address string, protocol string, timeoutSeconds int, keepOpenOnl ) // try to conntect to host (expects host in format [10.0.0.0]:443) - target := fmt.Sprintf("[%s]:%s", uri.Hostname(), uri.Port()) + target := fmt.Sprintf("%s:%s", uri.Hostname(), uri.Port()) conn, err := net.DialTimeout(protocol, target, timeoutDuration) if err != nil { asset.State = false From 51e24e2eddd343f8dd109e4b6fe0bfff3c724b76 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Sun, 11 Aug 2024 14:21:56 -0600 Subject: [PATCH 39/67] Changed how arguments are passed to update command --- cmd/update.go | 68 ++++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/cmd/update.go b/cmd/update.go index 67bba16..82e3106 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -1,6 +1,9 @@ package cmd import ( + "os" + "strings" + magellan "github.com/OpenCHAMI/magellan/internal" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -14,76 +17,75 @@ var ( firmwareVersion string component string transferProtocol string - status bool + showStatus bool ) // The `update` command provides an interface to easily update firmware // using Redfish. It also provides a simple way to check the status of // an update in-progress. var updateCmd = &cobra.Command{ - Use: "update", + Use: "update hosts...", Short: "Update BMC node firmware", Long: "Perform an firmware update using Redfish by providing a remote firmware URL and component.\n" + "Examples:\n" + " magellan update --bmc.host 172.16.0.108 --bmc.port 443 --username bmc_username --password bmc_password --firmware-url http://172.16.0.200:8005/firmware/bios/image.RBU --component BIOS\n" + " magellan update --status --bmc.host 172.16.0.108 --bmc.port 443 --username bmc_username --password bmc_password", Run: func(cmd *cobra.Command, args []string) { - // check if required params are set - if host == "" || username == "" || password == "" { - log.Error().Msg("requires host, user, and pass to be set") + // check that we have at least one host + if len(args) <= 0 { + log.Error().Msg("update requires at least one host") + os.Exit(1) } // get status if flag is set and exit - if status { - err := magellan.GetUpdateStatus(&magellan.UpdateParams{ + for _, arg := range args { + if showStatus { + err := magellan.GetUpdateStatus(&magellan.UpdateParams{ + FirmwarePath: firmwareUrl, + FirmwareVersion: firmwareVersion, + Component: component, + TransferProtocol: transferProtocol, + CollectParams: magellan.CollectParams{ + URI: arg, + Username: username, + Password: password, + Timeout: timeout, + }, + }) + if err != nil { + log.Error().Err(err).Msgf("failed to get update status") + } + return + } + + // initiate a remote update + err := magellan.UpdateFirmwareRemote(&magellan.UpdateParams{ FirmwarePath: firmwareUrl, FirmwareVersion: firmwareVersion, Component: component, - TransferProtocol: transferProtocol, + TransferProtocol: strings.ToUpper(transferProtocol), CollectParams: magellan.CollectParams{ - Host: host, + URI: host, Username: username, Password: password, Timeout: timeout, - Port: port, }, }) if err != nil { - log.Error().Err(err).Msgf("failed to get update status") + log.Error().Err(err).Msgf("failed to update firmware") } - return - } - - // initiate a remote update - err := magellan.UpdateFirmwareRemote(&magellan.UpdateParams{ - FirmwarePath: firmwareUrl, - FirmwareVersion: firmwareVersion, - Component: component, - TransferProtocol: transferProtocol, - CollectParams: magellan.CollectParams{ - Host: host, - Username: username, - Password: password, - Timeout: timeout, - Port: port, - }, - }) - if err != nil { - log.Error().Err(err).Msgf("failed to update firmware") } }, } func init() { - updateCmd.Flags().StringVar(&host, "bmc.host", "", "set the BMC host") - updateCmd.Flags().IntVar(&port, "bmc.port", 443, "set the BMC port") updateCmd.Flags().StringVar(&username, "username", "", "set the BMC user") updateCmd.Flags().StringVar(&password, "password", "", "set the BMC password") updateCmd.Flags().StringVar(&transferProtocol, "transfer-protocol", "HTTP", "set the transfer protocol") updateCmd.Flags().StringVar(&firmwareUrl, "firmware.url", "", "set the path to the firmware") updateCmd.Flags().StringVar(&firmwareVersion, "firmware.version", "", "set the version of firmware to be installed") updateCmd.Flags().StringVar(&component, "component", "", "set the component to upgrade") - updateCmd.Flags().BoolVar(&status, "status", false, "get the status of the update") + updateCmd.Flags().BoolVar(&showStatus, "status", false, "get the status of the update") viper.BindPFlag("update.bmc.host", updateCmd.Flags().Lookup("bmc.host")) viper.BindPFlag("update.bmc.port", updateCmd.Flags().Lookup("bmc.port")) From d930c4136fd6acc0f07ef61a7be49941b12bb8b3 Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Sun, 11 Aug 2024 14:23:19 -0600 Subject: [PATCH 40/67] Changed how based URL is derived in update functions --- internal/update.go | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/internal/update.go b/internal/update.go index 8fca958..b71002b 100644 --- a/internal/update.go +++ b/internal/update.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" "github.com/OpenCHAMI/magellan/internal/util" ) @@ -20,7 +21,15 @@ type UpdateParams struct { // The function expects the firmware URL, firmware version, and component flags to be // set from the CLI to perform a firmware update. func UpdateFirmwareRemote(q *UpdateParams) error { - url := baseRedfishUrl(&q.CollectParams) + "/redfish/v1/UpdateService/Actions/SimpleUpdate" + // parse URI to set up full address + uri, err := url.ParseRequestURI(q.URI) + if err != nil { + return fmt.Errorf("failed to parse URI: %w", err) + } + uri.User = url.UserPassword(q.Username, q.Password) + + // set up other vars + updateUrl := fmt.Sprintf("%s/redfish/v1/UpdateService/Actions/SimpleUpdate", uri.String()) headers := map[string]string{ "Content-Type": "application/json", "cache-control": "no-cache", @@ -34,11 +43,11 @@ func UpdateFirmwareRemote(q *UpdateParams) error { if err != nil { return fmt.Errorf("failed to marshal data: %v", err) } - res, body, err := util.MakeRequest(nil, url, "POST", data, headers) + res, body, err := util.MakeRequest(nil, updateUrl, "POST", data, headers) if err != nil { return fmt.Errorf("something went wrong: %v", err) } else if res == nil { - return fmt.Errorf("no response returned (url: %s)", url) + return fmt.Errorf("no response returned (url: %s)", updateUrl) } if len(body) > 0 { fmt.Printf("%d: %v\n", res.StatusCode, string(body)) @@ -47,12 +56,18 @@ func UpdateFirmwareRemote(q *UpdateParams) error { } func GetUpdateStatus(q *UpdateParams) error { - url := baseRedfishUrl(&q.CollectParams) + "/redfish/v1/UpdateService" - res, body, err := util.MakeRequest(nil, url, "GET", nil, nil) + // parse URI to set up full address + uri, err := url.ParseRequestURI(q.URI) + if err != nil { + return fmt.Errorf("failed to parse URI: %w", err) + } + uri.User = url.UserPassword(q.Username, q.Password) + updateUrl := fmt.Sprintf("%s/redfish/v1/UpdateService", uri.String()) + res, body, err := util.MakeRequest(nil, updateUrl, "GET", nil, nil) if err != nil { return fmt.Errorf("something went wrong: %v", err) } else if res == nil { - return fmt.Errorf("no response returned (url: %s)", url) + return fmt.Errorf("no response returned (url: %s)", updateUrl) } else if res.StatusCode != http.StatusOK { return fmt.Errorf("returned status code %d", res.StatusCode) } From a6c95ef646109365b219efeaf7003e768615daec Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Sun, 11 Aug 2024 14:24:08 -0600 Subject: [PATCH 41/67] Removed unused variables in client package --- pkg/client/smd.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pkg/client/smd.go b/pkg/client/smd.go index 2befba0..ca3865e 100644 --- a/pkg/client/smd.go +++ b/pkg/client/smd.go @@ -10,14 +10,9 @@ import ( "github.com/OpenCHAMI/magellan/internal/util" ) -var ( - Host = "http://localhost:27779" - BaseEndpoint = "/hsm/v2" -) - type SmdClient struct { *http.Client - Host string + URI string Xname string } @@ -26,7 +21,7 @@ func (c SmdClient) Name() string { } func (c SmdClient) RootEndpoint(endpoint string) string { - return fmt.Sprintf("/hsm/v2/%s%s", Host, endpoint) + return fmt.Sprintf("%s/hsm/v2%s", c.URI, endpoint) } func (c SmdClient) GetClient() *http.Client { From f7159c9b661796feb6741c2b62e839f0a42dd0dd Mon Sep 17 00:00:00 2001 From: "David J. Allen" Date: Sun, 11 Aug 2024 14:25:21 -0600 Subject: [PATCH 42/67] Fixed issue with collect requests and other minor changes --- cmd/collect.go | 18 +++++------ cmd/root.go | 5 ++- config.yaml | 2 +- internal/collect.go | 77 +++++++++++++++++++++------------------------ internal/scan.go | 2 +- 5 files changed, 49 insertions(+), 55 deletions(-) diff --git a/cmd/collect.go b/cmd/collect.go index af4df5e..26ed293 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -7,7 +7,6 @@ import ( magellan "github.com/OpenCHAMI/magellan/internal" "github.com/OpenCHAMI/magellan/internal/cache/sqlite" "github.com/OpenCHAMI/magellan/internal/util" - "github.com/OpenCHAMI/magellan/pkg/client" "github.com/cznic/mathutil" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -51,9 +50,10 @@ var collectCmd = &cobra.Command{ // if concurrency <= 0 { - concurrency = mathutil.Clamp(len(scannedResults), 1, 255) + concurrency = mathutil.Clamp(len(scannedResults), 1, 10000) } err = magellan.CollectInventory(&scannedResults, &magellan.CollectParams{ + URI: host, Username: username, Password: password, Timeout: timeout, @@ -72,14 +72,14 @@ var collectCmd = &cobra.Command{ func init() { currentUser, _ = user.Current() - collectCmd.PersistentFlags().StringVar(&client.Host, "host", "", "set the host:port to the SMD API") - collectCmd.PersistentFlags().StringVar(&username, "username", "", "set the BMC user") - collectCmd.PersistentFlags().StringVar(&password, "password", "", "set the BMC password") - collectCmd.PersistentFlags().StringVar(&scheme, "scheme", "https", "set the scheme used to query") - collectCmd.PersistentFlags().StringVar(&protocol, "protocol", "tcp", "set the protocol used to query") + collectCmd.PersistentFlags().StringVar(&host, "host", "", "Set the URI to the SMD API") + collectCmd.PersistentFlags().StringVar(&username, "username", "", "Set the BMC user") + collectCmd.PersistentFlags().StringVar(&password, "password", "", "Set the BMC password") + collectCmd.PersistentFlags().StringVar(&scheme, "scheme", "https", "Set the scheme used to query") + collectCmd.PersistentFlags().StringVar(&protocol, "protocol", "tcp", "Set the protocol used to query") collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/inventory/", currentUser.Username+"/"), "set the path to store collection data") - collectCmd.PersistentFlags().BoolVar(&forceUpdate, "force-update", false, "set flag to force update data sent to SMD") - collectCmd.PersistentFlags().StringVar(&cacertPath, "cacert", "", "path to CA cert. (defaults to system CAs)") + collectCmd.PersistentFlags().BoolVar(&forceUpdate, "force-update", false, "Set flag to force update data sent to SMD") + collectCmd.PersistentFlags().StringVar(&cacertPath, "cacert", "", "Path to CA cert. (defaults to system CAs)") // set flags to only be used together collectCmd.MarkFlagsRequiredTogether("username", "password") diff --git a/cmd/root.go b/cmd/root.go index 55365cd..a761f7d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -21,7 +21,6 @@ import ( "os/user" magellan "github.com/OpenCHAMI/magellan/internal" - "github.com/OpenCHAMI/magellan/pkg/client" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -112,14 +111,14 @@ func SetDefaults() { viper.SetDefault("config", "") viper.SetDefault("verbose", false) viper.SetDefault("debug", false) - viper.SetDefault("cache", fmt.Sprintf("/tmp/%s/magellan/magellan.db", currentUser.Username)) + viper.SetDefault("cache", fmt.Sprintf("/tmp/%s/magellan/assets.db", currentUser.Username)) viper.SetDefault("scan.hosts", []string{}) viper.SetDefault("scan.ports", []int{}) viper.SetDefault("scan.subnets", []string{}) viper.SetDefault("scan.subnet-masks", []net.IP{}) viper.SetDefault("scan.disable-probing", false) viper.SetDefault("collect.driver", []string{"redfish"}) - viper.SetDefault("collect.host", client.Host) + viper.SetDefault("collect.host", host) viper.SetDefault("collect.user", "") viper.SetDefault("collect.pass", "") viper.SetDefault("collect.protocol", "tcp") diff --git a/config.yaml b/config.yaml index 11b0c28..7d817ed 100644 --- a/config.yaml +++ b/config.yaml @@ -26,7 +26,7 @@ update: port: 443 username: "admin" password: "password" - transfer-protocol: "HTTPS" + transfer-protocol: "https" firmware: url: version: diff --git a/internal/collect.go b/internal/collect.go index 6f01199..79f873e 100644 --- a/internal/collect.go +++ b/internal/collect.go @@ -21,17 +21,10 @@ import ( "golang.org/x/exp/slices" ) -const ( - IPMI_PORT = 623 - SSH_PORT = 22 - HTTPS_PORT = 443 -) - // CollectParams is a collection of common parameters passed to the CLI // for the 'collect' subcommand. type CollectParams struct { - Host string // set by the 'host' flag - Port int // set by the 'port' flag + URI string // set by the 'host' flag Username string // set the BMC username with the 'username' flag Password string // set the BMC password with the 'password' flag Concurrency int // set the of concurrent jobs with the 'concurrency' flag @@ -48,38 +41,38 @@ type CollectParams struct { // // Requests can be made to several of the nodes using a goroutine by setting the q.Concurrency // property value between 1 and 255. -func CollectInventory(scannedResults *[]RemoteAsset, params *CollectParams) error { +func CollectInventory(assets *[]RemoteAsset, params *CollectParams) error { // check for available probe states - if scannedResults == nil { - return fmt.Errorf("no probe states found") + if assets == nil { + return fmt.Errorf("no assets found") } - if len(*scannedResults) <= 0 { - return fmt.Errorf("no probe states found") + if len(*assets) <= 0 { + return fmt.Errorf("no assets found") } // collect bmc information asynchronously var ( - offset = 0 - wg sync.WaitGroup - found = make([]string, 0, len(*scannedResults)) - done = make(chan struct{}, params.Concurrency+1) - chanScannedResult = make(chan RemoteAsset, params.Concurrency+1) - outputPath = path.Clean(params.OutputPath) - smdClient = client.NewClient[client.SmdClient]( + offset = 0 + wg sync.WaitGroup + found = make([]string, 0, len(*assets)) + done = make(chan struct{}, params.Concurrency+1) + chanAssets = make(chan RemoteAsset, params.Concurrency+1) + outputPath = path.Clean(params.OutputPath) + smdClient = client.NewClient[client.SmdClient]( client.WithSecureTLS[client.SmdClient](params.CaCertPath), ) ) + // set the client's host from the CLI param + smdClient.URI = params.URI wg.Add(params.Concurrency) for i := 0; i < params.Concurrency; i++ { go func() { for { - sr, ok := <-chanScannedResult + sr, ok := <-chanAssets if !ok { wg.Done() return } - params.Host = sr.Host - params.Port = sr.Port // generate custom xnames for bmcs node := xnames.Node{ @@ -142,7 +135,7 @@ func CollectInventory(scannedResults *[]RemoteAsset, params *CollectParams) erro log.Error().Err(err).Msg("failed to make output directory") } else { // write the output to the final path - err = os.WriteFile(path.Clean(fmt.Sprintf("%s/%s/%d.json", params.Host, outputPath, time.Now().Unix())), body, os.ModePerm) + err = os.WriteFile(path.Clean(fmt.Sprintf("%s/%s/%d.json", params.URI, outputPath, time.Now().Unix())), body, os.ModePerm) if err != nil { log.Error().Err(err).Msgf("failed to write data to file") } @@ -151,19 +144,25 @@ func CollectInventory(scannedResults *[]RemoteAsset, params *CollectParams) erro } } - // add all endpoints to smd - err = smdClient.Add(body, headers) - if err != nil { - log.Error().Err(err).Msgf("failed to add Redfish endpoint") + // add all endpoints to SMD ONLY if a host is provided + if smdClient.URI != "" { + err = smdClient.Add(body, headers) + if err != nil { + log.Error().Err(err).Msgf("failed to add Redfish endpoint") - // try updating instead - if params.ForceUpdate { - smdClient.Xname = data["ID"].(string) - err = smdClient.Update(body, headers) - if err != nil { - log.Error().Err(err).Msgf("failed to update Redfish endpoint") + // try updating instead + if params.ForceUpdate { + smdClient.Xname = data["ID"].(string) + err = smdClient.Update(body, headers) + if err != nil { + log.Error().Err(err).Msgf("failed to forcibly update Redfish endpoint") + } } } + } else { + if params.Verbose { + log.Warn().Msg("no request made (host argument is empty)") + } } // got host information, so add to list of already probed hosts @@ -173,13 +172,13 @@ func CollectInventory(scannedResults *[]RemoteAsset, params *CollectParams) erro } // use the found results to query bmc information - for _, ps := range *scannedResults { + for _, ps := range *assets { // skip if found info from host foundHost := slices.Index(found, ps.Host) if !ps.State || foundHost >= 0 { continue } - chanScannedResult <- ps + chanAssets <- ps } // handle goroutine paths @@ -193,13 +192,9 @@ func CollectInventory(scannedResults *[]RemoteAsset, params *CollectParams) erro } }() - close(chanScannedResult) + close(chanAssets) wg.Wait() close(done) return nil } - -func baseRedfishUrl(q *CollectParams) string { - return fmt.Sprintf("%s:%d", q.Host, q.Port) -} diff --git a/internal/scan.go b/internal/scan.go index 3ac6129..bdcc5ef 100644 --- a/internal/scan.go +++ b/internal/scan.go @@ -170,7 +170,7 @@ func GenerateHostsWithSubnet(subnet string, subnetMask *net.IPMask, additionalPo // GetDefaultPorts() returns a list of default ports. The only reason to have // this function is to add/remove ports without affecting usage. func GetDefaultPorts() []int { - return []int{HTTPS_PORT} + return []int{443} } // rawConnect() tries to connect to the host using DialTimeout() and waits From dce823c6d8f84618be69ff1c569335bceb5d1f92 Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 12 Aug 2024 13:04:52 -0600 Subject: [PATCH 43/67] Added URL sanitization for SMD host and moved auth from util --- cmd/collect.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cmd/collect.go b/cmd/collect.go index 26ed293..f5d83d7 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -3,10 +3,11 @@ package cmd import ( "fmt" "os/user" + "strings" magellan "github.com/OpenCHAMI/magellan/internal" "github.com/OpenCHAMI/magellan/internal/cache/sqlite" - "github.com/OpenCHAMI/magellan/internal/util" + "github.com/OpenCHAMI/magellan/pkg/auth" "github.com/cznic/mathutil" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -35,10 +36,14 @@ var collectCmd = &cobra.Command{ log.Error().Err(err).Msgf("failed to get scanned results from cache") } + // URL sanitanization for host argument + host = strings.TrimSuffix(host, "/") + host = strings.ReplaceAll(host, "//", "/") + // try to load access token either from env var, file, or config if var not set if accessToken == "" { var err error - accessToken, err = util.LoadAccessToken(tokenPath) + accessToken, err = auth.LoadAccessToken(tokenPath) if err != nil && verbose { log.Warn().Err(err).Msgf("could not load access token") } @@ -72,7 +77,7 @@ var collectCmd = &cobra.Command{ func init() { currentUser, _ = user.Current() - collectCmd.PersistentFlags().StringVar(&host, "host", "", "Set the URI to the SMD API") + collectCmd.PersistentFlags().StringVar(&host, "host", "", "Set the URI to the SMD root endpoint") collectCmd.PersistentFlags().StringVar(&username, "username", "", "Set the BMC user") collectCmd.PersistentFlags().StringVar(&password, "password", "", "Set the BMC password") collectCmd.PersistentFlags().StringVar(&scheme, "scheme", "https", "Set the scheme used to query") From 3287d765880ae7f98ee057bf0493bb1649447a57 Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 12 Aug 2024 13:05:28 -0600 Subject: [PATCH 44/67] Separated auth from util and fixed help strings --- cmd/crawl.go | 7 +++---- cmd/login.go | 4 ++-- cmd/scan.go | 8 ++++---- cmd/update.go | 2 +- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/cmd/crawl.go b/cmd/crawl.go index 2df487b..ac797a8 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -17,10 +17,9 @@ import ( var crawlCmd = &cobra.Command{ Use: "crawl [uri]", Short: "Crawl a single BMC for inventory information", - Long: "Crawl a single BMC for inventory information. This command does NOT store information" + - "store information about the scan into cache after completion. To do so, use the 'collect'" + - "command instead\n" + - "\n" + + Long: "Crawl a single BMC for inventory information. This command does NOT store information\n" + + "store information about the scan into cache after completion. To do so, use the 'collect'\n" + + "command instead\n\n" + "Examples:\n" + " magellan crawl https://bmc.example.com\n" + " magellan crawl https://bmc.example.com -i -u username -p password", diff --git a/cmd/login.go b/cmd/login.go index b0c0308..79a5243 100644 --- a/cmd/login.go +++ b/cmd/login.go @@ -7,7 +7,7 @@ import ( "os" magellan "github.com/OpenCHAMI/magellan/internal" - "github.com/OpenCHAMI/magellan/internal/util" + "github.com/OpenCHAMI/magellan/pkg/auth" "github.com/lestrrat-go/jwx/jwt" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -30,7 +30,7 @@ var loginCmd = &cobra.Command{ // check if we have a valid JWT before starting login if !forceLogin { // try getting the access token from env var - testToken, err := util.LoadAccessToken(tokenPath) + testToken, err := auth.LoadAccessToken(tokenPath) if err != nil { log.Error().Err(err).Msgf("failed to load access token") } diff --git a/cmd/scan.go b/cmd/scan.go index d246149..f8b4763 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -9,7 +9,7 @@ import ( magellan "github.com/OpenCHAMI/magellan/internal" "github.com/OpenCHAMI/magellan/internal/cache/sqlite" - "github.com/OpenCHAMI/magellan/internal/util" + "github.com/OpenCHAMI/magellan/pkg/client" "github.com/rs/zerolog/log" "github.com/cznic/mathutil" @@ -40,7 +40,7 @@ var scanCmd = &cobra.Command{ "by using the '--subnet' flag and providing an IP address on the subnet as well as a CIDR. If no CIDR is\n" + "provided, then the subnet mask specified with the '--subnet-mask' flag will be used instead (will use\n" + "default mask if not set).\n\n" + - "Similarly, any host provided with no port with use either the ports specified\n" + + "Similarly, any host provided with no port will use either the ports specified\n" + "with `--port` or the default port used with each specified protocol. The default protocol is 'tcp' unless\n" + "specified. The `--scheme` flag works similarly and the default value is 'https' in the host URL or with the\n" + "'--protocol' flag.\n\n" + @@ -72,8 +72,8 @@ var scanCmd = &cobra.Command{ } // format and combine flag and positional args - targetHosts = append(targetHosts, util.FormatHostUrls(args, ports, scheme, verbose)...) - targetHosts = append(targetHosts, util.FormatHostUrls(hosts, ports, scheme, verbose)...) + targetHosts = append(targetHosts, client.FormatHostUrls(args, ports, scheme, verbose)...) + targetHosts = append(targetHosts, client.FormatHostUrls(hosts, ports, scheme, verbose)...) // add more hosts specified with `--subnet` flag if debug { diff --git a/cmd/update.go b/cmd/update.go index 82e3106..66dcdc3 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -26,7 +26,7 @@ var ( var updateCmd = &cobra.Command{ Use: "update hosts...", Short: "Update BMC node firmware", - Long: "Perform an firmware update using Redfish by providing a remote firmware URL and component.\n" + + Long: "Perform an firmware update using Redfish by providing a remote firmware URL and component.\n\n" + "Examples:\n" + " magellan update --bmc.host 172.16.0.108 --bmc.port 443 --username bmc_username --password bmc_password --firmware-url http://172.16.0.200:8005/firmware/bios/image.RBU --component BIOS\n" + " magellan update --status --bmc.host 172.16.0.108 --bmc.port 443 --username bmc_username --password bmc_password", From a7af568ccfabea2a6486e5369cf68b609a21ffb4 Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 12 Aug 2024 13:06:02 -0600 Subject: [PATCH 45/67] Changed build rule and added release rule to Makefile --- Makefile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 0da4f4c..741a382 100644 --- a/Makefile +++ b/Makefile @@ -47,12 +47,15 @@ inst: ## go install tools go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.52.2 go install github.com/goreleaser/goreleaser@v1.18.2 -.PHONY: build -build: ## goreleaser build -build: +.PHONY: goreleaser +release: ## goreleaser build $(call print-target) goreleaser build --clean --single-target --snapshot +.PHONY: build +build: ## goreleaser build + go build --tags=all + .PHONY: docker docker: ## docker build docker: From d412e290df33afac79b20b0756e83d205e7d4e96 Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 12 Aug 2024 13:06:51 -0600 Subject: [PATCH 46/67] Removed unused port and clarified default in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cb53d7c..109ebb9 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ Flags: Use "magellan [command] --help" for more information about a command. ``` -To start a network scan for BMC nodes, use the `scan` command. If the port is not specified, `magellan` will probe ports 623 and 443 by default: +To start a network scan for BMC nodes, use the `scan` command. If the port is not specified, `magellan` will probe the common Redfish port 443 by default: ```bash ./magellan scan \ From 411102881de7fb68ad21013b6e67515ae699e14f Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 12 Aug 2024 13:07:30 -0600 Subject: [PATCH 47/67] Removed files from util --- internal/util/auth.go | 37 --------- internal/util/net.go | 178 ------------------------------------------ 2 files changed, 215 deletions(-) delete mode 100644 internal/util/auth.go delete mode 100644 internal/util/net.go diff --git a/internal/util/auth.go b/internal/util/auth.go deleted file mode 100644 index 9df0a11..0000000 --- a/internal/util/auth.go +++ /dev/null @@ -1,37 +0,0 @@ -package util - -import ( - "fmt" - "os" - - "github.com/spf13/viper" -) - -// LoadAccessToken() tries to load a JWT string from an environment -// variable, file, or config in that order. If loading the token -// fails with one options, it will fallback to the next option until -// all options are exhausted. -// -// Returns a token as a string with no error if successful. -// Alternatively, returns an empty string with an error if a token is -// not able to be loaded. -func LoadAccessToken(path string) (string, error) { - // try to load token from env var - testToken := os.Getenv("ACCESS_TOKEN") - if testToken != "" { - return testToken, nil - } - - // try reading access token from a file - b, err := os.ReadFile(path) - if err == nil { - return string(b), nil - } - - // TODO: try to load token from config - testToken = viper.GetString("access-token") - if testToken != "" { - return testToken, nil - } - return "", fmt.Errorf("failed to load token from environment variable, file, or config") -} diff --git a/internal/util/net.go b/internal/util/net.go deleted file mode 100644 index 76749aa..0000000 --- a/internal/util/net.go +++ /dev/null @@ -1,178 +0,0 @@ -package util - -import ( - "bytes" - "crypto/tls" - "fmt" - "io" - "net" - "net/http" - "net/url" - "strings" - - "github.com/rs/zerolog/log" -) - -// HTTP aliases for readibility -type HTTPHeader map[string]string -type HTTPBody []byte - -func (h HTTPHeader) Authorization(accessToken string) HTTPHeader { - if accessToken != "" { - h["Authorization"] = fmt.Sprintf("Bearer %s", accessToken) - } - return h -} - -func (h HTTPHeader) ContentType(contentType string) HTTPHeader { - h["Content-Type"] = contentType - return h -} - -// GetNextIP() returns the next IP address, but does not account -// for net masks. -func GetNextIP(ip *net.IP, inc uint) *net.IP { - if ip == nil { - return &net.IP{} - } - i := ip.To4() - v := uint(i[0])<<24 + uint(i[1])<<16 + uint(i[2])<<8 + uint(i[3]) - v += inc - v3 := byte(v & 0xFF) - v2 := byte((v >> 8) & 0xFF) - v1 := byte((v >> 16) & 0xFF) - v0 := byte((v >> 24) & 0xFF) - // return &net.IP{[]byte{v0, v1, v2, v3}} - r := net.IPv4(v0, v1, v2, v3) - return &r -} - -// MakeRequest() is a wrapper function that condenses simple HTTP -// requests done to a single call. It expects an optional HTTP client, -// URL, HTTP method, request body, and request headers. This function -// is useful when making many requests where only these few arguments -// are changing. -// -// Returns a HTTP response object, response body as byte array, and any -// error that may have occurred with making the request. -func MakeRequest(client *http.Client, url string, httpMethod string, body HTTPBody, header HTTPHeader) (*http.Response, HTTPBody, error) { - // use defaults if no client provided - if client == nil { - client = http.DefaultClient - client.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - } - req, err := http.NewRequest(httpMethod, url, bytes.NewBuffer(body)) - if err != nil { - return nil, nil, fmt.Errorf("failed to create new HTTP request: %v", err) - } - req.Header.Add("User-Agent", "magellan") - for k, v := range header { - req.Header.Add(k, v) - } - res, err := client.Do(req) - if err != nil { - return nil, nil, fmt.Errorf("failed to make request: %v", err) - } - b, err := io.ReadAll(res.Body) - res.Body.Close() - if err != nil { - return nil, nil, fmt.Errorf("failed to read response body: %v", err) - } - return res, b, err -} - -// FormatHostUrls() takes a list of hosts and ports and builds full URLs in the -// form of scheme://host:port. If no scheme is provided, it will use "https" by -// default. -// -// Returns a 2D string slice where each slice contains URL host strings for each -// port. The intention is to have all of the URLs for a single host combined into -// a single slice to initiate one goroutine per host, but making request to multiple -// ports. -func FormatHostUrls(hosts []string, ports []int, scheme string, verbose bool) [][]string { - // format each positional arg as a complete URL - var formattedHosts [][]string - for _, host := range hosts { - uri, err := url.ParseRequestURI(host) - if err != nil { - if verbose { - log.Warn().Msgf("invalid URI parsed: %s", host) - } - continue - } - - // check if scheme is set, if not set it with flag or default value ('https' if flag is not set) - if uri.Scheme == "" { - if scheme != "" { - uri.Scheme = scheme - } else { - // hardcoded assumption - uri.Scheme = "https" - } - } - - // tidy up slashes and update arg with new value - uri.Path = strings.TrimSuffix(uri.Path, "/") - uri.Path = strings.ReplaceAll(uri.Path, "//", "/") - - // for hosts with unspecified ports, add ports to scan from flag - if uri.Port() == "" { - var tmp []string - for _, port := range ports { - uri.Host += fmt.Sprintf(":%d", port) - tmp = append(tmp, uri.String()) - } - formattedHosts = append(formattedHosts, tmp) - } else { - formattedHosts = append(formattedHosts, []string{uri.String()}) - } - - } - return formattedHosts -} - -// FormatIPUrls() takes a list of IP addresses and ports and builds full URLs in the -// form of scheme://host:port. If no scheme is provided, it will use "https" by -// default. -// -// Returns a 2D string slice where each slice contains URL host strings for each -// port. The intention is to have all of the URLs for a single host combined into -// a single slice to initiate one goroutine per host, but making request to multiple -// ports. -func FormatIPUrls(ips []string, ports []int, scheme string, verbose bool) [][]string { - // format each positional arg as a complete URL - var formattedHosts [][]string - for _, ip := range ips { - if scheme == "" { - scheme = "https" - } - // make an entirely new object since we're expecting just IPs - uri := &url.URL{ - Scheme: scheme, - Host: ip, - } - - // tidy up slashes and update arg with new value - uri.Path = strings.ReplaceAll(uri.Path, "//", "/") - uri.Path = strings.TrimSuffix(uri.Path, "/") - - // for hosts with unspecified ports, add ports to scan from flag - if uri.Port() == "" { - if len(ports) == 0 { - ports = append(ports, 443) - } - var tmp []string - for _, port := range ports { - uri.Host += fmt.Sprintf(":%d", port) - tmp = append(tmp, uri.String()) - } - formattedHosts = append(formattedHosts, tmp) - } else { - formattedHosts = append(formattedHosts, []string{uri.String()}) - } - - } - return formattedHosts -} From da2abdfa8df3b12b8ec5dc79e910e820c38f96f7 Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 12 Aug 2024 13:07:57 -0600 Subject: [PATCH 48/67] Updated tests to reflect new API changes --- tests/api_test.go | 25 +++++++++++++++++++------ tests/compatibility_test.go | 6 +++--- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/tests/api_test.go b/tests/api_test.go index 05f7cd2..4b8ddf3 100644 --- a/tests/api_test.go +++ b/tests/api_test.go @@ -13,18 +13,31 @@ import ( magellan "github.com/OpenCHAMI/magellan/internal" ) +var ( + scanParams = &magellan.ScanParams{ + TargetHosts: [][]string{ + []string{ + "http://127.0.0.1:443", + "http://127.0.0.1:5000", + }, + }, + Scheme: "https", + Protocol: "tcp", + Concurrency: 1, + Timeout: 30, + DisableProbing: false, + Verbose: false, + } +) + func TestScanAndCollect(t *testing.T) { - var ( - hosts = []string{"http://127.0.0.1"} - ports = []int{5000} - ) // do a scan on the emulator cluster with probing disabled and check results - results := magellan.ScanForAssets(hosts, ports, 1, 30, true, false) + results := magellan.ScanForAssets(scanParams) if len(results) <= 0 { t.Fatal("expected to find at least one BMC node, but found none") } // do a scan on the emulator cluster with probing enabled - results = magellan.ScanForAssets(hosts, ports, 1, 30, false, false) + results = magellan.ScanForAssets(scanParams) if len(results) <= 0 { t.Fatal("expected to find at least one BMC node, but found none") } diff --git a/tests/compatibility_test.go b/tests/compatibility_test.go index 5517bd2..b879536 100644 --- a/tests/compatibility_test.go +++ b/tests/compatibility_test.go @@ -13,7 +13,7 @@ import ( "net/http" "testing" - "github.com/OpenCHAMI/magellan/internal/util" + "github.com/OpenCHAMI/magellan/pkg/client" "github.com/OpenCHAMI/magellan/pkg/crawler" ) @@ -30,7 +30,7 @@ func TestRedfishV1Availability(t *testing.T) { body = []byte{} headers = map[string]string{} ) - res, b, err := util.MakeRequest(nil, url, http.MethodGet, body, headers) + res, b, err := client.MakeRequest(nil, url, http.MethodGet, body, headers) if err != nil { t.Fatalf("failed to make request to BMC: %v", err) } @@ -60,7 +60,7 @@ func TestRedfishVersion(t *testing.T) { headers = map[string]string{} ) - util.MakeRequest(nil, url, http.MethodGet, body, headers) + client.MakeRequest(nil, url, http.MethodGet, body, headers) } // Crawls a BMC node and checks that we're able to query certain properties From e02558fd00afd4208ff8e70c2ced905f7dccb642 Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 12 Aug 2024 13:08:30 -0600 Subject: [PATCH 49/67] Minor changes --- internal/collect.go | 5 +++-- internal/scan.go | 8 ++++---- internal/update.go | 6 +++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/internal/collect.go b/internal/collect.go index 79f873e..4715ec3 100644 --- a/internal/collect.go +++ b/internal/collect.go @@ -75,6 +75,7 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams) error { } // generate custom xnames for bmcs + // TODO: add xname customization via CLI node := xnames.Node{ Cabinet: 1000, Chassis: 1, @@ -83,7 +84,7 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams) error { } offset += 1 - // TODO: use pkg/crawler to request inventory data via Redfish + // crawl BMC node to fetch inventory data via Redfish systems, err := crawler.CrawlBMC(crawler.CrawlerConfig{ URI: fmt.Sprintf("%s:%d", sr.Host, sr.Port), Username: params.Username, @@ -107,7 +108,7 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams) error { } // create and set headers for request - headers := util.HTTPHeader{} + headers := client.HTTPHeader{} headers.Authorization(params.AccessToken) headers.ContentType("application/json") diff --git a/internal/scan.go b/internal/scan.go index bdcc5ef..736329d 100644 --- a/internal/scan.go +++ b/internal/scan.go @@ -10,7 +10,7 @@ import ( "sync" "time" - "github.com/OpenCHAMI/magellan/internal/util" + "github.com/OpenCHAMI/magellan/pkg/client" "github.com/rs/zerolog/log" ) @@ -84,7 +84,7 @@ func ScanForAssets(params *ScanParams) []RemoteAsset { assetsToAdd := []RemoteAsset{} for _, foundAsset := range foundAssets { url := fmt.Sprintf("%s://%s:%d/redfish/v1/", params.Scheme, foundAsset.Host, foundAsset.Port) - res, _, err := util.MakeRequest(nil, url, http.MethodGet, nil, nil) + res, _, err := client.MakeRequest(nil, url, http.MethodGet, nil, nil) if err != nil || res == nil { if params.Verbose { log.Printf("failed to make request: %v\n", err) @@ -164,7 +164,7 @@ func GenerateHostsWithSubnet(subnet string, subnetMask *net.IPMask, additionalPo // generate new IPs from subnet and format to full URL subnetIps := generateIPsWithSubnet(&subnetIp, subnetMask) - return util.FormatIPUrls(subnetIps, additionalPorts, defaultScheme, false) + return client.FormatIPUrls(subnetIps, additionalPorts, defaultScheme, false) } // GetDefaultPorts() returns a list of default ports. The only reason to have @@ -238,7 +238,7 @@ func generateIPsWithSubnet(ip *net.IP, mask *net.IPMask) []string { hosts := []string{} end := int(math.Pow(2, float64((bits-ones)))) - 1 for i := 0; i < end; i++ { - ip = util.GetNextIP(ip, 1) + ip = client.GetNextIP(ip, 1) if ip == nil { continue } diff --git a/internal/update.go b/internal/update.go index b71002b..9191818 100644 --- a/internal/update.go +++ b/internal/update.go @@ -6,7 +6,7 @@ import ( "net/http" "net/url" - "github.com/OpenCHAMI/magellan/internal/util" + "github.com/OpenCHAMI/magellan/pkg/client" ) type UpdateParams struct { @@ -43,7 +43,7 @@ func UpdateFirmwareRemote(q *UpdateParams) error { if err != nil { return fmt.Errorf("failed to marshal data: %v", err) } - res, body, err := util.MakeRequest(nil, updateUrl, "POST", data, headers) + res, body, err := client.MakeRequest(nil, updateUrl, "POST", data, headers) if err != nil { return fmt.Errorf("something went wrong: %v", err) } else if res == nil { @@ -63,7 +63,7 @@ func GetUpdateStatus(q *UpdateParams) error { } uri.User = url.UserPassword(q.Username, q.Password) updateUrl := fmt.Sprintf("%s/redfish/v1/UpdateService", uri.String()) - res, body, err := util.MakeRequest(nil, updateUrl, "GET", nil, nil) + res, body, err := client.MakeRequest(nil, updateUrl, "GET", nil, nil) if err != nil { return fmt.Errorf("something went wrong: %v", err) } else if res == nil { From c0a6d3bb6ff328095cd3604b551c2df8b5419336 Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 12 Aug 2024 13:09:15 -0600 Subject: [PATCH 50/67] More minor changes --- pkg/auth/auth.go | 37 +++++++++ pkg/client/client.go | 10 +-- pkg/client/default.go | 10 +-- pkg/client/net.go | 178 ++++++++++++++++++++++++++++++++++++++++++ pkg/client/smd.go | 10 +-- 5 files changed, 227 insertions(+), 18 deletions(-) create mode 100644 pkg/auth/auth.go create mode 100644 pkg/client/net.go diff --git a/pkg/auth/auth.go b/pkg/auth/auth.go new file mode 100644 index 0000000..da5285d --- /dev/null +++ b/pkg/auth/auth.go @@ -0,0 +1,37 @@ +package auth + +import ( + "fmt" + "os" + + "github.com/spf13/viper" +) + +// LoadAccessToken() tries to load a JWT string from an environment +// variable, file, or config in that order. If loading the token +// fails with one options, it will fallback to the next option until +// all options are exhausted. +// +// Returns a token as a string with no error if successful. +// Alternatively, returns an empty string with an error if a token is +// not able to be loaded. +func LoadAccessToken(path string) (string, error) { + // try to load token from env var + testToken := os.Getenv("ACCESS_TOKEN") + if testToken != "" { + return testToken, nil + } + + // try reading access token from a file + b, err := os.ReadFile(path) + if err == nil { + return string(b), nil + } + + // TODO: try to load token from config + testToken = viper.GetString("access-token") + if testToken != "" { + return testToken, nil + } + return "", fmt.Errorf("failed to load token from environment variable, file, or config") +} diff --git a/pkg/client/client.go b/pkg/client/client.go index 739332a..0005254 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -9,8 +9,6 @@ import ( "net/http" "os" "time" - - "github.com/OpenCHAMI/magellan/internal/util" ) type Option[T Client] func(client T) @@ -24,8 +22,8 @@ type Client interface { RootEndpoint(endpoint string) string // functions needed to make request - Add(data util.HTTPBody, headers util.HTTPHeader) error - Update(data util.HTTPBody, headers util.HTTPHeader) error + Add(data HTTPBody, headers HTTPHeader) error + Update(data HTTPBody, headers HTTPHeader) error } // NewClient() creates a new client @@ -71,11 +69,11 @@ func WithSecureTLS[T Client](certPath string) func(T) { // Post() is a simplified wrapper function that packages all of the // that marshals a mapper into a JSON-formatted byte array, and then performs // a request to the specified URL. -func (c *MagellanClient) Post(url string, data map[string]any, header util.HTTPHeader) (*http.Response, util.HTTPBody, error) { +func (c *MagellanClient) Post(url string, data map[string]any, header HTTPHeader) (*http.Response, HTTPBody, error) { // serialize data into byte array body, err := json.Marshal(data) if err != nil { return nil, nil, fmt.Errorf("failed to marshal data for request: %v", err) } - return util.MakeRequest(c.Client, url, http.MethodPost, body, header) + return MakeRequest(c.Client, url, http.MethodPost, body, header) } diff --git a/pkg/client/default.go b/pkg/client/default.go index e466125..2830921 100644 --- a/pkg/client/default.go +++ b/pkg/client/default.go @@ -3,8 +3,6 @@ package client import ( "fmt" "net/http" - - "github.com/OpenCHAMI/magellan/internal/util" ) type MagellanClient struct { @@ -20,13 +18,13 @@ func (c *MagellanClient) Name() string { // the first argument with no data processing or manipulation. The function sends // the data to a set callback URL (which may be changed to use a configurable value // instead). -func (c *MagellanClient) Add(data util.HTTPBody, headers util.HTTPHeader) error { +func (c *MagellanClient) Add(data HTTPBody, headers HTTPHeader) error { if data == nil { return fmt.Errorf("no data found") } path := "/inventory/add" - res, body, err := util.MakeRequest(c.Client, path, http.MethodPost, data, headers) + res, body, err := MakeRequest(c.Client, path, http.MethodPost, data, headers) if res != nil { statusOk := res.StatusCode >= 200 && res.StatusCode < 300 if !statusOk { @@ -37,13 +35,13 @@ func (c *MagellanClient) Add(data util.HTTPBody, headers util.HTTPHeader) error return err } -func (c *MagellanClient) Update(data util.HTTPBody, headers util.HTTPHeader) error { +func (c *MagellanClient) Update(data HTTPBody, headers HTTPHeader) error { if data == nil { return fmt.Errorf("no data found") } path := "/inventory/update" - res, body, err := util.MakeRequest(c.Client, path, http.MethodPut, data, headers) + res, body, err := MakeRequest(c.Client, path, http.MethodPut, data, headers) if res != nil { statusOk := res.StatusCode >= 200 && res.StatusCode < 300 if !statusOk { diff --git a/pkg/client/net.go b/pkg/client/net.go new file mode 100644 index 0000000..af69a53 --- /dev/null +++ b/pkg/client/net.go @@ -0,0 +1,178 @@ +package client + +import ( + "bytes" + "crypto/tls" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + + "github.com/rs/zerolog/log" +) + +// HTTP aliases for readibility +type HTTPHeader map[string]string +type HTTPBody []byte + +func (h HTTPHeader) Authorization(accessToken string) HTTPHeader { + if accessToken != "" { + h["Authorization"] = fmt.Sprintf("Bearer %s", accessToken) + } + return h +} + +func (h HTTPHeader) ContentType(contentType string) HTTPHeader { + h["Content-Type"] = contentType + return h +} + +// GetNextIP() returns the next IP address, but does not account +// for net masks. +func GetNextIP(ip *net.IP, inc uint) *net.IP { + if ip == nil { + return &net.IP{} + } + i := ip.To4() + v := uint(i[0])<<24 + uint(i[1])<<16 + uint(i[2])<<8 + uint(i[3]) + v += inc + v3 := byte(v & 0xFF) + v2 := byte((v >> 8) & 0xFF) + v1 := byte((v >> 16) & 0xFF) + v0 := byte((v >> 24) & 0xFF) + // return &net.IP{[]byte{v0, v1, v2, v3}} + r := net.IPv4(v0, v1, v2, v3) + return &r +} + +// MakeRequest() is a wrapper function that condenses simple HTTP +// requests done to a single call. It expects an optional HTTP client, +// URL, HTTP method, request body, and request headers. This function +// is useful when making many requests where only these few arguments +// are changing. +// +// Returns a HTTP response object, response body as byte array, and any +// error that may have occurred with making the request. +func MakeRequest(client *http.Client, url string, httpMethod string, body HTTPBody, header HTTPHeader) (*http.Response, HTTPBody, error) { + // use defaults if no client provided + if client == nil { + client = http.DefaultClient + client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + req, err := http.NewRequest(httpMethod, url, bytes.NewBuffer(body)) + if err != nil { + return nil, nil, fmt.Errorf("failed to create new HTTP request: %v", err) + } + req.Header.Add("User-Agent", "magellan") + for k, v := range header { + req.Header.Add(k, v) + } + res, err := client.Do(req) + if err != nil { + return nil, nil, fmt.Errorf("failed to make request: %v", err) + } + b, err := io.ReadAll(res.Body) + res.Body.Close() + if err != nil { + return nil, nil, fmt.Errorf("failed to read response body: %v", err) + } + return res, b, err +} + +// FormatHostUrls() takes a list of hosts and ports and builds full URLs in the +// form of scheme://host:port. If no scheme is provided, it will use "https" by +// default. +// +// Returns a 2D string slice where each slice contains URL host strings for each +// port. The intention is to have all of the URLs for a single host combined into +// a single slice to initiate one goroutine per host, but making request to multiple +// ports. +func FormatHostUrls(hosts []string, ports []int, scheme string, verbose bool) [][]string { + // format each positional arg as a complete URL + var formattedHosts [][]string + for _, host := range hosts { + uri, err := url.ParseRequestURI(host) + if err != nil { + if verbose { + log.Warn().Msgf("invalid URI parsed: %s", host) + } + continue + } + + // check if scheme is set, if not set it with flag or default value ('https' if flag is not set) + if uri.Scheme == "" { + if scheme != "" { + uri.Scheme = scheme + } else { + // hardcoded assumption + uri.Scheme = "https" + } + } + + // tidy up slashes and update arg with new value + uri.Path = strings.TrimSuffix(uri.Path, "/") + uri.Path = strings.ReplaceAll(uri.Path, "//", "/") + + // for hosts with unspecified ports, add ports to scan from flag + if uri.Port() == "" { + var tmp []string + for _, port := range ports { + uri.Host += fmt.Sprintf(":%d", port) + tmp = append(tmp, uri.String()) + } + formattedHosts = append(formattedHosts, tmp) + } else { + formattedHosts = append(formattedHosts, []string{uri.String()}) + } + + } + return formattedHosts +} + +// FormatIPUrls() takes a list of IP addresses and ports and builds full URLs in the +// form of scheme://host:port. If no scheme is provided, it will use "https" by +// default. +// +// Returns a 2D string slice where each slice contains URL host strings for each +// port. The intention is to have all of the URLs for a single host combined into +// a single slice to initiate one goroutine per host, but making request to multiple +// ports. +func FormatIPUrls(ips []string, ports []int, scheme string, verbose bool) [][]string { + // format each positional arg as a complete URL + var formattedHosts [][]string + for _, ip := range ips { + if scheme == "" { + scheme = "https" + } + // make an entirely new object since we're expecting just IPs + uri := &url.URL{ + Scheme: scheme, + Host: ip, + } + + // tidy up slashes and update arg with new value + uri.Path = strings.ReplaceAll(uri.Path, "//", "/") + uri.Path = strings.TrimSuffix(uri.Path, "/") + + // for hosts with unspecified ports, add ports to scan from flag + if uri.Port() == "" { + if len(ports) == 0 { + ports = append(ports, 443) + } + var tmp []string + for _, port := range ports { + uri.Host += fmt.Sprintf(":%d", port) + tmp = append(tmp, uri.String()) + } + formattedHosts = append(formattedHosts, tmp) + } else { + formattedHosts = append(formattedHosts, []string{uri.String()}) + } + + } + return formattedHosts +} diff --git a/pkg/client/smd.go b/pkg/client/smd.go index ca3865e..f39d3ce 100644 --- a/pkg/client/smd.go +++ b/pkg/client/smd.go @@ -6,8 +6,6 @@ package client import ( "fmt" "net/http" - - "github.com/OpenCHAMI/magellan/internal/util" ) type SmdClient struct { @@ -31,14 +29,14 @@ func (c SmdClient) GetClient() *http.Client { // Add() has a similar function definition to that of the default implementation, // but also allows further customization and data/header manipulation that would // be specific and/or unique to SMD's API. -func (c SmdClient) Add(data util.HTTPBody, headers util.HTTPHeader) error { +func (c SmdClient) Add(data HTTPBody, headers HTTPHeader) error { if data == nil { return fmt.Errorf("failed to add redfish endpoint: no data found") } // Add redfish endpoint via POST `/hsm/v2/Inventory/RedfishEndpoints` endpoint url := c.RootEndpoint("/Inventory/RedfishEndpoints") - res, body, err := util.MakeRequest(c.Client, url, http.MethodPost, data, headers) + res, body, err := MakeRequest(c.Client, url, http.MethodPost, data, headers) if res != nil { statusOk := res.StatusCode >= 200 && res.StatusCode < 300 if !statusOk { @@ -49,13 +47,13 @@ func (c SmdClient) Add(data util.HTTPBody, headers util.HTTPHeader) error { return err } -func (c SmdClient) Update(data util.HTTPBody, headers util.HTTPHeader) error { +func (c SmdClient) Update(data HTTPBody, headers HTTPHeader) error { if data == nil { return fmt.Errorf("failed to add redfish endpoint: no data found") } // Update redfish endpoint via PUT `/hsm/v2/Inventory/RedfishEndpoints` endpoint url := c.RootEndpoint("/Inventory/RedfishEndpoints/" + c.Xname) - res, body, err := util.MakeRequest(c.Client, url, http.MethodPut, data, headers) + res, body, err := MakeRequest(c.Client, url, http.MethodPut, data, headers) fmt.Printf("%v (%v)\n%s\n", url, res.Status, string(body)) if res != nil { statusOk := res.StatusCode >= 200 && res.StatusCode < 300 From 4aef5166a539c0e44a6b778a580fa4932c9d1fbf Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 12 Aug 2024 13:59:28 -0600 Subject: [PATCH 51/67] Fixed typo errors in changelog and readme --- CHANGELOG.md | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f999cdb..ebc8586 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Ability to update firmware * Refactored connection handling for faster scanning - * Updated to refelct home at github.com/OpenCHAMI + * Updated to reflect home at github.com/OpenCHAMI * Updated to reflect ghcr.io as container home ## [Unreleased] diff --git a/README.md b/README.md index 109ebb9..79527bb 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ Note: If the `cache` flag is not set, `magellan` will use "/tmp/$USER/magellan.d ### Updating Firmware -The `magellan` tool is capable of updating firmware with using the `update` subcommand via the Redfish API. This may sometimes necessary if some of the `collect` output is missing or is not including what is expected. The subcommand expects there to be a running HTTP/HTTPS server running that has an accessbile URL path to the firmware download. Specify the URL with the `--firmware-path` flag and the firmware type with the `--component` flag with all the other usual arguments like in the example below: +The `magellan` tool is capable of updating firmware with using the `update` subcommand via the Redfish API. This may sometimes necessary if some of the `collect` output is missing or is not including what is expected. The subcommand expects there to be a running HTTP/HTTPS server running that has an accessible URL path to the firmware download. Specify the URL with the `--firmware-path` flag and the firmware type with the `--component` flag with all the other usual arguments like in the example below: ```bash ./magellan update \ @@ -228,7 +228,7 @@ At its core, `magellan` is designed to do three basic things: First, the tool performs a scan to find running services on a network. This is done by sending a raw TCP packet to all specified hosts (either IP or host name) and taking note which services respond. At this point, `magellan` has no way of knowing whether this is a Redfish service or not, so another HTTP request is made to verify. Once the BMC responds with an OK status code, `magellan` will store the necessary information in a local cache database to allow collecting more information about the node later. This allows for users to only have to scan their cluster once to find systems that are currently available and scannable. -Next, the tool queries information about the BMC node using `gofish` API functions, but requires access to BMC node found in the scanning step mentioned above to work. If the node requires basic authentication, a user name and password is required to be supplied as well. Once the BMC information is retrived from each node, the info is aggregated and a HTTP request is made to a SMD instance to be stored. Optionally, the information can be written to disk for inspection and debugging purposes. +Next, the tool queries information about the BMC node using `gofish` API functions, but requires access to BMC node found in the scanning step mentioned above to work. If the node requires basic authentication, a user name and password is required to be supplied as well. Once the BMC information is retrieved from each node, the info is aggregated and a HTTP request is made to a SMD instance to be stored. Optionally, the information can be written to disk for inspection and debugging purposes. In summary, `magellan` needs at minimum the following configured to work on each node: From e3d0791ec158c01a2db518337ef35dcee465d8be Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 12 Aug 2024 16:35:21 -0600 Subject: [PATCH 52/67] Changed 'docker' rule to 'container' --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 741a382..43aea4b 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ SHELL := /bin/bash .DEFAULT_GOAL := all .PHONY: all all: ## build pipeline -all: mod inst build spell lint test +all: mod inst build lint test .PHONY: ci ci: ## CI build pipeline @@ -57,8 +57,8 @@ build: ## goreleaser build go build --tags=all .PHONY: docker -docker: ## docker build -docker: +container: ## docker build +container: $(call print-target) docker build . --build-arg REGISTRY_HOST=${REGISTRY_HOST} --no-cache --pull --tag '${NAME}:${VERSION}' From 5a2356d7c277e4cb03eae630170096d6b0a3ef16 Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 12 Aug 2024 16:35:58 -0600 Subject: [PATCH 53/67] Fixed crawl command help string --- cmd/crawl.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/crawl.go b/cmd/crawl.go index ac797a8..1eff5ad 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -18,8 +18,7 @@ var crawlCmd = &cobra.Command{ Use: "crawl [uri]", Short: "Crawl a single BMC for inventory information", Long: "Crawl a single BMC for inventory information. This command does NOT store information\n" + - "store information about the scan into cache after completion. To do so, use the 'collect'\n" + - "command instead\n\n" + + "about the scan into cache after completion. To do so, use the 'collect' command instead\n\n" + "Examples:\n" + " magellan crawl https://bmc.example.com\n" + " magellan crawl https://bmc.example.com -i -u username -p password", From 128f9ad42d6ff566e3d41c4c735d05eb54c1a2a5 Mon Sep 17 00:00:00 2001 From: David Allen Date: Mon, 12 Aug 2024 17:28:05 -0600 Subject: [PATCH 54/67] Updated Makefile to include GOPATH in some targets --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 43aea4b..f8de63c 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ $(error VERSION is not set. Please review and copy config.env.default to config endif SHELL := /bin/bash +GOPATH ?= $(shell echo $${GOPATH:-~/go}) .DEFAULT_GOAL := all .PHONY: all @@ -50,7 +51,7 @@ inst: ## go install tools .PHONY: goreleaser release: ## goreleaser build $(call print-target) - goreleaser build --clean --single-target --snapshot + $(GOPATH)/bin/goreleaser build --clean --single-target --snapshot .PHONY: build build: ## goreleaser build @@ -65,12 +66,12 @@ container: .PHONY: spell spell: ## misspell $(call print-target) - misspell -error -locale=US -w **.md + $(GOPATH)/bin/misspell -error -locale=US -w **.md .PHONY: lint lint: ## golangci-lint $(call print-target) - golangci-lint run --fix + $(GOPATH)/bin/golangci-lint run --fix .PHONY: test test: ## go test From 81ec43a923cc750a69b2870e0f8ffede9b7db910 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 13 Aug 2024 10:41:06 -0600 Subject: [PATCH 55/67] Minor changes to fix lint errors --- cmd/collect.go | 25 ++++++++---------- cmd/crawl.go | 4 +-- cmd/list.go | 4 +-- cmd/login.go | 12 ++++----- cmd/root.go | 51 ++++++++++++++++++++++--------------- cmd/scan.go | 13 ++++++---- cmd/update.go | 34 +++++++++++-------------- internal/util/error.go | 2 +- pkg/client/smd.go | 2 +- tests/api_test.go | 6 ++++- tests/compatibility_test.go | 15 +++++++---- 11 files changed, 90 insertions(+), 78 deletions(-) diff --git a/cmd/collect.go b/cmd/collect.go index f5d83d7..74d3521 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -14,10 +14,6 @@ import ( "github.com/spf13/viper" ) -var ( - forceUpdate bool -) - // The `collect` command fetches data from a collection of BMC nodes. // This command should be ran after the `scan` to find available hosts // on a subnet. @@ -82,7 +78,7 @@ func init() { collectCmd.PersistentFlags().StringVar(&password, "password", "", "Set the BMC password") collectCmd.PersistentFlags().StringVar(&scheme, "scheme", "https", "Set the scheme used to query") collectCmd.PersistentFlags().StringVar(&protocol, "protocol", "tcp", "Set the protocol used to query") - collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/inventory/", currentUser.Username+"/"), "set the path to store collection data") + collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/inventory/", currentUser.Username+"/"), "Set the path to store collection data") collectCmd.PersistentFlags().BoolVar(&forceUpdate, "force-update", false, "Set flag to force update data sent to SMD") collectCmd.PersistentFlags().StringVar(&cacertPath, "cacert", "", "Path to CA cert. (defaults to system CAs)") @@ -90,16 +86,15 @@ func init() { collectCmd.MarkFlagsRequiredTogether("username", "password") // bind flags to config properties - viper.BindPFlag("collect.driver", collectCmd.Flags().Lookup("driver")) - viper.BindPFlag("collect.host", collectCmd.Flags().Lookup("host")) - viper.BindPFlag("collect.port", collectCmd.Flags().Lookup("port")) - viper.BindPFlag("collect.username", collectCmd.Flags().Lookup("username")) - viper.BindPFlag("collect.password", collectCmd.Flags().Lookup("password")) - viper.BindPFlag("collect.protocol", collectCmd.Flags().Lookup("protocol")) - viper.BindPFlag("collect.output", collectCmd.Flags().Lookup("output")) - viper.BindPFlag("collect.force-update", collectCmd.Flags().Lookup("force-update")) - viper.BindPFlag("collect.cacert", collectCmd.Flags().Lookup("secure-tls")) - viper.BindPFlags(collectCmd.Flags()) + checkBindFlagError(viper.BindPFlag("collect.host", collectCmd.Flags().Lookup("collect.host"))) + checkBindFlagError(viper.BindPFlag("collect.username", collectCmd.Flags().Lookup("collect.username"))) + checkBindFlagError(viper.BindPFlag("collect.password", collectCmd.Flags().Lookup("collect.password"))) + checkBindFlagError(viper.BindPFlag("collect.scheme", collectCmd.Flags().Lookup("collect.scheme"))) + checkBindFlagError(viper.BindPFlag("collect.protocol", collectCmd.Flags().Lookup("collect.protocol"))) + checkBindFlagError(viper.BindPFlag("collect.output", collectCmd.Flags().Lookup("collect.output"))) + checkBindFlagError(viper.BindPFlag("collect.force-update", collectCmd.Flags().Lookup("collect.force-update"))) + checkBindFlagError(viper.BindPFlag("collect.cacert", collectCmd.Flags().Lookup("collect.cacert"))) + checkBindFlagError(viper.BindPFlags(collectCmd.Flags())) rootCmd.AddCommand(collectCmd) } diff --git a/cmd/crawl.go b/cmd/crawl.go index 1eff5ad..4dab37d 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -62,8 +62,8 @@ var crawlCmd = &cobra.Command{ } func init() { - crawlCmd.Flags().StringP("username", "u", "", "Username for the BMC") - crawlCmd.Flags().StringP("password", "p", "", "Password for the BMC") + crawlCmd.Flags().StringP("username", "u", "", "Set the username for the BMC") + crawlCmd.Flags().StringP("password", "p", "", "Set the password for the BMC") crawlCmd.Flags().BoolP("insecure", "i", false, "Ignore SSL errors") rootCmd.AddCommand(crawlCmd) diff --git a/cmd/list.go b/cmd/list.go index 826f1e3..e09299f 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -55,7 +55,7 @@ var listCmd = &cobra.Command{ } func init() { - listCmd.Flags().StringVar(&format, "format", "", "set the output format (json|default)") - listCmd.Flags().BoolVar(&showCache, "cache-info", false, "show cache information and exit") + listCmd.Flags().StringVar(&format, "format", "", "Set the output format (json|default)") + listCmd.Flags().BoolVar(&showCache, "cache-info", false, "Show cache information and exit") rootCmd.AddCommand(listCmd) } diff --git a/cmd/login.go b/cmd/login.go index 79a5243..47aec06 100644 --- a/cmd/login.go +++ b/cmd/login.go @@ -77,11 +77,11 @@ var loginCmd = &cobra.Command{ } func init() { - loginCmd.Flags().StringVar(&loginUrl, "url", "http://127.0.0.1:3333/login", "set the login URL") - loginCmd.Flags().StringVar(&targetHost, "target-host", "127.0.0.1", "set the target host to return the access code") - loginCmd.Flags().IntVar(&targetPort, "target-port", 5000, "set the target host to return the access code") - loginCmd.Flags().BoolVarP(&forceLogin, "force", "f", false, "start the login process even with a valid token") - loginCmd.Flags().StringVar(&tokenPath, "token-path", ".ochami-token", "set the path the load/save the access token") - loginCmd.Flags().BoolVar(&noBrowser, "no-browser", false, "prevent the default browser from being opened automatically") + loginCmd.Flags().StringVar(&loginUrl, "url", "http://127.0.0.1:3333/login", "Set the login URL") + loginCmd.Flags().StringVar(&targetHost, "target-host", "127.0.0.1", "Set the target host to return the access code") + loginCmd.Flags().IntVar(&targetPort, "target-port", 5000, "Set the target host to return the access code") + loginCmd.Flags().BoolVarP(&forceLogin, "force", "f", false, "Start the login process even with a valid token") + loginCmd.Flags().StringVar(&tokenPath, "token-path", ".ochami-token", "Set the path to load/save the access token") + loginCmd.Flags().BoolVar(&noBrowser, "no-browser", false, "Prevent the default browser from being opened automatically") rootCmd.AddCommand(loginCmd) } diff --git a/cmd/root.go b/cmd/root.go index a761f7d..e9a497f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -43,6 +43,7 @@ var ( configPath string verbose bool debug bool + forceUpdate bool ) // The `root` command doesn't do anything on it's own except display @@ -53,7 +54,10 @@ var rootCmd = &cobra.Command{ Long: "", Run: func(cmd *cobra.Command, args []string) { if len(args) == 0 { - cmd.Help() + err := cmd.Help() + if err != nil { + log.Error().Err(err).Msg("failed to print help") + } os.Exit(0) } }, @@ -79,11 +83,19 @@ func init() { rootCmd.PersistentFlags().StringVar(&cachePath, "cache", fmt.Sprintf("/tmp/%s/magellan/assets.db", currentUser.Username), "set the scanning result cache path") // bind viper config flags with cobra - viper.BindPFlag("concurrency", rootCmd.Flags().Lookup("concurrency")) - viper.BindPFlag("timeout", rootCmd.Flags().Lookup("timeout")) - viper.BindPFlag("verbose", rootCmd.Flags().Lookup("verbose")) - viper.BindPFlag("cache", rootCmd.Flags().Lookup("cache")) - viper.BindPFlags(rootCmd.Flags()) + checkBindFlagError(viper.BindPFlag("concurrency", rootCmd.Flags().Lookup("concurrency"))) + checkBindFlagError(viper.BindPFlag("timeout", rootCmd.Flags().Lookup("timeout"))) + checkBindFlagError(viper.BindPFlag("verbose", rootCmd.Flags().Lookup("verbose"))) + checkBindFlagError(viper.BindPFlag("debug", rootCmd.Flags().Lookup("debug"))) + checkBindFlagError(viper.BindPFlag("access-token", rootCmd.Flags().Lookup("verbose"))) + checkBindFlagError(viper.BindPFlag("cache", rootCmd.Flags().Lookup("cache"))) + checkBindFlagError(viper.BindPFlags(rootCmd.Flags())) +} + +func checkBindFlagError(err error) { + if err != nil { + log.Error().Err(err).Msg("failed to bind flag") + } } // InitializeConfig() initializes a new config object by loading it @@ -117,24 +129,21 @@ func SetDefaults() { viper.SetDefault("scan.subnets", []string{}) viper.SetDefault("scan.subnet-masks", []net.IP{}) viper.SetDefault("scan.disable-probing", false) - viper.SetDefault("collect.driver", []string{"redfish"}) + viper.SetDefault("scan.disable-cache", false) viper.SetDefault("collect.host", host) - viper.SetDefault("collect.user", "") - viper.SetDefault("collect.pass", "") + viper.SetDefault("collect.username", "") + viper.SetDefault("collect.password", "") viper.SetDefault("collect.protocol", "tcp") viper.SetDefault("collect.output", "/tmp/magellan/data/") viper.SetDefault("collect.force-update", false) - viper.SetDefault("collect.ca-cert", "") - viper.SetDefault("bmc-host", "") - viper.SetDefault("bmc-port", 443) - viper.SetDefault("user", "") - viper.SetDefault("pass", "") - viper.SetDefault("transfer-protocol", "HTTP") - viper.SetDefault("protocol", "tcp") - viper.SetDefault("firmware-url", "") - viper.SetDefault("firmware-version", "") - viper.SetDefault("component", "") - viper.SetDefault("secure-tls", false) - viper.SetDefault("status", false) + viper.SetDefault("collect.cacert", "") + viper.SetDefault("update.username", "") + viper.SetDefault("update.password", "") + viper.SetDefault("update.transfer-protocol", "https") + viper.SetDefault("update.protocol", "tcp") + viper.SetDefault("update.firmware.url", "") + viper.SetDefault("update.firmware.version", "") + viper.SetDefault("update.component", "") + viper.SetDefault("update.status", false) } diff --git a/cmd/scan.go b/cmd/scan.go index f8b4763..8aab3e8 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -184,11 +184,14 @@ func init() { scanCmd.Flags().BoolVar(&disableProbing, "disable-probing", false, "Disable probing found assets for Redfish service(s) running on BMC nodes") scanCmd.Flags().BoolVar(&disableCache, "disable-cache", false, "Disable saving found assets to a cache database specified with 'cache' flag") - viper.BindPFlag("scan.hosts", scanCmd.Flags().Lookup("host")) - viper.BindPFlag("scan.ports", scanCmd.Flags().Lookup("port")) - viper.BindPFlag("scan.subnets", scanCmd.Flags().Lookup("subnet")) - viper.BindPFlag("scan.subnet-masks", scanCmd.Flags().Lookup("subnet-mask")) - viper.BindPFlag("scan.disable-probing", scanCmd.Flags().Lookup("disable-probing")) + checkBindFlagError(viper.BindPFlag("scan.hosts", scanCmd.Flags().Lookup("host"))) + checkBindFlagError(viper.BindPFlag("scan.ports", scanCmd.Flags().Lookup("port"))) + checkBindFlagError(viper.BindPFlag("scan.scheme", scanCmd.Flags().Lookup("scheme"))) + checkBindFlagError(viper.BindPFlag("scan.protocol", scanCmd.Flags().Lookup("protocol"))) + checkBindFlagError(viper.BindPFlag("scan.subnets", scanCmd.Flags().Lookup("subnet"))) + checkBindFlagError(viper.BindPFlag("scan.subnet-masks", scanCmd.Flags().Lookup("subnet-mask"))) + checkBindFlagError(viper.BindPFlag("scan.disable-probing", scanCmd.Flags().Lookup("disable-probing"))) + checkBindFlagError(viper.BindPFlag("scan.disable-cache", scanCmd.Flags().Lookup("disable-cache"))) rootCmd.AddCommand(scanCmd) } diff --git a/cmd/update.go b/cmd/update.go index 66dcdc3..2c99de9 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -12,7 +12,6 @@ import ( var ( host string - port int firmwareUrl string firmwareVersion string component string @@ -79,25 +78,22 @@ var updateCmd = &cobra.Command{ } func init() { - updateCmd.Flags().StringVar(&username, "username", "", "set the BMC user") - updateCmd.Flags().StringVar(&password, "password", "", "set the BMC password") - updateCmd.Flags().StringVar(&transferProtocol, "transfer-protocol", "HTTP", "set the transfer protocol") - updateCmd.Flags().StringVar(&firmwareUrl, "firmware.url", "", "set the path to the firmware") - updateCmd.Flags().StringVar(&firmwareVersion, "firmware.version", "", "set the version of firmware to be installed") - updateCmd.Flags().StringVar(&component, "component", "", "set the component to upgrade") - updateCmd.Flags().BoolVar(&showStatus, "status", false, "get the status of the update") + updateCmd.Flags().StringVar(&username, "username", "", "Set the BMC user") + updateCmd.Flags().StringVar(&password, "password", "", "Set the BMC password") + updateCmd.Flags().StringVar(&transferProtocol, "transfer-protocol", "HTTP", "Set the transfer protocol") + updateCmd.Flags().StringVar(&firmwareUrl, "firmware.url", "", "Set the path to the firmware") + updateCmd.Flags().StringVar(&firmwareVersion, "firmware.version", "", "Set the version of firmware to be installed") + updateCmd.Flags().StringVar(&component, "component", "", "Set the component to upgrade (BMC|BIOS)") + updateCmd.Flags().BoolVar(&showStatus, "status", false, "Get the status of the update") - viper.BindPFlag("update.bmc.host", updateCmd.Flags().Lookup("bmc.host")) - viper.BindPFlag("update.bmc.port", updateCmd.Flags().Lookup("bmc.port")) - viper.BindPFlag("update.username", updateCmd.Flags().Lookup("username")) - viper.BindPFlag("update.password", updateCmd.Flags().Lookup("password")) - viper.BindPFlag("update.transfer-protocol", updateCmd.Flags().Lookup("transfer-protocol")) - viper.BindPFlag("update.protocol", updateCmd.Flags().Lookup("protocol")) - viper.BindPFlag("update.firmware.url", updateCmd.Flags().Lookup("firmware.url")) - viper.BindPFlag("update.firmware.version", updateCmd.Flags().Lookup("firmware.version")) - viper.BindPFlag("update.component", updateCmd.Flags().Lookup("component")) - viper.BindPFlag("update.secure-tls", updateCmd.Flags().Lookup("secure-tls")) - viper.BindPFlag("update.status", updateCmd.Flags().Lookup("status")) + checkBindFlagError(viper.BindPFlag("update.username", updateCmd.Flags().Lookup("username"))) + checkBindFlagError(viper.BindPFlag("update.password", updateCmd.Flags().Lookup("password"))) + checkBindFlagError(viper.BindPFlag("update.transfer-protocol", updateCmd.Flags().Lookup("transfer-protocol"))) + checkBindFlagError(viper.BindPFlag("update.protocol", updateCmd.Flags().Lookup("protocol"))) + checkBindFlagError(viper.BindPFlag("update.firmware.url", updateCmd.Flags().Lookup("firmware.url"))) + checkBindFlagError(viper.BindPFlag("update.firmware.version", updateCmd.Flags().Lookup("firmware.version"))) + checkBindFlagError(viper.BindPFlag("update.component", updateCmd.Flags().Lookup("component"))) + checkBindFlagError(viper.BindPFlag("update.status", updateCmd.Flags().Lookup("status"))) rootCmd.AddCommand(updateCmd) } diff --git a/internal/util/error.go b/internal/util/error.go index addca80..0141ea7 100644 --- a/internal/util/error.go +++ b/internal/util/error.go @@ -11,8 +11,8 @@ import "fmt" func FormatErrorList(errList []error) error { var err error for i, e := range errList { + // NOTE: for multi-error formating, we want to include \n here err = fmt.Errorf("\t[%d] %v\n", i, e) - i += 1 } return err } diff --git a/pkg/client/smd.go b/pkg/client/smd.go index f39d3ce..69fbf17 100644 --- a/pkg/client/smd.go +++ b/pkg/client/smd.go @@ -54,12 +54,12 @@ func (c SmdClient) Update(data HTTPBody, headers HTTPHeader) error { // Update redfish endpoint via PUT `/hsm/v2/Inventory/RedfishEndpoints` endpoint url := c.RootEndpoint("/Inventory/RedfishEndpoints/" + c.Xname) res, body, err := MakeRequest(c.Client, url, http.MethodPut, data, headers) - fmt.Printf("%v (%v)\n%s\n", url, res.Status, string(body)) if res != nil { statusOk := res.StatusCode >= 200 && res.StatusCode < 300 if !statusOk { return fmt.Errorf("failed to update redfish endpoint (returned %s)", res.Status) } + fmt.Printf("%v (%v)\n%s\n", url, res.Status, string(body)) } return err } diff --git a/tests/api_test.go b/tests/api_test.go index 4b8ddf3..c213451 100644 --- a/tests/api_test.go +++ b/tests/api_test.go @@ -11,6 +11,7 @@ import ( "testing" magellan "github.com/OpenCHAMI/magellan/internal" + "github.com/rs/zerolog/log" ) var ( @@ -43,7 +44,10 @@ func TestScanAndCollect(t *testing.T) { } // do a collect on the emulator cluster to collect Redfish info - magellan.CollectInventory(&results, &magellan.CollectParams{}) + err := magellan.CollectInventory(&results, &magellan.CollectParams{}) + if err != nil { + log.Error().Err(err).Msg("failed to collect inventory") + } } func TestCrawlCommand(t *testing.T) { diff --git a/tests/compatibility_test.go b/tests/compatibility_test.go index b879536..86e3a4f 100644 --- a/tests/compatibility_test.go +++ b/tests/compatibility_test.go @@ -15,6 +15,7 @@ import ( "github.com/OpenCHAMI/magellan/pkg/client" "github.com/OpenCHAMI/magellan/pkg/crawler" + "github.com/rs/zerolog/log" ) var ( @@ -26,7 +27,7 @@ var ( // Simple test to fetch the base Redfish URL and assert a 200 OK response. func TestRedfishV1Availability(t *testing.T) { var ( - url = fmt.Sprintf("%s/redfish/v1", host) + url = fmt.Sprintf("%s/redfish/v1", *host) body = []byte{} headers = map[string]string{} ) @@ -55,12 +56,16 @@ func TestRedfishV1Availability(t *testing.T) { // Simple test to ensure an expected Redfish version minimum requirement. func TestRedfishVersion(t *testing.T) { var ( - url = fmt.Sprintf("%s/redfish/v1", host) - body = []byte{} - headers = map[string]string{} + url string = fmt.Sprintf("%s/redfish/v1", *host) + body client.HTTPBody = []byte{} + headers client.HTTPHeader = map[string]string{} + err error ) - client.MakeRequest(nil, url, http.MethodGet, body, headers) + _, _, err = client.MakeRequest(nil, url, http.MethodGet, body, headers) + if err != nil { + log.Error().Err(err).Msg("failed to make request") + } } // Crawls a BMC node and checks that we're able to query certain properties From 15de7c16c09d628e58e97aeafb4d76249c907bd1 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 13 Aug 2024 13:40:26 -0600 Subject: [PATCH 56/67] Fixed viper flag binding in collect cmd --- cmd/collect.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/collect.go b/cmd/collect.go index 74d3521..f33a52c 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -86,14 +86,14 @@ func init() { collectCmd.MarkFlagsRequiredTogether("username", "password") // bind flags to config properties - checkBindFlagError(viper.BindPFlag("collect.host", collectCmd.Flags().Lookup("collect.host"))) - checkBindFlagError(viper.BindPFlag("collect.username", collectCmd.Flags().Lookup("collect.username"))) - checkBindFlagError(viper.BindPFlag("collect.password", collectCmd.Flags().Lookup("collect.password"))) - checkBindFlagError(viper.BindPFlag("collect.scheme", collectCmd.Flags().Lookup("collect.scheme"))) - checkBindFlagError(viper.BindPFlag("collect.protocol", collectCmd.Flags().Lookup("collect.protocol"))) - checkBindFlagError(viper.BindPFlag("collect.output", collectCmd.Flags().Lookup("collect.output"))) - checkBindFlagError(viper.BindPFlag("collect.force-update", collectCmd.Flags().Lookup("collect.force-update"))) - checkBindFlagError(viper.BindPFlag("collect.cacert", collectCmd.Flags().Lookup("collect.cacert"))) + checkBindFlagError(viper.BindPFlag("collect.host", collectCmd.Flags().Lookup("host"))) + checkBindFlagError(viper.BindPFlag("collect.username", collectCmd.Flags().Lookup("username"))) + checkBindFlagError(viper.BindPFlag("collect.password", collectCmd.Flags().Lookup("password"))) + checkBindFlagError(viper.BindPFlag("collect.scheme", collectCmd.Flags().Lookup("scheme"))) + checkBindFlagError(viper.BindPFlag("collect.protocol", collectCmd.Flags().Lookup("protocol"))) + checkBindFlagError(viper.BindPFlag("collect.output", collectCmd.Flags().Lookup("output"))) + checkBindFlagError(viper.BindPFlag("collect.force-update", collectCmd.Flags().Lookup("force-update"))) + checkBindFlagError(viper.BindPFlag("collect.cacert", collectCmd.Flags().Lookup("cacert"))) checkBindFlagError(viper.BindPFlags(collectCmd.Flags())) rootCmd.AddCommand(collectCmd) From 809aa91ec6191f334dd225950a532ab95ff01702 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 13 Aug 2024 14:11:14 -0600 Subject: [PATCH 57/67] Changed transfer-protocol flag to scheme to match other commands --- cmd/update.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/update.go b/cmd/update.go index 2c99de9..9c30a31 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -27,8 +27,8 @@ var updateCmd = &cobra.Command{ Short: "Update BMC node firmware", Long: "Perform an firmware update using Redfish by providing a remote firmware URL and component.\n\n" + "Examples:\n" + - " magellan update --bmc.host 172.16.0.108 --bmc.port 443 --username bmc_username --password bmc_password --firmware-url http://172.16.0.200:8005/firmware/bios/image.RBU --component BIOS\n" + - " magellan update --status --bmc.host 172.16.0.108 --bmc.port 443 --username bmc_username --password bmc_password", + " magellan update 172.16.0.108:443 --username bmc_username --password bmc_password --firmware-url http://172.16.0.200:8005/firmware/bios/image.RBU --component BIOS\n" + + " magellan update 172.16.0.108:443 --status --username bmc_username --password bmc_password", Run: func(cmd *cobra.Command, args []string) { // check that we have at least one host if len(args) <= 0 { @@ -80,7 +80,7 @@ var updateCmd = &cobra.Command{ func init() { updateCmd.Flags().StringVar(&username, "username", "", "Set the BMC user") updateCmd.Flags().StringVar(&password, "password", "", "Set the BMC password") - updateCmd.Flags().StringVar(&transferProtocol, "transfer-protocol", "HTTP", "Set the transfer protocol") + updateCmd.Flags().StringVar(&transferProtocol, "scheme", "https", "Set the transfer protocol") updateCmd.Flags().StringVar(&firmwareUrl, "firmware.url", "", "Set the path to the firmware") updateCmd.Flags().StringVar(&firmwareVersion, "firmware.version", "", "Set the version of firmware to be installed") updateCmd.Flags().StringVar(&component, "component", "", "Set the component to upgrade (BMC|BIOS)") From 7285492815c41e39c47fcc50bb78ab9465392b88 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 13 Aug 2024 14:44:11 -0600 Subject: [PATCH 58/67] Fixed root persistent flags not binding correctly --- cmd/crawl.go | 5 +++++ cmd/root.go | 13 ++++++------- cmd/update.go | 3 +-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cmd/crawl.go b/cmd/crawl.go index 4dab37d..c7f60bf 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -9,6 +9,7 @@ import ( "github.com/OpenCHAMI/magellan/pkg/crawler" "github.com/spf13/cobra" + "github.com/spf13/viper" ) // The `crawl` command walks a collection of Redfish endpoints to collect @@ -66,5 +67,9 @@ func init() { crawlCmd.Flags().StringP("password", "p", "", "Set the password for the BMC") crawlCmd.Flags().BoolP("insecure", "i", false, "Ignore SSL errors") + viper.BindPFlag("crawl.username", crawlCmd.Flags().Lookup("username")) + viper.BindPFlag("crawl.password", crawlCmd.Flags().Lookup("password")) + viper.BindPFlag("crawl.insecure", crawlCmd.Flags().Lookup("insecure")) + rootCmd.AddCommand(crawlCmd) } diff --git a/cmd/root.go b/cmd/root.go index e9a497f..b21e22d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -83,13 +83,12 @@ func init() { rootCmd.PersistentFlags().StringVar(&cachePath, "cache", fmt.Sprintf("/tmp/%s/magellan/assets.db", currentUser.Username), "set the scanning result cache path") // bind viper config flags with cobra - checkBindFlagError(viper.BindPFlag("concurrency", rootCmd.Flags().Lookup("concurrency"))) - checkBindFlagError(viper.BindPFlag("timeout", rootCmd.Flags().Lookup("timeout"))) - checkBindFlagError(viper.BindPFlag("verbose", rootCmd.Flags().Lookup("verbose"))) - checkBindFlagError(viper.BindPFlag("debug", rootCmd.Flags().Lookup("debug"))) - checkBindFlagError(viper.BindPFlag("access-token", rootCmd.Flags().Lookup("verbose"))) - checkBindFlagError(viper.BindPFlag("cache", rootCmd.Flags().Lookup("cache"))) - checkBindFlagError(viper.BindPFlags(rootCmd.Flags())) + checkBindFlagError(viper.BindPFlag("concurrency", rootCmd.PersistentFlags().Lookup("concurrency"))) + checkBindFlagError(viper.BindPFlag("timeout", rootCmd.PersistentFlags().Lookup("timeout"))) + checkBindFlagError(viper.BindPFlag("verbose", rootCmd.PersistentFlags().Lookup("verbose"))) + checkBindFlagError(viper.BindPFlag("debug", rootCmd.PersistentFlags().Lookup("debug"))) + checkBindFlagError(viper.BindPFlag("access-token", rootCmd.PersistentFlags().Lookup("verbose"))) + checkBindFlagError(viper.BindPFlag("cache", rootCmd.PersistentFlags().Lookup("cache"))) } func checkBindFlagError(err error) { diff --git a/cmd/update.go b/cmd/update.go index 9c30a31..c4e04c6 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -88,8 +88,7 @@ func init() { checkBindFlagError(viper.BindPFlag("update.username", updateCmd.Flags().Lookup("username"))) checkBindFlagError(viper.BindPFlag("update.password", updateCmd.Flags().Lookup("password"))) - checkBindFlagError(viper.BindPFlag("update.transfer-protocol", updateCmd.Flags().Lookup("transfer-protocol"))) - checkBindFlagError(viper.BindPFlag("update.protocol", updateCmd.Flags().Lookup("protocol"))) + checkBindFlagError(viper.BindPFlag("update.scheme", updateCmd.Flags().Lookup("scheme"))) checkBindFlagError(viper.BindPFlag("update.firmware.url", updateCmd.Flags().Lookup("firmware.url"))) checkBindFlagError(viper.BindPFlag("update.firmware.version", updateCmd.Flags().Lookup("firmware.version"))) checkBindFlagError(viper.BindPFlag("update.component", updateCmd.Flags().Lookup("component"))) From dc07b5396b6c688f20276d1865791c80188e4f81 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 13 Aug 2024 16:48:11 -0600 Subject: [PATCH 59/67] Changed saving host to include scheme for collect --- internal/collect.go | 2 +- internal/scan.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/collect.go b/internal/collect.go index 4715ec3..524c5c6 100644 --- a/internal/collect.go +++ b/internal/collect.go @@ -58,7 +58,7 @@ func CollectInventory(assets *[]RemoteAsset, params *CollectParams) error { done = make(chan struct{}, params.Concurrency+1) chanAssets = make(chan RemoteAsset, params.Concurrency+1) outputPath = path.Clean(params.OutputPath) - smdClient = client.NewClient[client.SmdClient]( + smdClient = client.NewClient( client.WithSecureTLS[client.SmdClient](params.CaCertPath), ) ) diff --git a/internal/scan.go b/internal/scan.go index 736329d..06ad174 100644 --- a/internal/scan.go +++ b/internal/scan.go @@ -83,7 +83,7 @@ func ScanForAssets(params *ScanParams) []RemoteAsset { if !params.DisableProbing { assetsToAdd := []RemoteAsset{} for _, foundAsset := range foundAssets { - url := fmt.Sprintf("%s://%s:%d/redfish/v1/", params.Scheme, foundAsset.Host, foundAsset.Port) + url := fmt.Sprintf("%s:%d/redfish/v1/", foundAsset.Host, foundAsset.Port) res, _, err := client.MakeRequest(nil, url, http.MethodGet, nil, nil) if err != nil || res == nil { if params.Verbose { From bb49129d3117245fe4fb60ece76fe11a8817affb Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 13 Aug 2024 16:48:34 -0600 Subject: [PATCH 60/67] Changed showing target host to use debug instead of verbose flag --- cmd/scan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/scan.go b/cmd/scan.go index 8aab3e8..b0ed529 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -144,7 +144,7 @@ var scanCmd = &cobra.Command{ Debug: debug, }) - if len(foundAssets) > 0 && verbose { + if len(foundAssets) > 0 && debug { log.Info().Any("assets", foundAssets).Msgf("found assets from scan") } From 2b291f811af347c8238f0d146fe6d9da75b8e4b9 Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 13 Aug 2024 17:01:21 -0600 Subject: [PATCH 61/67] Fixed scan not probing the host correctly --- internal/scan.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/scan.go b/internal/scan.go index 06ad174..455749d 100644 --- a/internal/scan.go +++ b/internal/scan.go @@ -75,7 +75,7 @@ func ScanForAssets(params *ScanParams) []RemoteAsset { // if we failed to connect, exit from the function if err != nil { if params.Verbose { - log.Debug().Err(err).Msgf("failed to connect to host (%s)", host) + log.Debug().Err(err).Msgf("failed to connect to host") } wg.Done() return @@ -193,7 +193,7 @@ func rawConnect(address string, protocol string, timeoutSeconds int, keepOpenOnl timeoutDuration = time.Second * time.Duration(timeoutSeconds) assets []RemoteAsset asset = RemoteAsset{ - Host: uri.Hostname(), + Host: fmt.Sprintf("%s://%s", uri.Scheme, uri.Hostname()), Port: port, Protocol: protocol, State: false, From 42dbfeb4f98d7662c54ae7cade245ba2d654a25b Mon Sep 17 00:00:00 2001 From: David Allen Date: Tue, 13 Aug 2024 17:45:08 -0600 Subject: [PATCH 62/67] Fixed lint errors --- cmd/crawl.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/crawl.go b/cmd/crawl.go index c7f60bf..ed70e5b 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -67,9 +67,9 @@ func init() { crawlCmd.Flags().StringP("password", "p", "", "Set the password for the BMC") crawlCmd.Flags().BoolP("insecure", "i", false, "Ignore SSL errors") - viper.BindPFlag("crawl.username", crawlCmd.Flags().Lookup("username")) - viper.BindPFlag("crawl.password", crawlCmd.Flags().Lookup("password")) - viper.BindPFlag("crawl.insecure", crawlCmd.Flags().Lookup("insecure")) + checkBindFlagError(viper.BindPFlag("crawl.username", crawlCmd.Flags().Lookup("username"))) + checkBindFlagError(viper.BindPFlag("crawl.password", crawlCmd.Flags().Lookup("password"))) + checkBindFlagError(viper.BindPFlag("crawl.insecure", crawlCmd.Flags().Lookup("insecure"))) rootCmd.AddCommand(crawlCmd) } From 4444a1d2995bd8e9f14b190aeea4a6e52a94a156 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 14 Aug 2024 10:56:20 -0600 Subject: [PATCH 63/67] Updated go deps --- go.mod | 12 ++---------- go.sum | 13 ------------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 4816d47..e00981d 100644 --- a/go.mod +++ b/go.mod @@ -4,29 +4,24 @@ go 1.21 require ( github.com/Cray-HPE/hms-xname v1.3.0 - - github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18 - github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 github.com/go-chi/chi/v5 v5.1.0 github.com/jmoiron/sqlx v1.4.0 github.com/lestrrat-go/jwx v1.2.29 github.com/mattn/go-sqlite3 v1.14.22 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c - github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 github.com/stmcginnis/gofish v0.19.0 golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 ) +require github.com/rs/zerolog v1.33.0 + require ( github.com/google/go-cmp v0.6.0 // indirect - github.com/lestrrat-go/httprc v1.0.4 // indirect - github.com/lestrrat-go/jwx/v2 v2.0.20 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/segmentio/asm v1.2.0 // indirect ) @@ -46,7 +41,6 @@ require ( github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/rs/zerolog v1.33.0 github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -56,10 +50,8 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.22.0 // indirect golang.org/x/sys v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index b5325f3..befbed6 100644 --- a/go.sum +++ b/go.sum @@ -2,9 +2,6 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/Cray-HPE/hms-xname v1.3.0 h1:DQmetMniubqcaL6Cxarz9+7KFfWGSEizIhfPHIgC3Gw= github.com/Cray-HPE/hms-xname v1.3.0/go.mod h1:XKdjQSzoTps5KDOE8yWojBTAWASGaS6LfRrVDxwTQO8= -github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18 h1:oBPtXp9RVm9lk5zTmDLf+Vh21yDHpulBxUqGJQjwQCk= -github.com/OpenCHAMI/jwtauth/v5 v5.0.0-20240321222802-e6cb468a2a18/go.mod h1:ggNHWgLfW/WRXcE8ZZC4S7UwHif16HVmyowOCWdNSN8= - github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= @@ -48,14 +45,10 @@ github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= -github.com/lestrrat-go/httprc v1.0.4 h1:bAZymwoZQb+Oq8MEbyipag7iSq6YIga8Wj6GOiJGdI8= -github.com/lestrrat-go/httprc v1.0.4/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= github.com/lestrrat-go/jwx v1.2.29 h1:QT0utmUJ4/12rmsVQrJ3u55bycPkKqGYuGT4tyRhxSQ= github.com/lestrrat-go/jwx v1.2.29/go.mod h1:hU8k2l6WF0ncx20uQdOmik/Gjg6E3/wIRtXSNFeZuB8= -github.com/lestrrat-go/jwx/v2 v2.0.20 h1:sAgXuWS/t8ykxS9Bi2Qtn5Qhpakw1wrcjxChudjolCc= -github.com/lestrrat-go/jwx/v2 v2.0.20/go.mod h1:UlCSmKqw+agm5BsOBfEAbTvKsEApaGNqHAEUTv5PJC4= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= @@ -94,10 +87,6 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= -github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= @@ -118,7 +107,6 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -154,7 +142,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From 4597f63d1264b8d336510d9e5dc6f8f0f2226fdc Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 14 Aug 2024 10:57:30 -0600 Subject: [PATCH 64/67] Fixed issue with host string and added internal url package --- cmd/collect.go | 8 +-- cmd/crawl.go | 14 ++--- cmd/scan.go | 6 +-- internal/cache/cache.go | 1 + internal/scan.go | 3 +- internal/url/url.go | 116 ++++++++++++++++++++++++++++++++++++++++ pkg/client/net.go | 98 --------------------------------- 7 files changed, 131 insertions(+), 115 deletions(-) create mode 100644 internal/url/url.go diff --git a/cmd/collect.go b/cmd/collect.go index f33a52c..1f3288a 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -3,10 +3,10 @@ package cmd import ( "fmt" "os/user" - "strings" magellan "github.com/OpenCHAMI/magellan/internal" "github.com/OpenCHAMI/magellan/internal/cache/sqlite" + urlx "github.com/OpenCHAMI/magellan/internal/url" "github.com/OpenCHAMI/magellan/pkg/auth" "github.com/cznic/mathutil" "github.com/rs/zerolog/log" @@ -33,8 +33,10 @@ var collectCmd = &cobra.Command{ } // URL sanitanization for host argument - host = strings.TrimSuffix(host, "/") - host = strings.ReplaceAll(host, "//", "/") + host, err = urlx.Sanitize(host) + if err != nil { + log.Error().Err(err).Msg("failed to sanitize host") + } // try to load access token either from env var, file, or config if var not set if accessToken == "" { diff --git a/cmd/crawl.go b/cmd/crawl.go index ed70e5b..8069c9b 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -4,9 +4,8 @@ import ( "encoding/json" "fmt" "log" - "net/url" - "strings" + urlx "github.com/OpenCHAMI/magellan/internal/url" "github.com/OpenCHAMI/magellan/pkg/crawler" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -25,19 +24,14 @@ var crawlCmd = &cobra.Command{ " magellan crawl https://bmc.example.com -i -u username -p password", Args: func(cmd *cobra.Command, args []string) error { // Validate that the only argument is a valid URI + var err error if err := cobra.ExactArgs(1)(cmd, args); err != nil { return err } - parsedURI, err := url.ParseRequestURI(args[0]) + args[0], err = urlx.Sanitize(args[0]) if err != nil { - return fmt.Errorf("invalid URI specified: %s", args[0]) + return fmt.Errorf("failed to sanitize URI: %w", err) } - // Remove any trailing slashes - parsedURI.Path = strings.TrimSuffix(parsedURI.Path, "/") - // Collapse any doubled slashes - parsedURI.Path = strings.ReplaceAll(parsedURI.Path, "//", "/") - // Update the URI in the args slice - args[0] = parsedURI.String() return nil }, Run: func(cmd *cobra.Command, args []string) { diff --git a/cmd/scan.go b/cmd/scan.go index b0ed529..fedc691 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -9,9 +9,9 @@ import ( magellan "github.com/OpenCHAMI/magellan/internal" "github.com/OpenCHAMI/magellan/internal/cache/sqlite" - "github.com/OpenCHAMI/magellan/pkg/client" "github.com/rs/zerolog/log" + urlx "github.com/OpenCHAMI/magellan/internal/url" "github.com/cznic/mathutil" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -72,8 +72,8 @@ var scanCmd = &cobra.Command{ } // format and combine flag and positional args - targetHosts = append(targetHosts, client.FormatHostUrls(args, ports, scheme, verbose)...) - targetHosts = append(targetHosts, client.FormatHostUrls(hosts, ports, scheme, verbose)...) + targetHosts = append(targetHosts, urlx.FormatHosts(args, ports, scheme, verbose)...) + targetHosts = append(targetHosts, urlx.FormatHosts(hosts, ports, scheme, verbose)...) // add more hosts specified with `--subnet` flag if debug { diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 96513de..7b4eea1 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -4,6 +4,7 @@ import ( "database/sql/driver" ) +// TODO: implement extendable storage drivers using cache interface (sqlite, duckdb, etc.) type Cache[T any] interface { CreateIfNotExists(path string) (driver.Connector, error) Insert(path string, data ...T) error diff --git a/internal/scan.go b/internal/scan.go index 455749d..a88116d 100644 --- a/internal/scan.go +++ b/internal/scan.go @@ -10,6 +10,7 @@ import ( "sync" "time" + urlx "github.com/OpenCHAMI/magellan/internal/url" "github.com/OpenCHAMI/magellan/pkg/client" "github.com/rs/zerolog/log" ) @@ -164,7 +165,7 @@ func GenerateHostsWithSubnet(subnet string, subnetMask *net.IPMask, additionalPo // generate new IPs from subnet and format to full URL subnetIps := generateIPsWithSubnet(&subnetIp, subnetMask) - return client.FormatIPUrls(subnetIps, additionalPorts, defaultScheme, false) + return urlx.FormatIPs(subnetIps, additionalPorts, defaultScheme, false) } // GetDefaultPorts() returns a list of default ports. The only reason to have diff --git a/internal/url/url.go b/internal/url/url.go new file mode 100644 index 0000000..ba8ad3e --- /dev/null +++ b/internal/url/url.go @@ -0,0 +1,116 @@ +package url + +import ( + "fmt" + "net/url" + "strings" + + "github.com/rs/zerolog/log" +) + +func Sanitize(uri string) (string, error) { + // URL sanitanization for host argument + parsedURI, err := url.ParseRequestURI(uri) + if err != nil { + return "", fmt.Errorf("failed to parse URI: %w", err) + } + // Remove any trailing slashes + parsedURI.Path = strings.TrimSuffix(uri, "/") + // Collapse any doubled slashes + parsedURI.Path = strings.ReplaceAll(uri, "//", "/") + return parsedURI.String(), nil +} + +// FormatHosts() takes a list of hosts and ports and builds full URLs in the +// form of scheme://host:port. If no scheme is provided, it will use "https" by +// default. +// +// Returns a 2D string slice where each slice contains URL host strings for each +// port. The intention is to have all of the URLs for a single host combined into +// a single slice to initiate one goroutine per host, but making request to multiple +// ports. +func FormatHosts(hosts []string, ports []int, scheme string, verbose bool) [][]string { + // format each positional arg as a complete URL + var formattedHosts [][]string + for _, host := range hosts { + uri, err := url.ParseRequestURI(host) + if err != nil { + if verbose { + log.Warn().Msgf("invalid URI parsed: %s", host) + } + continue + } + + // check if scheme is set, if not set it with flag or default value ('https' if flag is not set) + if uri.Scheme == "" { + if scheme != "" { + uri.Scheme = scheme + } else { + // hardcoded assumption + uri.Scheme = "https" + } + } + + // tidy up slashes and update arg with new value + uri.Path = strings.TrimSuffix(uri.Path, "/") + uri.Path = strings.ReplaceAll(uri.Path, "//", "/") + + // for hosts with unspecified ports, add ports to scan from flag + if uri.Port() == "" { + var tmp []string + for _, port := range ports { + uri.Host += fmt.Sprintf(":%d", port) + tmp = append(tmp, uri.String()) + } + formattedHosts = append(formattedHosts, tmp) + } else { + formattedHosts = append(formattedHosts, []string{uri.String()}) + } + + } + return formattedHosts +} + +// FormatIPs() takes a list of IP addresses and ports and builds full URLs in the +// form of scheme://host:port. If no scheme is provided, it will use "https" by +// default. +// +// Returns a 2D string slice where each slice contains URL host strings for each +// port. The intention is to have all of the URLs for a single host combined into +// a single slice to initiate one goroutine per host, but making request to multiple +// ports. +func FormatIPs(ips []string, ports []int, scheme string, verbose bool) [][]string { + // format each positional arg as a complete URL + var formattedHosts [][]string + for _, ip := range ips { + if scheme == "" { + scheme = "https" + } + // make an entirely new object since we're expecting just IPs + uri := &url.URL{ + Scheme: scheme, + Host: ip, + } + + // tidy up slashes and update arg with new value + uri.Path = strings.ReplaceAll(uri.Path, "//", "/") + uri.Path = strings.TrimSuffix(uri.Path, "/") + + // for hosts with unspecified ports, add ports to scan from flag + if uri.Port() == "" { + if len(ports) == 0 { + ports = append(ports, 443) + } + var tmp []string + for _, port := range ports { + uri.Host += fmt.Sprintf(":%d", port) + tmp = append(tmp, uri.String()) + } + formattedHosts = append(formattedHosts, tmp) + } else { + formattedHosts = append(formattedHosts, []string{uri.String()}) + } + + } + return formattedHosts +} diff --git a/pkg/client/net.go b/pkg/client/net.go index af69a53..9a41d77 100644 --- a/pkg/client/net.go +++ b/pkg/client/net.go @@ -7,10 +7,6 @@ import ( "io" "net" "net/http" - "net/url" - "strings" - - "github.com/rs/zerolog/log" ) // HTTP aliases for readibility @@ -82,97 +78,3 @@ func MakeRequest(client *http.Client, url string, httpMethod string, body HTTPBo } return res, b, err } - -// FormatHostUrls() takes a list of hosts and ports and builds full URLs in the -// form of scheme://host:port. If no scheme is provided, it will use "https" by -// default. -// -// Returns a 2D string slice where each slice contains URL host strings for each -// port. The intention is to have all of the URLs for a single host combined into -// a single slice to initiate one goroutine per host, but making request to multiple -// ports. -func FormatHostUrls(hosts []string, ports []int, scheme string, verbose bool) [][]string { - // format each positional arg as a complete URL - var formattedHosts [][]string - for _, host := range hosts { - uri, err := url.ParseRequestURI(host) - if err != nil { - if verbose { - log.Warn().Msgf("invalid URI parsed: %s", host) - } - continue - } - - // check if scheme is set, if not set it with flag or default value ('https' if flag is not set) - if uri.Scheme == "" { - if scheme != "" { - uri.Scheme = scheme - } else { - // hardcoded assumption - uri.Scheme = "https" - } - } - - // tidy up slashes and update arg with new value - uri.Path = strings.TrimSuffix(uri.Path, "/") - uri.Path = strings.ReplaceAll(uri.Path, "//", "/") - - // for hosts with unspecified ports, add ports to scan from flag - if uri.Port() == "" { - var tmp []string - for _, port := range ports { - uri.Host += fmt.Sprintf(":%d", port) - tmp = append(tmp, uri.String()) - } - formattedHosts = append(formattedHosts, tmp) - } else { - formattedHosts = append(formattedHosts, []string{uri.String()}) - } - - } - return formattedHosts -} - -// FormatIPUrls() takes a list of IP addresses and ports and builds full URLs in the -// form of scheme://host:port. If no scheme is provided, it will use "https" by -// default. -// -// Returns a 2D string slice where each slice contains URL host strings for each -// port. The intention is to have all of the URLs for a single host combined into -// a single slice to initiate one goroutine per host, but making request to multiple -// ports. -func FormatIPUrls(ips []string, ports []int, scheme string, verbose bool) [][]string { - // format each positional arg as a complete URL - var formattedHosts [][]string - for _, ip := range ips { - if scheme == "" { - scheme = "https" - } - // make an entirely new object since we're expecting just IPs - uri := &url.URL{ - Scheme: scheme, - Host: ip, - } - - // tidy up slashes and update arg with new value - uri.Path = strings.ReplaceAll(uri.Path, "//", "/") - uri.Path = strings.TrimSuffix(uri.Path, "/") - - // for hosts with unspecified ports, add ports to scan from flag - if uri.Port() == "" { - if len(ports) == 0 { - ports = append(ports, 443) - } - var tmp []string - for _, port := range ports { - uri.Host += fmt.Sprintf(":%d", port) - tmp = append(tmp, uri.String()) - } - formattedHosts = append(formattedHosts, tmp) - } else { - formattedHosts = append(formattedHosts, []string{uri.String()}) - } - - } - return formattedHosts -} From 046a8339410ac96b3bdd76901ac9dd16b66c62e3 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 14 Aug 2024 13:33:12 -0600 Subject: [PATCH 65/67] Fixed passing the correct argument in Sanitize() --- internal/url/url.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/url/url.go b/internal/url/url.go index ba8ad3e..38f0eed 100644 --- a/internal/url/url.go +++ b/internal/url/url.go @@ -15,9 +15,9 @@ func Sanitize(uri string) (string, error) { return "", fmt.Errorf("failed to parse URI: %w", err) } // Remove any trailing slashes - parsedURI.Path = strings.TrimSuffix(uri, "/") + parsedURI.Path = strings.TrimSuffix(parsedURI.Path, "/") // Collapse any doubled slashes - parsedURI.Path = strings.ReplaceAll(uri, "//", "/") + parsedURI.Path = strings.ReplaceAll(parsedURI.Path, "//", "/") return parsedURI.String(), nil } From 5b1e3e732eb3c230d8a1ea6f98e699b375b9e922 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 14 Aug 2024 14:34:16 -0600 Subject: [PATCH 66/67] Changed firmware.* back to firmware-* --- cmd/update.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/update.go b/cmd/update.go index c4e04c6..d5f9a50 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -81,16 +81,16 @@ func init() { updateCmd.Flags().StringVar(&username, "username", "", "Set the BMC user") updateCmd.Flags().StringVar(&password, "password", "", "Set the BMC password") updateCmd.Flags().StringVar(&transferProtocol, "scheme", "https", "Set the transfer protocol") - updateCmd.Flags().StringVar(&firmwareUrl, "firmware.url", "", "Set the path to the firmware") - updateCmd.Flags().StringVar(&firmwareVersion, "firmware.version", "", "Set the version of firmware to be installed") + updateCmd.Flags().StringVar(&firmwareUrl, "firmware-url", "", "Set the path to the firmware") + updateCmd.Flags().StringVar(&firmwareVersion, "firmware-version", "", "Set the version of firmware to be installed") updateCmd.Flags().StringVar(&component, "component", "", "Set the component to upgrade (BMC|BIOS)") updateCmd.Flags().BoolVar(&showStatus, "status", false, "Get the status of the update") checkBindFlagError(viper.BindPFlag("update.username", updateCmd.Flags().Lookup("username"))) checkBindFlagError(viper.BindPFlag("update.password", updateCmd.Flags().Lookup("password"))) checkBindFlagError(viper.BindPFlag("update.scheme", updateCmd.Flags().Lookup("scheme"))) - checkBindFlagError(viper.BindPFlag("update.firmware.url", updateCmd.Flags().Lookup("firmware.url"))) - checkBindFlagError(viper.BindPFlag("update.firmware.version", updateCmd.Flags().Lookup("firmware.version"))) + checkBindFlagError(viper.BindPFlag("update.firmware-url", updateCmd.Flags().Lookup("firmware-url"))) + checkBindFlagError(viper.BindPFlag("update.firmware-version", updateCmd.Flags().Lookup("firmware-version"))) checkBindFlagError(viper.BindPFlag("update.component", updateCmd.Flags().Lookup("component"))) checkBindFlagError(viper.BindPFlag("update.status", updateCmd.Flags().Lookup("status"))) From db44c5111361c2693652bca05b0b2b31ebcc12f0 Mon Sep 17 00:00:00 2001 From: David Allen Date: Wed, 14 Aug 2024 14:44:22 -0600 Subject: [PATCH 67/67] Added disclaimer about incompatibility with SMD --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 79527bb..a9e9cc1 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ The `magellan` CLI tool is a Redfish-based, board management controller (BMC) discovery tool designed to scan networks and is written in Go. The tool collects information from BMC nodes using the provided Redfish RESTful API with [`gofish`](https://github.com/stmcginnis/gofish) and loads the queried data into an [SMD](https://github.com/OpenCHAMI/smd/tree/master) instance. The tool strives to be more flexible by implementing multiple methods of discovery to work for a wider range of systems (WIP) and is capable of using independently of other tools or services. +**Note: `magellan` v0.1.0 is incompatible with SMD v2.15.3 and earlier.** + ## Main Features The `magellan` tool comes packed with a handleful of features for doing discovery, such as: