From f08953584fae3cc814dd564d6e41f015564c2449 Mon Sep 17 00:00:00 2001 From: "R.I.Pienaar" Date: Mon, 16 Dec 2024 13:49:23 +0100 Subject: [PATCH] New server report health Signed-off-by: R.I.Pienaar --- cli/server_report_command.go | 104 +++++++++++++++++++++++++++++++++- cli/server_request_command.go | 61 ++++++++++---------- 2 files changed, 132 insertions(+), 33 deletions(-) diff --git a/cli/server_report_command.go b/cli/server_report_command.go index 3a534851..f548fff6 100644 --- a/cli/server_report_command.go +++ b/cli/server_report_command.go @@ -51,6 +51,10 @@ type SrvReportCmd struct { filterReason string skipDiscoverClusterSize bool gatewayName string + jsEnabled bool + jsServerOnly bool + stream string + consumer string } type srvReportAccountInfo struct { @@ -106,6 +110,13 @@ func configureServerReportCommand(srv *fisk.CmdClause) { gateways.Flag("filter-name", "Limits responses to a certain name").StringVar(&c.gatewayName) gateways.Flag("sort", "Sorts by a specific property (server,cluster)").Default("cluster").EnumVar(&c.sort, "server", "cluster") + health := report.Command("health", "Report on Server health").Action(c.reportHealth) + health.Flag("js-enabled", "Checks that JetStream should be enabled on all servers").Short('J').BoolVar(&c.jsEnabled) + health.Flag("server-only", "Restricts the health check to the JetStream server only, do not check streams and consumers").Short('S').BoolVar(&c.jsServerOnly) + health.Flag("account", "Check only a specific Account").StringVar(&c.account) + health.Flag("stream", "Check only a specific Stream").StringVar(&c.stream) + health.Flag("consumer", "Check only a specific Consumer").StringVar(&c.consumer) + jsz := report.Command("jetstream", "Report on JetStream activity").Alias("jsz").Alias("js").Action(c.reportJetStream) jsz.Arg("limit", "Limit the responses to a certain amount of servers").IntVar(&c.waitFor) addFilterOpts(jsz) @@ -135,13 +146,99 @@ func (c *SrvReportCmd) parseRtt(rtt string, crit time.Duration) string { return color.RedString(f(d)) } +func (c *SrvReportCmd) reportHealth(_ *fisk.ParseContext) error { + nc, _, err := prepareHelper("", natsOpts()...) + if err != nil { + return err + } + + req := server.HealthzEventOptions{ + HealthzOptions: server.HealthzOptions{ + JSEnabledOnly: c.jsEnabled, + JSServerOnly: c.jsServerOnly, + Account: c.account, + Stream: c.stream, + Consumer: c.consumer, + Details: true, + }, + EventFilterOptions: c.reqFilter(), + } + results, err := doReq(req, "$SYS.REQ.SERVER.PING.HEALTHZ", c.waitFor, nc) + if err != nil { + return err + } + + var servers []server.ServerAPIHealthzResponse + for _, result := range results { + s := &server.ServerAPIHealthzResponse{} + err := json.Unmarshal(result, s) + if err != nil { + return err + } + + if s.Error != nil { + return fmt.Errorf("%v", s.Error.Error()) + } + + servers = append(servers, *s) + } + + sort.Slice(servers, func(i, j int) bool { + return c.boolReverse(servers[i].Server.Name < servers[j].Server.Name) + }) + + tbl := iu.NewTableWriter(opts(), "Health Report") + tbl.AddHeaders("Server", "Cluster", "Domain", "Status", "Type", "Error") + + for _, srv := range servers { + tbl.AddRow( + srv.Server.Name, + srv.Server.Cluster, + srv.Server.Domain, + fmt.Sprintf("%s (%d)", srv.Data.Status, srv.Data.StatusCode), + ) + + ecnt := len(srv.Data.Errors) + if ecnt == 0 { + continue + } + + show := ecnt + if ecnt > 10 { + show = 9 + } + + for _, errStatus := range srv.Data.Errors[0:show] { + tbl.AddRow( + "", "", "", "", + errStatus.Type.String(), + errStatus.Error, + ) + } + if show != ecnt { + tbl.AddRow("", "", "", fmt.Sprintf("%d more errors", ecnt-show)) + } + } + + fmt.Println(tbl.Render()) + + return nil +} + func (c *SrvReportCmd) reportGateway(_ *fisk.ParseContext) error { nc, _, err := prepareHelper("", natsOpts()...) if err != nil { return err } - req := &server.GatewayzOptions{Name: c.gatewayName, Accounts: true} + req := &server.GatewayzEventOptions{ + EventFilterOptions: c.reqFilter(), + GatewayzOptions: server.GatewayzOptions{ + Name: c.gatewayName, + Accounts: true, + }, + } + results, err := doReq(req, "$SYS.REQ.SERVER.PING.GATEWAYZ", c.waitFor, nc) if err != nil { return err @@ -162,7 +259,6 @@ func (c *SrvReportCmd) reportGateway(_ *fisk.ParseContext) error { gateways = append(gateways, *g) } - // TODO: sort sort.Slice(gateways, func(i, j int) bool { switch c.sort { case "server": @@ -271,7 +367,9 @@ func (c *SrvReportCmd) reportRoute(_ *fisk.ParseContext) error { return err } - req := &server.RoutezEventOptions{EventFilterOptions: c.reqFilter()} + req := &server.RoutezEventOptions{ + EventFilterOptions: c.reqFilter(), + } results, err := doReq(req, "$SYS.REQ.SERVER.PING.ROUTEZ", c.waitFor, nc) if err != nil { return err diff --git a/cli/server_request_command.go b/cli/server_request_command.go index 8e4f2ffe..1eba5fa5 100644 --- a/cli/server_request_command.go +++ b/cli/server_request_command.go @@ -80,14 +80,9 @@ func configureServerRequestCommand(srv *fisk.CmdClause) { req.Flag("cluster", "Limit to servers matching a cluster name").StringVar(&c.cluster) req.Flag("tags", "Limit to servers with these configured tags").StringsVar(&c.tags) - subz := req.Command("subscriptions", "Show subscription information").Alias("sub").Alias("subsz").Action(c.subs) - subz.Arg("wait", "Wait for a certain number of responses").Uint32Var(&c.waitFor) - subz.Flag("detail", "Include detail about all subscriptions").UnNegatableBoolVar(&c.detail) - subz.Flag("filter-account", "Filter on a specific account").PlaceHolder("ACCOUNT").StringVar(&c.accountFilter) - subz.Flag("filter-subject", "Filter based on subscriptions matching this subject").PlaceHolder("SUBJECT").StringVar(&c.subjectFilter) - - varz := req.Command("variables", "Show runtime variables").Alias("var").Alias("varz").Action(c.varz) - varz.Arg("wait", "Wait for a certain number of responses").Uint32Var(&c.waitFor) + accountz := req.Command("accounts", "Show account details").Alias("accountz").Alias("acct").Action(c.accountz) + accountz.Arg("wait", "Wait for a certain number of responses").Uint32Var(&c.waitFor) + accountz.Flag("account", "Retrieve information for a specific account").StringVar(&c.account) connz := req.Command("connections", "Show connection details").Alias("conn").Alias("connz").Action(c.conns) connz.Arg("wait", "Wait for a certain number of responses").Uint32Var(&c.waitFor) @@ -100,23 +95,20 @@ func configureServerRequestCommand(srv *fisk.CmdClause) { connz.Flag("filter-subject", "Limits responses only to those connections with matching subscription interest").PlaceHolder("SUBJECT").StringVar(&c.subjectFilter) connz.Flag("filter-empty", "Only shows responses that have connections").Default("false").UnNegatableBoolVar(&c.filterEmpty) - routez := req.Command("routes", "Show route details").Alias("route").Alias("routez").Action(c.routez) - routez.Arg("wait", "Wait for a certain number of responses").Uint32Var(&c.waitFor) - routez.Flag("subscriptions", "Show subscription detail").UnNegatableBoolVar(&c.detail) - gwyz := req.Command("gateways", "Show gateway details").Alias("gateway").Alias("gwy").Alias("gatewayz").Action(c.gwyz) gwyz.Arg("wait", "Wait for a certain number of responses").Uint32Var(&c.waitFor) gwyz.Arg("filter-name", "Filter results on gateway name").PlaceHolder("NAME").StringVar(&c.nameFilter) gwyz.Flag("filter-account", "Show only a certain account in account detail").PlaceHolder("ACCOUNT").StringVar(&c.accountFilter) gwyz.Flag("accounts", "Show account detail").UnNegatableBoolVar(&c.detail) - leafz := req.Command("leafnodes", "Show leafnode details").Alias("leaf").Alias("leafz").Action(c.leafz) - leafz.Arg("wait", "Wait for a certain number of responses").Uint32Var(&c.waitFor) - leafz.Flag("subscriptions", "Show subscription detail").UnNegatableBoolVar(&c.detail) - - accountz := req.Command("accounts", "Show account details").Alias("accountz").Alias("acct").Action(c.accountz) - accountz.Arg("wait", "Wait for a certain number of responses").Uint32Var(&c.waitFor) - accountz.Flag("account", "Retrieve information for a specific account").StringVar(&c.account) + healthz := req.Command("jetstream-health", "Request JetStream health status").Alias("healthz").Action(c.healthz) + healthz.Arg("wait", "Wait for a certain number of responses").Uint32Var(&c.waitFor) + healthz.Flag("js-enabled", "Checks that JetStream should be enabled on all servers").Short('J').BoolVar(&c.jsEnabled) + healthz.Flag("server-only", "Restricts the health check to the JetStream server only, do not check streams and consumers").Short('S').BoolVar(&c.jsServerOnly) + healthz.Flag("account", "Check only a specific Account").StringVar(&c.account) + healthz.Flag("stream", "Check only a specific Stream").StringVar(&c.stream) + healthz.Flag("consumer", "Check only a specific Consumer").StringVar(&c.consumer) + healthz.Flag("details", "Include extended details about all failures").Default("true").BoolVar(&c.includeDetails) jsz := req.Command("jetstream", "Show JetStream details").Alias("jsz").Alias("js").Action(c.jsz) jsz.Arg("wait", "Wait for a certain number of responses").Uint32Var(&c.waitFor) @@ -129,23 +121,32 @@ func configureServerRequestCommand(srv *fisk.CmdClause) { jsz.Flag("leader", "Request a response from the Meta-group leader only").UnNegatableBoolVar(&c.leaderOnly) jsz.Flag("all", "Include accounts, streams, consumers and configuration").UnNegatableBoolVar(&c.includeAll) - healthz := req.Command("jetstream-health", "Request JetStream health status").Alias("healthz").Action(c.healthz) - healthz.Arg("wait", "Wait for a certain number of responses").Uint32Var(&c.waitFor) - healthz.Flag("js-enabled", "Checks that JetStream should be enabled on all servers").Short('J').BoolVar(&c.jsEnabled) - healthz.Flag("server-only", "Restricts the health check to the JetStream server only, do not check streams and consumers").Short('S').BoolVar(&c.jsServerOnly) - healthz.Flag("account", "Check only a specific Account").StringVar(&c.account) - healthz.Flag("stream", "Check only a specific Stream").StringVar(&c.stream) - healthz.Flag("consumer", "Check only a specific Consumer").StringVar(&c.consumer) - healthz.Flag("details", "Include extended details about all failures").Default("true").BoolVar(&c.includeDetails) + kick := req.Command("kick", "Disconnects a client immediately").Action(c.kick) + kick.Arg("client", "The Client ID to disconnect").Required().PlaceHolder("ID").Uint64Var(&c.cid) + kick.Arg("server", "The Server ID to disconnect the client from").Required().PlaceHolder("SERVER_ID").StringVar(&c.host) + + leafz := req.Command("leafnodes", "Show leafnode details").Alias("leaf").Alias("leafz").Action(c.leafz) + leafz.Arg("wait", "Wait for a certain number of responses").Uint32Var(&c.waitFor) + leafz.Flag("subscriptions", "Show subscription detail").UnNegatableBoolVar(&c.detail) profilez := req.Command("profile", "Run a profile").Action(c.profilez) profilez.Arg("profile", "Specify the name of the profile to run (allocs, heap, goroutine, mutex, threadcreate, block, cpu)").Required().EnumVar(&c.profileName, "allocs", "heap", "goroutine", "mutex", "threadcreate", "block", "cpu") profilez.Arg("dir", "Set the output directory for profile files").Default(".").ExistingDirVar(&c.profileDir) profilez.Flag("level", "Set the debug level of the profile").IntVar(&c.profileDebug) - kick := req.Command("kick", "Disconnects a client immediately").Action(c.kick) - kick.Arg("client", "The Client ID to disconnect").Required().PlaceHolder("ID").Uint64Var(&c.cid) - kick.Arg("server", "The Server ID to disconnect the client from").Required().PlaceHolder("SERVER_ID").StringVar(&c.host) + routez := req.Command("routes", "Show route details").Alias("route").Alias("routez").Action(c.routez) + routez.Arg("wait", "Wait for a certain number of responses").Uint32Var(&c.waitFor) + routez.Flag("subscriptions", "Show subscription detail").UnNegatableBoolVar(&c.detail) + + subz := req.Command("subscriptions", "Show subscription information").Alias("sub").Alias("subsz").Action(c.subs) + subz.Arg("wait", "Wait for a certain number of responses").Uint32Var(&c.waitFor) + subz.Flag("detail", "Include detail about all subscriptions").UnNegatableBoolVar(&c.detail) + subz.Flag("filter-account", "Filter on a specific account").PlaceHolder("ACCOUNT").StringVar(&c.accountFilter) + subz.Flag("filter-subject", "Filter based on subscriptions matching this subject").PlaceHolder("SUBJECT").StringVar(&c.subjectFilter) + + varz := req.Command("variables", "Show runtime variables").Alias("var").Alias("varz").Action(c.varz) + varz.Arg("wait", "Wait for a certain number of responses").Uint32Var(&c.waitFor) + } func (c *SrvRequestCmd) kick(_ *fisk.ParseContext) error {