From 14f52667020fd9f7b1916f0e08735bfa5e92a394 Mon Sep 17 00:00:00 2001 From: Yuxiang Cao Date: Tue, 7 May 2024 16:19:29 -0700 Subject: [PATCH] feat: print result in format - Refactor code to have types to carry test config and results. - Add support to print test result in Json. --- cmd/loadTest.go | 75 ++++++++++++----- cmd/loadTestAsymmetricCrypto.go | 5 +- cmd/loadTestGenerateKey.go | 29 +++++-- cmd/loadTestInvokePlugin.go | 8 +- cmd/loadTestSignVerify.go | 5 +- cmd/loadTestSymmetricCrypto.go | 5 +- cmd/loadTestVersion.go | 2 +- cmd/load_test_types.go | 143 +++++++++++++++++++++++++++----- cmd/load_test_types_test.go | 4 +- cmd/root.go | 23 ++++- 10 files changed, 239 insertions(+), 60 deletions(-) diff --git a/cmd/loadTest.go b/cmd/loadTest.go index a4cb61b..2680e87 100644 --- a/cmd/loadTest.go +++ b/cmd/loadTest.go @@ -9,6 +9,7 @@ package cmd import ( "fmt" "log" + "os" "sync" "time" @@ -51,26 +52,33 @@ const ( testStage ) -type setupFunc func(client *sdkms.Client) (interface{}, error) +type setupFunc func(client *sdkms.Client, testConfig *TestConfig) (interface{}, error) type testFunc func(client *sdkms.Client, stage loadTestStage, arg interface{}) (interface{}, time.Duration, profilingMetricStr, error) type cleanupFunc func(client *sdkms.Client) func loadTest(name string, setup setupFunc, test testFunc, cleanup cleanupFunc) { - testConfig := &TestConfig{} - testConfig.TestName = name - testConfig.WarmupDuration = warmupDuration - testConfig.TestDuration = testDuration - testConfig.TargetQPS = queriesPerSecond - testConfig.Connections = connections - testConfig.CreateSession = createSession - - fmt.Printf(" Load test: %v\n", name) - fmt.Printf(" Server: %v:%v\n", serverName, serverPort) - fmt.Printf(" Target QPS: %v\n", queriesPerSecond) - fmt.Printf(" Connections: %v\n", connections) - fmt.Printf(" Test Duration: %v\n", testDuration) + testTime := time.Now() + + fmt.Printf("Load test: %v\n", name) + fmt.Printf("Server: %v:%v\n", serverName, serverPort) + fmt.Printf("Target QPS: %v\n", queriesPerSecond) + fmt.Printf("Connections: %v\n", connections) + fmt.Printf("Test Duration: %v\n", testDuration) fmt.Printf("Warmup Duration: %v\n", warmupDuration) fmt.Println() + + testConfig := TestConfig{ + TestName: name, + ServerName: serverName, + ServerPort: serverPort, + VerifyTls: !insecureTLS, + Connections: connections, + CreateSession: createSession, + WarmupDuration: warmupDuration, + TestDuration: testDuration, + TargetQPS: queriesPerSecond, + } + type testMetric struct { t time.Time d time.Duration @@ -105,7 +113,7 @@ func loadTest(name string, setup setupFunc, test testFunc, cleanup cleanupFunc) defer wg1.Done() client := sdkmsClient() - arg, err := setup(&client) + arg, err := setup(&client, &testConfig) if err != nil { log.Fatalf("Fatal error: %v\n", err) } @@ -176,12 +184,6 @@ func loadTest(name string, setup setupFunc, test testFunc, cleanup cleanupFunc) sendDuration := lastTick.Sub(t0) testDuration := t1.Sub(t0) - fmt.Printf(" Warmup: %v queries, %v\n", len(warmups), summarizeTimings(warmups)) - fmt.Printf(" Test: %v queries, %v\n", len(tests), summarizeTimings(tests)) - fmt.Printf(" Test duration: %v (%0.2f QPS)\n", testDuration, float64(len(tests))/testDuration.Seconds()) - fmt.Printf(" Send duration: %v (%0.2f QPS)\n", sendDuration, float64(len(tests))/sendDuration.Seconds()) - fmt.Printf("Profiling data: %v examples\n", len(profilingMetricStrArr)) - if len(profilingMetricStrArr) != 0 { dataArr := parseProfilingMetricStrArr(profilingMetricStrArr) summarizeProfilingMetrics(dataArr) @@ -189,5 +191,34 @@ func loadTest(name string, setup setupFunc, test testFunc, cleanup cleanupFunc) saveProfilingMetricsToCSV(dataArr) } } - fmt.Print("\n\n") + testResult := TestResult{ + Warmup: StatisticFromDurations(warmups, warmupDuration), + Test: StatisticFromDurations(tests, testDuration), + ActualTestDuration: testDuration, + SendDuration: sendDuration, + ProfilingResults: &ProfilingStatistics{}, + } + + testSummary := &TestSummary{ + TestTime: testTime, + Config: &testConfig, + Result: &testResult, + } + + switch outputFormat { + case Plain: + err := testSummary.WritePlain(os.Stdout) + if err != nil { + log.Fatalf("failed to write test summary in plain: %v\n", err) + } + case JSON: + err := testSummary.WriteJson(os.Stdout) + if err != nil { + log.Fatalf("failed to write test summary in json: %v\n", err) + } + case YAML: + log.Fatalf("write test summary in yaml is not yet supported\n") + default: + log.Fatalf("unreachable: unacceptable output format option: %v\n", outputFormat) + } } diff --git a/cmd/loadTestAsymmetricCrypto.go b/cmd/loadTestAsymmetricCrypto.go index 9efe961..641a6d7 100644 --- a/cmd/loadTestAsymmetricCrypto.go +++ b/cmd/loadTestAsymmetricCrypto.go @@ -39,7 +39,10 @@ func asymmetricCryptoLoadTest() { // get basic info of the given sobject key := GetSobject(&keyID) - setup := func(client *sdkms.Client) (interface{}, error) { + setup := func(client *sdkms.Client, testConfig *TestConfig) (interface{}, error) { + if testConfig.Sobject != nil { + testConfig.Sobject = key + } if createSession { _, err := client.AuthenticateWithAPIKey(context.Background(), apiKey) return nil, err diff --git a/cmd/loadTestGenerateKey.go b/cmd/loadTestGenerateKey.go index bf0f4fc..168974d 100644 --- a/cmd/loadTestGenerateKey.go +++ b/cmd/loadTestGenerateKey.go @@ -47,9 +47,24 @@ func loadTestGenerateKey() { keySize = 2048 } } - setup := func(client *sdkms.Client) (interface{}, error) { - _, err := client.AuthenticateWithAPIKey(context.Background(), apiKey) - return nil, err + setup := func(client *sdkms.Client, testConfig *TestConfig) (interface{}, error) { + if createSession { + _, err := client.AuthenticateWithAPIKey(context.Background(), apiKey) + if err != nil { + return nil, err + } + } + client.Auth = sdkms.APIKey(apiKey) + + // Create one example key to fill the test config + if testConfig.Sobject != nil { + key, _, _, err := generateKey(client) + if err != nil { + return nil, err + } + testConfig.Sobject = key + } + return nil, nil } cleanup := func(client *sdkms.Client) { client.TerminateSession(context.Background()) @@ -59,7 +74,7 @@ func loadTestGenerateKey() { if stage == warmupStage { return nil, 0, "", nil } - d, p, err := generateKey(client) + _, d, p, err := generateKey(client) return nil, d, p, err } name := fmt.Sprintf("generate %v key", keyType) @@ -69,7 +84,7 @@ func loadTestGenerateKey() { loadTest(name, setup, test, cleanup) } -func generateKey(client *sdkms.Client) (time.Duration, profilingMetricStr, error) { +func generateKey(client *sdkms.Client) (*sdkms.Sobject, time.Duration, profilingMetricStr, error) { req := sdkms.SobjectRequest{ Transient: someBool(true), ObjType: convertObjectType(keyType), @@ -80,13 +95,13 @@ func generateKey(client *sdkms.Client) (time.Duration, profilingMetricStr, error ctx := context.WithValue(context.Background(), responseHeaderKey, http.Header{}) t0 := time.Now() - _, err := client.CreateSobject(ctx, req) + key, err := client.CreateSobject(ctx, req) d := time.Since(t0) header := ctx.Value(responseHeaderKey).(http.Header) p := profilingMetricStr(header.Get("Profiling-Data")) - return d, p, err + return key, d, p, err } func someBool(x bool) *bool { return &x } diff --git a/cmd/loadTestInvokePlugin.go b/cmd/loadTestInvokePlugin.go index e92a4f8..44097f5 100644 --- a/cmd/loadTestInvokePlugin.go +++ b/cmd/loadTestInvokePlugin.go @@ -54,7 +54,13 @@ func invokePluginLoadTest() { log.Fatalf("Plugin input must be valid JSON: %v\n", err) } - setup := func(client *sdkms.Client) (interface{}, error) { + setup := func(client *sdkms.Client, testConfig *TestConfig) (interface{}, error) { + if testConfig.Plugin != nil { + testConfig.Plugin = plugin + } + if testConfig.PluginInput != nil { + testConfig.PluginInput = &input + } if createSession { _, err := client.AuthenticateWithAPIKey(context.Background(), apiKey) return nil, err diff --git a/cmd/loadTestSignVerify.go b/cmd/loadTestSignVerify.go index 151dd57..e75d995 100644 --- a/cmd/loadTestSignVerify.go +++ b/cmd/loadTestSignVerify.go @@ -43,7 +43,10 @@ func signVerifyLoadTest() { // get basic info of the given sobject key := GetSobject(&signKeyID) - setup := func(client *sdkms.Client) (interface{}, error) { + setup := func(client *sdkms.Client, testConfig *TestConfig) (interface{}, error) { + if testConfig.Sobject != nil { + testConfig.Sobject = key + } if createSession { _, err := client.AuthenticateWithAPIKey(context.Background(), apiKey) return nil, err diff --git a/cmd/loadTestSymmetricCrypto.go b/cmd/loadTestSymmetricCrypto.go index a6709ed..3b0ee8e 100644 --- a/cmd/loadTestSymmetricCrypto.go +++ b/cmd/loadTestSymmetricCrypto.go @@ -49,7 +49,10 @@ func symmetricCryptoLoadTest() { // get basic info of the given sobject key := GetSobject(&keyID) - setup := func(client *sdkms.Client) (interface{}, error) { + setup := func(client *sdkms.Client, testConfig *TestConfig) (interface{}, error) { + if testConfig.Sobject != nil { + testConfig.Sobject = key + } if createSession { _, err := client.AuthenticateWithAPIKey(context.Background(), apiKey) return nil, err diff --git a/cmd/loadTestVersion.go b/cmd/loadTestVersion.go index 6cceaca..4f68ca5 100644 --- a/cmd/loadTestVersion.go +++ b/cmd/loadTestVersion.go @@ -29,7 +29,7 @@ func init() { } func versionLoadTest() { - setup := func(client *sdkms.Client) (interface{}, error) { + setup := func(client *sdkms.Client, testConfig *TestConfig) (interface{}, error) { return nil, nil } cleanup := func(client *sdkms.Client) {} diff --git a/cmd/load_test_types.go b/cmd/load_test_types.go index c1912c8..adf6288 100644 --- a/cmd/load_test_types.go +++ b/cmd/load_test_types.go @@ -7,50 +7,73 @@ package cmd import ( + "bytes" "encoding/json" + "fmt" "io" "time" "github.com/fortanix/sdkms-client-go/sdkms" + "github.com/montanaflynn/stats" ) type TestSummary struct { - TestTime time.Time `json:"test_time" yaml:"test_time"` - Config TestConfig `json:"config" yaml:"config"` - Result TestResult `json:"result" yaml:"result"` + TestTime time.Time `json:"test_time" yaml:"test_time"` + Config *TestConfig `json:"config" yaml:"config"` + Result *TestResult `json:"result" yaml:"result"` } -type TestSummaryJsonWriter interface { - WriteJson(w io.Writer) error +type TestConfig struct { + TestName string `json:"test_name" yaml:"test_name"` + ServerName string `json:"server_name" yaml:"server_name"` + ServerPort uint16 `json:"server_port" yaml:"server_port"` + VerifyTls bool `json:"verify_tls" yaml:"verify_tls"` + Connections uint `json:"connections" yaml:"connections"` + CreateSession bool `json:"create_session" yaml:"create_session"` + WarmupDuration time.Duration `json:"warmup_duration" yaml:"warmup_duration"` + TestDuration time.Duration `json:"test_duration" yaml:"test_duration"` + TargetQPS uint `json:"target_qps" yaml:"target_qps"` + Sobject *sdkms.Sobject `json:"sobject" yaml:"sobject"` + Plugin *sdkms.Plugin `json:"plugin" yaml:"plugin"` + PluginInput *json.RawMessage `json:"plugin_input" yaml:"plugin_input"` } -type TestConfig struct { - TestName string `json:"test_name" yaml:"test_name"` - ServerName string `json:"server_name" yaml:"server_name"` - ServerPort uint16 `json:"server_port" yaml:"server_port"` - VerifyTls bool `json:"verify_tls" yaml:"verify_tls"` - Connections uint `json:"connections" yaml:"connections"` - CreateSession bool `json:"create_session" yaml:"create_session"` - WarmupDuration time.Duration `json:"warmup_duration" yaml:"warmup_duration"` - TestDuration time.Duration `json:"test_duration" yaml:"test_duration"` - TargetQPS uint `json:"target_qps" yaml:"target_qps"` - ApiName string `json:"api_name" yaml:"api_name"` - Sobject *sdkms.Sobject `json:"sobject" yaml:"sobject"` - PluginName string `json:"plugin_name" yaml:"plugin_name"` +func (tc *TestConfig) Print(w io.Writer) { + fmt.Fprintf(w, "TestName: %s\n", tc.TestName) + fmt.Fprintf(w, "ServerName: %s\n", tc.ServerName) + fmt.Fprintf(w, "ServerPort: %d\n", tc.ServerPort) + fmt.Fprintf(w, "VerifyTls: %t\n", tc.VerifyTls) + fmt.Fprintf(w, "Connections: %d\n", tc.Connections) + fmt.Fprintf(w, "CreateSession: %t\n", tc.CreateSession) + fmt.Fprintf(w, "WarmupDuration: %s\n", tc.WarmupDuration) + fmt.Fprintf(w, "TestDuration: %s\n", tc.TestDuration) + fmt.Fprintf(w, "TargetQPS: %d\n", tc.TargetQPS) + fmt.Fprintf(w, "Sobject: %+v\n", tc.Sobject) + fmt.Fprintf(w, "Plugin: %+v\n", tc.Plugin) + fmt.Fprintf(w, "PluginInput: %+v\n", tc.PluginInput) } type TestResult struct { - Warmup Statistic `json:"warmup" yaml:"warmup"` - Test Statistic `json:"test" yaml:"test"` - ProfilingResults ProfilingStatistics `json:"profiling_results" yaml:"profiling_results"` + Warmup *Statistic `json:"warmup" yaml:"warmup"` + Test *Statistic `json:"test" yaml:"test"` + ActualTestDuration time.Duration `json:"actual_test_duration" yaml:"actual_test_duration"` + SendDuration time.Duration `json:"send_duration" yaml:"send_duration"` + ProfilingResults *ProfilingStatistics `json:"profiling_results" yaml:"profiling_results"` +} + +func (tr *TestResult) Print(w io.Writer) { + fmt.Fprintf(w, "Warmup: %s\n", tr.Warmup.String()) + fmt.Fprintf(w, "Test: %s\n", tr.Test.String()) + fmt.Fprintf(w, "ActualTestDuration: %s\n", tr.ActualTestDuration) + fmt.Fprintf(w, "SendDuration: %s\n", tr.ActualTestDuration) } type Statistic struct { QueryNumber uint `json:"query_number" yaml:"query_number"` QPS float64 `json:"qps" yaml:"qps"` + Avg float64 `json:"avg" yaml:"avg"` Min float64 `json:"min" yaml:"min"` Max float64 `json:"max" yaml:"max"` - Avg float64 `json:"avg" yaml:"avg"` P50 float64 `json:"p50" yaml:"p50"` P75 float64 `json:"p75" yaml:"p75"` P90 float64 `json:"p90" yaml:"p90"` @@ -58,6 +81,53 @@ type Statistic struct { P99 float64 `json:"p99" yaml:"p99"` } +func StatisticFromDurations(times []time.Duration, duration time.Duration) *Statistic { + queryNumber := uint(len(times)) + if queryNumber == 0 { + return &Statistic{} + } + data := stats.LoadRawData(times) + min, _ := data.Min() + max, _ := data.Max() + avg, _ := data.Mean() + p50, _ := data.Percentile(50) + p75, _ := data.Percentile(75) + p90, _ := data.Percentile(90) + p95, _ := data.Percentile(95) + p99, _ := data.Percentile(99) + return &Statistic{ + QueryNumber: queryNumber, + QPS: float64(queryNumber) / testDuration.Seconds(), + Avg: avg, + Min: min, + Max: max, + P50: p50, + P75: p75, + P90: p90, + P95: p95, + P99: p99, + } +} + +func (st *Statistic) Print(w io.Writer) { + fmt.Fprintf(w, "QueryNumber: %d,", st.QueryNumber) + fmt.Fprintf(w, "QPS: %f,", st.QPS) + fmt.Fprintf(w, "Avg: %f,", st.Avg) + fmt.Fprintf(w, "Min: %f,", st.Min) + fmt.Fprintf(w, "Max: %f,", st.Max) + fmt.Fprintf(w, "P50: %f,", st.P50) + fmt.Fprintf(w, "P75: %f,", st.P75) + fmt.Fprintf(w, "P90: %f,", st.P90) + fmt.Fprintf(w, "P95: %f,", st.P95) + fmt.Fprintf(w, "P99: %f", st.P99) +} + +func (st *Statistic) String() string { + buf := new(bytes.Buffer) + st.Print(buf) + return buf.String() +} + type ProfilingStatistics struct { InQueue Statistic `json:"in_queue" yaml:"in_queue"` ParseRequest Statistic `json:"parse_request" yaml:"parse_request"` @@ -69,6 +139,35 @@ type ProfilingStatistics struct { Total Statistic `json:"total" yaml:"total"` } +func (ps *ProfilingStatistics) Print(w io.Writer) { + fmt.Fprintf(w, "InQueue: %s\n", ps.InQueue.String()) + fmt.Fprintf(w, "ParseRequest: %s\n", ps.ParseRequest.String()) + fmt.Fprintf(w, "SessionLookup: %s\n", ps.SessionLookup.String()) + fmt.Fprintf(w, "ValidateInput: %s\n", ps.ValidateInput.String()) + fmt.Fprintf(w, "CheckAccess: %s\n", ps.CheckAccess.String()) + fmt.Fprintf(w, "Operate: %s\n", ps.Operate.String()) + fmt.Fprintf(w, "DbFlush: %s\n", ps.DbFlush.String()) + fmt.Fprintf(w, "Total: %s\n", ps.Total.String()) +} + +type TestSummaryJsonWriter interface { + WriteJson(w io.Writer) error +} + +type TestSummaryPlainWriter interface { + WritePlain(w io.Writer) error +} + +func (ts *TestSummary) WritePlain(w io.Writer) error { + fmt.Fprintf(w, "-----BEGIN TEST SUMMARY-----\n") + fmt.Fprintf(w, "TestTime: %s\n", ts.TestTime) + ts.Config.Print(w) + fmt.Fprintf(w, "\n") + ts.Result.Print(w) + fmt.Fprintf(w, "-----END TEST SUMMARY-----\n") + return nil +} + func (ts *TestSummary) WriteJson(w io.Writer) error { encoder := json.NewEncoder(w) encoder.SetIndent("", " ") diff --git a/cmd/load_test_types_test.go b/cmd/load_test_types_test.go index d7ba541..41ea346 100644 --- a/cmd/load_test_types_test.go +++ b/cmd/load_test_types_test.go @@ -69,14 +69,12 @@ func newTestSummary() *TestSummary { WarmupDuration: 10 * time.Second, TestDuration: 30 * time.Second, TargetQPS: 1000, - ApiName: "api1", Sobject: nil, - PluginName: "plugin1", }, Result: TestResult{ Warmup: newRandomStatistic(), Test: newRandomStatistic(), - ProfilingResults: ProfilingStatistics{ + ProfilingResults: &ProfilingStatistics{ InQueue: newRandomStatistic(), ParseRequest: newRandomStatistic(), SessionLookup: newRandomStatistic(), diff --git a/cmd/root.go b/cmd/root.go index 0eef0ac..8081956 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -14,17 +14,38 @@ import ( "github.com/spf13/cobra" ) +const ( + Plain string = "plain" + JSON string = "json" + YAML string = "yaml" +) + // TODO: get rid of global variables, tracking issue: #16 var serverName string var serverPort uint16 var insecureTLS bool var requestTimeout time.Duration var idleConnectionTimeout time.Duration +var outputFormat string var rootCmd = &cobra.Command{ Use: "dsm-perf-tool", Short: "DSM performance tool", Long: `DSM performance tool`, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + // validate arguments + switch outputFormat { + case Plain: + // Plain is accepted + case JSON: + // JSON is accepted + case YAML: + // YAML is accepted + default: + return fmt.Errorf("unacceptable output format option: %v", outputFormat) + } + return nil + }, } func Execute() { @@ -38,7 +59,7 @@ func init() { rootCmd.PersistentFlags().StringVarP(&serverName, "server", "s", "sdkms.test.fortanix.com", "DSM server host name") rootCmd.PersistentFlags().Uint16VarP(&serverPort, "port", "p", 443, "DSM server port") rootCmd.PersistentFlags().BoolVar(&insecureTLS, "insecure", false, "Do not validate server's TLS certificate") + rootCmd.PersistentFlags().StringVar(&outputFormat, "output-format", Plain, "Output format, accepted options are: 'plain', 'json', 'yaml'") rootCmd.PersistentFlags().DurationVar(&requestTimeout, "request-timeout", 60*time.Second, "HTTP request timeout, 0 means no timeout") rootCmd.PersistentFlags().DurationVar(&idleConnectionTimeout, "idle-connection-timeout", 0, "Idle connection timeout, 0 means no timeout (default behavior)") - }