feat: To detect KPI, let's use whole workload as deployment might tak…

…e tenths of seconds as well (#806) * feat: To detect KPI, let's use whole workload as deployment might take tenths of seconds as well * feat: Also store number of errors from final run * feat(max-concurrency): move workload KPI calculation to load test itself Signed-off-by: Pavel Macík <[email protected]> * feat: Also compute KPI number from before threshold and after threshold numbers. * fix: Typo * feat: Add integration test and deployment time to the max-concurrency.csv * fix: `$t` is not index, it is value * fix: Also update default results JSON Co-authored-by: Pavel Macík <[email protected]> * fix: Add sane value if we fail to reach the threshold Co-authored-by: Pavel Macík <[email protected]> --------- Signed-off-by: Pavel Macík <[email protected]> Co-authored-by: Pavel Macík <[email protected]>
konflux-ci · Oct 10, 2023 · 10f9279 · 10f9279
1 parent 2df740c
commit 10f9279
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 6 deletions.
diff --git a/cmd/loadTests.go b/cmd/loadTests.go
@@ -177,6 +177,8 @@ type LogData struct {
 	IntegrationTestsPipelineRunFailureCount int64   `json:"integrationTestsRunPipelineFailures"`
 	IntegrationTestsPipelineRunFailureRate  float64 `json:"integrationTestsRunPipelineFailureRate"`
 
+	WorkloadKPI float64 `json:"workloadKPI"`
+
 	ErrorCounts []ErrorCount      `json:"errorCounts"`
 	Errors      []ErrorOccurrence `json:"errors"`
 	ErrorsTotal int               `json:"errorsTotal"`
@@ -623,12 +625,17 @@ func setup(cmd *cobra.Command, args []string) {
 	deploymentFailureRate := float64(deploymentFailureCount) / float64(overallCount)
 	logData.DeploymentFailureRate = deploymentFailureRate
 
+	workloadKPI := logData.AverageTimeToCreateApplications + logData.AverageTimeToCreateCDQs + logData.AverageTimeToCreateComponents + logData.AverageTimeToRunPipelineSucceeded + logData.AverageTimeToDeploymentSucceeded
+	logData.WorkloadKPI = workloadKPI
 	if stage {
 		StageCleanup(selectedUsers)
 	}
 
 	klog.Infof("🏁 Load Test Completed!")
 	klog.Infof("📈 Results 📉")
+
+	klog.Infof("Workload KPI: %.2f", workloadKPI)
+
 	klog.Infof("Avg/max time to spin up users: %.2f s/%.2f s", averageTimeToSpinUpUsers, logData.MaxTimeToSpinUpUsers)
 	klog.Infof("Avg/max time to create application: %.2f s/%.2f s", averageTimeToCreateApplications, logData.MaxTimeToCreateApplications)
 	klog.Infof("Avg/max time to create cdq: %.2f s/%.2f s", averageTimeToCreateCDQs, logData.MaxTimeToCreateCDQs)

diff --git a/tests/load-tests/ci-scripts/max-concurrency/collect-results.sh b/tests/load-tests/ci-scripts/max-concurrency/collect-results.sh
@@ -67,6 +67,7 @@ dt_format='"%Y-%m-%dT%H:%M:%SZ"'
 ## Max concurrency scalability
 max_concurrency_csv=$ARTIFACT_DIR/max-concurrency.csv
 echo "Threads\
+${csv_delim}WorkloadKPI\
 ${csv_delim}Errors\
 ${csv_delim}UserAvgTime\
 ${csv_delim}UserMaxTime\
@@ -78,6 +79,10 @@ ${csv_delim}ComponentsAvgTime\
 ${csv_delim}ComponentsMaxTime\
 ${csv_delim}PipelineRunAvgTime\
 ${csv_delim}PipelineRunMaxTime\
+${csv_delim}integrationTestsRunPipelineSucceededTimeAvg\
+${csv_delim}integrationTestsRunPipelineSucceededTimeMax\
+${csv_delim}deploymentSucceededTimeAvg\
+${csv_delim}deploymentSucceededTimeMax\
 ${csv_delim}ClusterCPUUsageAvg\
 ${csv_delim}ClusterDiskUsageAvg\
 ${csv_delim}ClusterMemoryUsageAvg\
@@ -91,7 +96,7 @@ ${csv_delim}TokenPoolRateSecondaryAvg\
 ${csv_delim}ClusterPipelineRunCountAvg\
 ${csv_delim}ClusterPipelineWorkqueueDepthAvg\
 ${csv_delim}ClusterPipelineScheduleFirstPodAvg\
-${csv_delim}ClusterTaskrunThrottledByNodeResourcesAvg\
+${csv_delim}ClusterTaskRunThrottledByNodeResourcesAvg\
 ${csv_delim}ClusterTaskRunThrottledByDefinedQuotaAvg\
 ${csv_delim}EtcdRequestDurationSecondsAvg\
 ${csv_delim}ClusterNetworkBytesTotalAvg\
@@ -103,6 +108,7 @@ mc_files=$(find "$output_dir" -type f -name 'load-tests.max-concurrency.*.json')
 if [ -n "$mc_files" ]; then
     cat $mc_files |
         jq -rc "(.threads | tostring) \
+        + $csv_delim_quoted + (.workloadKPI | tostring) \
         + $csv_delim_quoted + (.errorsTotal | tostring) \
         + $csv_delim_quoted + (.createUserTimeAvg | tostring) \
         + $csv_delim_quoted + (.createUserTimeMax | tostring) \
@@ -114,6 +120,10 @@ if [ -n "$mc_files" ]; then
         + $csv_delim_quoted + (.createComponentsTimeMax | tostring) \
         + $csv_delim_quoted + (.runPipelineSucceededTimeAvg | tostring) \
         + $csv_delim_quoted + (.runPipelineSucceededTimeMax | tostring) \
+        + $csv_delim_quoted + (.integrationTestsRunPipelineSucceededTimeAvg | tostring) \
+        + $csv_delim_quoted + (.integrationTestsRunPipelineSucceededTimeMax | tostring) \
+        + $csv_delim_quoted + (.deploymentSucceededTimeAvg | tostring) \
+        + $csv_delim_quoted + (.deploymentSucceededTimeMax | tostring) \
         + $csv_delim_quoted + (.measurements.cluster_cpu_usage_seconds_total_rate.mean | tostring) \
         + $csv_delim_quoted + (.measurements.cluster_disk_throughput_total.mean | tostring) \
         + $csv_delim_quoted + (.measurements.cluster_memory_usage_rss_total.mean | tostring) \

diff --git a/tests/load-tests/run-max-concurrency.sh b/tests/load-tests/run-max-concurrency.sh
@@ -43,7 +43,7 @@ else
     IFS="," read -r -a maxConcurrencySteps <<<"$(echo "${MAX_CONCURRENCY_STEPS:-1\ 5\ 10\ 25\ 50\ 100\ 150\ 200}" | sed 's/ /,/g')"
     maxThreads=${MAX_THREADS:-10}
     threshold=${THRESHOLD:-300}
-    echo '{"startTimestamp":"'$(date +%FT%T%:z)'", "maxThreads": '"$maxThreads"', "maxConcurrencySteps": "'"${maxConcurrencySteps[*]}"'", "threshold": '"$threshold"', "maxConcurrencyReached": 0, "endTimestamp": ""}' | jq >"$output"
+    echo '{"startTimestamp":"'$(date +%FT%T%:z)'", "maxThreads": '"$maxThreads"', "maxConcurrencySteps": "'"${maxConcurrencySteps[*]}"'", "threshold": '"$threshold"', "maxConcurrencyReached": 0, "computedConcurrency": 0, "workloadKPI": 0, "endTimestamp": "", "errorsTotal": -1}' | jq >"$output"
     for t in "${maxConcurrencySteps[@]}"; do
         if (("$t" > "$maxThreads")); then
             break
@@ -73,14 +73,21 @@ else
         index=$(printf "%04d" "$t")
         cp -vf "$output_dir/load-tests.json" "$output_dir/load-tests.max-concurrency.$index.json"
         cp -vf "$output_dir/load-tests.log" "$output_dir/load-tests.max-concurrency.$index.log"
-        pipelineRunThresholdExceeded=$(jq -rc ".runPipelineSucceededTimeMax > $threshold" "$output_dir/load-tests.json")
-        pipelineRunKPI=$(jq -rc ".runPipelineSucceededTimeMax" "$output_dir/load-tests.json")
-        if [ "$pipelineRunThresholdExceeded" = "true" ]; then
-            echo "The maximal time a pipeline run took to succeed (${pipelineRunKPI}s) has exceeded a threshold of ${threshold}s with $t threads."
+        workloadKPI=$(jq '.workloadKPI' "$output_dir/load-tests.json")
+        if awk "BEGIN { exit !($workloadKPI > $threshold)}"; then
+            echo "The average time a workload took to succeed (${workloadKPI}s) has exceeded a threshold of ${threshold}s with $t threads."
+            workloadKPIOld=$(jq '.workloadKPI' "$output")
+            threadsOld=$(jq '.maxConcurrencyReached' "$output")
+            computedConcurrency=$(python -c "import sys; t = float(sys.argv[1]); a = float(sys.argv[2]); b = float(sys.argv[3]); c = float(sys.argv[4]); d = float(sys.argv[5]); print((t - b) / ((d - b) / (c - a)) + a)" "$threshold" "$threadsOld" "$workloadKPIOld" "$t" "$workloadKPI")
+            jq ".computedConcurrency = $computedConcurrency" "$output" >"$output_dir/$$.json" && mv -f "$output_dir/$$.json" "$output"
             break
         else
             jq ".maxConcurrencyReached = $t" "$output" >"$output_dir/$$.json" && mv -f "$output_dir/$$.json" "$output"
+            jq ".workloadKPI = $workloadKPI" "$output" >"$output_dir/$$.json" && mv -f "$output_dir/$$.json" "$output"
+            jq ".computedConcurrency = $t" "$output" >"$output_dir/$$.json" && mv -f "$output_dir/$$.json" "$output"
             jq '.endTimestamp = "'$(date +%FT%T%:z)'"' "$output" >"$output_dir/$$.json" && mv -f "$output_dir/$$.json" "$output"
+            errorsTotal=$(jq '.errorsTotal' "$output_dir/load-tests.json")
+            jq ".errorsTotal = $errorsTotal" "$output" >"$output_dir/$$.json" && mv -f "$output_dir/$$.json" "$output"
         fi
     done
     DRY_RUN=false ./clear.sh "$USER_PREFIX"