From 3e98cb16253b6135ed2cbd894ead3bed439f0185 Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Tue, 17 Sep 2024 22:29:02 +0100 Subject: [PATCH] [CI] fix benchmark marathon (#16447) - split main.sh to core.sh and main.sh - rename all.sh to marathon.sh - fix vault expiry issue for long running task - fix unable to call main function - update save object runtime field `release` to return true when `version` contains "SNAPSHOT" --- ...ne.yml => benchmark_marathon_pipeline.yml} | 4 +- .buildkite/scripts/benchmark/all.sh | 18 - .buildkite/scripts/benchmark/core.sh | 315 +++++++++++++++++ .buildkite/scripts/benchmark/main.sh | 316 +----------------- .buildkite/scripts/benchmark/marathon.sh | 41 +++ .../benchmark/save-objects/CHANGELOG.md | 5 + .../save-objects/benchmark_objects.ndjson | 6 +- catalog-info.yaml | 20 +- 8 files changed, 379 insertions(+), 346 deletions(-) rename .buildkite/{benchmark_versions_pipeline.yml => benchmark_marathon_pipeline.yml} (68%) delete mode 100755 .buildkite/scripts/benchmark/all.sh create mode 100755 .buildkite/scripts/benchmark/core.sh create mode 100755 .buildkite/scripts/benchmark/marathon.sh create mode 100644 .buildkite/scripts/benchmark/save-objects/CHANGELOG.md diff --git a/.buildkite/benchmark_versions_pipeline.yml b/.buildkite/benchmark_marathon_pipeline.yml similarity index 68% rename from .buildkite/benchmark_versions_pipeline.yml rename to .buildkite/benchmark_marathon_pipeline.yml index d60166ac441..d2f8b657726 100644 --- a/.buildkite/benchmark_versions_pipeline.yml +++ b/.buildkite/benchmark_marathon_pipeline.yml @@ -7,5 +7,5 @@ agents: diskType: pd-ssd steps: - - label: "Benchmark multiple versions" - command: .buildkite/scripts/benchmark/all.sh \ No newline at end of file + - label: "Benchmark Marathon" + command: .buildkite/scripts/benchmark/marathon.sh \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/all.sh b/.buildkite/scripts/benchmark/all.sh deleted file mode 100755 index 282ed3d488a..00000000000 --- a/.buildkite/scripts/benchmark/all.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# ******************************************************* -# Run benchmark for versions that have flow metrics -# Usage: -# nohup bash -x all.sh > log.log 2>&1 & -# Accept env vars: -# STACK_VERSIONS=8.15.0,8.15.1,8.16.0-SNAPSHOT # versions to test. It is comma separator string -# ******************************************************* - -IFS=',' -STACK_VERSIONS="${STACK_VERSIONS:-8.6.0,8.7.0,8.8.0,8.9.0,8.10.0,8.11.0,8.12.0,8.13.0,8.14.0,8.15.0}" -read -ra STACK_VERSIONS <<< "$STACK_VERSIONS" - -for V in "${STACK_VERSIONS[@]}" ; do - LS_VERSION="$V" "main.sh" -done \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/core.sh b/.buildkite/scripts/benchmark/core.sh new file mode 100755 index 00000000000..a7bb733713b --- /dev/null +++ b/.buildkite/scripts/benchmark/core.sh @@ -0,0 +1,315 @@ +#!/usr/bin/env bash +set -eo pipefail + +SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")" +CONFIG_PATH="$SCRIPT_PATH/config" +source "$SCRIPT_PATH/util.sh" + +usage() { + echo "Usage: $0 [FB_CNT] [QTYPE] [CPU] [MEM]" + echo "Example: $0 4 {persisted|memory|all} 2 2" + exit 1 +} + +parse_args() { + while [[ "$#" -gt 0 ]]; do + if [ -z "$FB_CNT" ]; then + FB_CNT=$1 + elif [ -z "$QTYPE" ]; then + case $1 in + all | persisted | memory) + QTYPE=$1 + ;; + *) + echo "Error: wrong queue type $1" + usage + ;; + esac + elif [ -z "$CPU" ]; then + CPU=$1 + elif [ -z "$MEM" ]; then + MEM=$1 + else + echo "Error: Too many arguments" + usage + fi + shift + done + + # set default value + # number of filebeat + FB_CNT=${FB_CNT:-4} + # all | persisted | memory + QTYPE=${QTYPE:-all} + CPU=${CPU:-4} + MEM=${MEM:-4} + XMX=$((MEM / 2)) + + IFS=',' + # worker multiplier: 1,2,4 + MULTIPLIERS="${MULTIPLIERS:-1,2,4}" + read -ra MULTIPLIERS <<< "$MULTIPLIERS" + BATCH_SIZES="${BATCH_SIZES:-500}" + read -ra BATCH_SIZES <<< "$BATCH_SIZES" + # tags to json array + read -ra TAG_ARRAY <<< "$TAGS" + JSON_TAGS=$(printf '"%s",' "${TAG_ARRAY[@]}" | sed 's/,$//') + JSON_TAGS="[$JSON_TAGS]" + + IFS=' ' + echo "filebeats: $FB_CNT, cpu: $CPU, mem: $MEM, Queue: $QTYPE, worker multiplier: ${MULTIPLIERS[@]}, batch size: ${BATCH_SIZES[@]}" +} + +get_secret() { + VAULT_PATH=${VAULT_PATH:-secret/ci/elastic-logstash/benchmark} + VAULT_DATA=$(vault kv get -format json $VAULT_PATH) + BENCHMARK_ES_HOST=$(echo $VAULT_DATA | jq -r '.data.es_host') + BENCHMARK_ES_USER=$(echo $VAULT_DATA | jq -r '.data.es_user') + BENCHMARK_ES_PW=$(echo $VAULT_DATA | jq -r '.data.es_pw') + + MONITOR_ES_HOST=$(echo $VAULT_DATA | jq -r '.data.monitor_es_host') + MONITOR_ES_USER=$(echo $VAULT_DATA | jq -r '.data.monitor_es_user') + MONITOR_ES_PW=$(echo $VAULT_DATA | jq -r '.data.monitor_es_pw') +} + +pull_images() { + echo "--- Pull docker images" + + if [[ -n "$LS_VERSION" ]]; then + # pull image if it doesn't exist in local + [[ -z $(docker images -q docker.elastic.co/logstash/logstash:$LS_VERSION) ]] && docker pull "docker.elastic.co/logstash/logstash:$LS_VERSION" + else + # pull the latest snapshot logstash image + # select the SNAPSHOT artifact with the highest semantic version number + LS_VERSION=$( curl --retry-all-errors --retry 5 --retry-delay 1 -s https://artifacts-api.elastic.co/v1/versions | jq -r '.versions | map(select(endswith("-SNAPSHOT"))) | max_by(rtrimstr("-SNAPSHOT")|split(".")|map(tonumber))' ) + BUILD_ID=$(curl --retry-all-errors --retry 5 --retry-delay 1 -s "https://artifacts-api.elastic.co/v1/versions/${LS_VERSION}/builds/latest" | jq -re '.build.build_id') + ARCH=$(arch) + IMAGE_URL="https://snapshots.elastic.co/${BUILD_ID}/downloads/logstash/logstash-$LS_VERSION-docker-image-$ARCH.tar.gz" + IMAGE_FILENAME="$LS_VERSION.tar.gz" + + echo "Download $LS_VERSION from $IMAGE_URL" + [[ ! -e $IMAGE_FILENAME ]] && curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o "$IMAGE_FILENAME" "$IMAGE_URL" + [[ -z $(docker images -q docker.elastic.co/logstash/logstash:$LS_VERSION) ]] && docker load -i "$IMAGE_FILENAME" + fi + + # pull filebeat image + FB_DEFAULT_VERSION="8.13.4" + FB_VERSION=${FB_VERSION:-$FB_DEFAULT_VERSION} + docker pull "docker.elastic.co/beats/filebeat:$FB_VERSION" +} + +generate_logs() { + FLOG_FILE_CNT=${FLOG_FILE_CNT:-4} + SINGLE_SIZE=524288000 + TOTAL_SIZE="$((FLOG_FILE_CNT * SINGLE_SIZE))" + FLOG_PATH="$SCRIPT_PATH/flog" + mkdir -p $FLOG_PATH + + if [[ ! -e "$FLOG_PATH/log${FLOG_FILE_CNT}.log" ]]; then + echo "--- Generate logs in background. log: ${FLOG_FILE_CNT}, each size: 500mb" + docker run -d --name=flog --rm -v $FLOG_PATH:/go/src/data mingrammer/flog -t log -w -o "/go/src/data/log.log" -b $TOTAL_SIZE -p $SINGLE_SIZE + fi +} + +check_logs() { + echo "--- Check log generation" + + local cnt=0 + until [[ -e "$FLOG_PATH/log${FLOG_FILE_CNT}.log" || $cnt -gt 600 ]]; do + echo "wait 30s" && sleep 30 + cnt=$((cnt + 30)) + done + + ls -lah $FLOG_PATH +} + +start_logstash() { + LS_CONFIG_PATH=$SCRIPT_PATH/ls/config + mkdir -p $LS_CONFIG_PATH + + cp $CONFIG_PATH/pipelines.yml $LS_CONFIG_PATH/pipelines.yml + cp $CONFIG_PATH/logstash.yml $LS_CONFIG_PATH/logstash.yml + cp $CONFIG_PATH/uuid $LS_CONFIG_PATH/uuid + + LS_JAVA_OPTS=${LS_JAVA_OPTS:--Xmx${XMX}g} + docker run -d --name=ls --net=host --cpus=$CPU --memory=${MEM}g -e LS_JAVA_OPTS="$LS_JAVA_OPTS" \ + -e QTYPE="$QTYPE" -e WORKER="$WORKER" -e BATCH_SIZE="$BATCH_SIZE" \ + -e BENCHMARK_ES_HOST="$BENCHMARK_ES_HOST" -e BENCHMARK_ES_USER="$BENCHMARK_ES_USER" -e BENCHMARK_ES_PW="$BENCHMARK_ES_PW" \ + -e MONITOR_ES_HOST="$MONITOR_ES_HOST" -e MONITOR_ES_USER="$MONITOR_ES_USER" -e MONITOR_ES_PW="$MONITOR_ES_PW" \ + -v $LS_CONFIG_PATH/logstash.yml:/usr/share/logstash/config/logstash.yml:ro \ + -v $LS_CONFIG_PATH/pipelines.yml:/usr/share/logstash/config/pipelines.yml:ro \ + -v $LS_CONFIG_PATH/uuid:/usr/share/logstash/data/uuid:ro \ + docker.elastic.co/logstash/logstash:$LS_VERSION +} + +start_filebeat() { + for ((i = 0; i < FB_CNT; i++)); do + FB_PATH="$SCRIPT_PATH/fb${i}" + mkdir -p $FB_PATH + + cp $CONFIG_PATH/filebeat.yml $FB_PATH/filebeat.yml + + docker run -d --name=fb$i --net=host --user=root \ + -v $FB_PATH/filebeat.yml:/usr/share/filebeat/filebeat.yml \ + -v $SCRIPT_PATH/flog:/usr/share/filebeat/flog \ + docker.elastic.co/beats/filebeat:$FB_VERSION filebeat -e --strict.perms=false + done +} + +capture_stats() { + CURRENT=$(jq -r '.flow.output_throughput.current' $NS_JSON) + local eps_1m=$(jq -r '.flow.output_throughput.last_1_minute' $NS_JSON) + local eps_5m=$(jq -r '.flow.output_throughput.last_5_minutes' $NS_JSON) + local worker_util=$(jq -r '.pipelines.main.flow.worker_utilization.last_1_minute' $NS_JSON) + local worker_concurr=$(jq -r '.pipelines.main.flow.worker_concurrency.last_1_minute' $NS_JSON) + local cpu_percent=$(jq -r '.process.cpu.percent' $NS_JSON) + local heap=$(jq -r '.jvm.mem.heap_used_in_bytes' $NS_JSON) + local non_heap=$(jq -r '.jvm.mem.non_heap_used_in_bytes' $NS_JSON) + local q_event_cnt=$(jq -r '.pipelines.main.queue.events_count' $NS_JSON) + local q_size=$(jq -r '.pipelines.main.queue.queue_size_in_bytes' $NS_JSON) + TOTAL_EVENTS_OUT=$(jq -r '.pipelines.main.events.out' $NS_JSON) + printf "current: %s, 1m: %s, 5m: %s, worker_utilization: %s, worker_concurrency: %s, cpu: %s, heap: %s, non-heap: %s, q_events: %s, q_size: %s, total_events_out: %s\n" \ + $CURRENT $eps_1m $eps_5m $worker_util $worker_concurr $cpu_percent $heap $non_heap $q_event_cnt $q_size $TOTAL_EVENTS_OUT +} + +aggregate_stats() { + local file_glob="$SCRIPT_PATH/$NS_DIR/${QTYPE:0:1}_w${WORKER}b${BATCH_SIZE}_*.json" + MAX_EPS_1M=$( jqmax '.flow.output_throughput.last_1_minute' "$file_glob" ) + MAX_EPS_5M=$( jqmax '.flow.output_throughput.last_5_minutes' "$file_glob" ) + MAX_WORKER_UTIL=$( jqmax '.pipelines.main.flow.worker_utilization.last_1_minute' "$file_glob" ) + MAX_WORKER_CONCURR=$( jqmax '.pipelines.main.flow.worker_concurrency.last_1_minute' "$file_glob" ) + MAX_Q_EVENT_CNT=$( jqmax '.pipelines.main.queue.events_count' "$file_glob" ) + MAX_Q_SIZE=$( jqmax '.pipelines.main.queue.queue_size_in_bytes' "$file_glob" ) + + AVG_CPU_PERCENT=$( jqavg '.process.cpu.percent' "$file_glob" ) + AVG_VIRTUAL_MEM=$( jqavg '.process.mem.total_virtual_in_bytes' "$file_glob" ) + AVG_HEAP=$( jqavg '.jvm.mem.heap_used_in_bytes' "$file_glob" ) + AVG_NON_HEAP=$( jqavg '.jvm.mem.non_heap_used_in_bytes' "$file_glob" ) +} + +send_summary() { + echo "--- Send summary to Elasticsearch" + + # build json + local timestamp + timestamp=$(date -u +"%Y-%m-%dT%H:%M:%S") + SUMMARY="{\"timestamp\": \"$timestamp\", \"version\": \"$LS_VERSION\", \"cpu\": \"$CPU\", \"mem\": \"$MEM\", \"workers\": \"$WORKER\", \"batch_size\": \"$BATCH_SIZE\", \"queue_type\": \"$QTYPE\"" + not_empty "$TOTAL_EVENTS_OUT" && SUMMARY="$SUMMARY, \"total_events_out\": \"$TOTAL_EVENTS_OUT\"" + not_empty "$MAX_EPS_1M" && SUMMARY="$SUMMARY, \"max_eps_1m\": \"$MAX_EPS_1M\"" + not_empty "$MAX_EPS_5M" && SUMMARY="$SUMMARY, \"max_eps_5m\": \"$MAX_EPS_5M\"" + not_empty "$MAX_WORKER_UTIL" && SUMMARY="$SUMMARY, \"max_worker_utilization\": \"$MAX_WORKER_UTIL\"" + not_empty "$MAX_WORKER_CONCURR" && SUMMARY="$SUMMARY, \"max_worker_concurrency\": \"$MAX_WORKER_CONCURR\"" + not_empty "$AVG_CPU_PERCENT" && SUMMARY="$SUMMARY, \"avg_cpu_percentage\": \"$AVG_CPU_PERCENT\"" + not_empty "$AVG_HEAP" && SUMMARY="$SUMMARY, \"avg_heap\": \"$AVG_HEAP\"" + not_empty "$AVG_NON_HEAP" && SUMMARY="$SUMMARY, \"avg_non_heap\": \"$AVG_NON_HEAP\"" + not_empty "$AVG_VIRTUAL_MEM" && SUMMARY="$SUMMARY, \"avg_virtual_memory\": \"$AVG_VIRTUAL_MEM\"" + not_empty "$MAX_Q_EVENT_CNT" && SUMMARY="$SUMMARY, \"max_queue_events\": \"$MAX_Q_EVENT_CNT\"" + not_empty "$MAX_Q_SIZE" && SUMMARY="$SUMMARY, \"max_queue_bytes_size\": \"$MAX_Q_SIZE\"" + not_empty "$TAGS" && SUMMARY="$SUMMARY, \"tags\": $JSON_TAGS" + SUMMARY="$SUMMARY}" + + tee summary.json << EOF +{"index": {}} +$SUMMARY +EOF + + # send to ES + local resp + local err_status + resp=$(curl -s -X POST -u "$BENCHMARK_ES_USER:$BENCHMARK_ES_PW" "$BENCHMARK_ES_HOST/benchmark_summary/_bulk" -H 'Content-Type: application/json' --data-binary @"summary.json") + echo "$resp" + err_status=$(echo "$resp" | jq -r ".errors") + if [[ "$err_status" == "true" ]]; then + echo "Failed to send summary" + exit 1 + fi +} + +# $1: snapshot index +node_stats() { + NS_JSON="$SCRIPT_PATH/$NS_DIR/${QTYPE:0:1}_w${WORKER}b${BATCH_SIZE}_$1.json" # m_w8b1000_0.json + + # curl inside container because docker on mac cannot resolve localhost to host network interface + docker exec -i ls curl localhost:9600/_node/stats > "$NS_JSON" 2> /dev/null +} + +# $1: index +snapshot() { + node_stats $1 + capture_stats +} + +create_directory() { + NS_DIR="fb${FB_CNT}c${CPU}m${MEM}" # fb4c4m4 + mkdir -p "$SCRIPT_PATH/$NS_DIR" +} + +queue() { + for QTYPE in "persisted" "memory"; do + worker + done +} + +worker() { + for m in "${MULTIPLIERS[@]}"; do + WORKER=$((CPU * m)) + batch + done +} + +batch() { + for BATCH_SIZE in "${BATCH_SIZES[@]}"; do + run_pipeline + stop_pipeline + done +} + +run_pipeline() { + echo "--- Run pipeline. queue type: $QTYPE, worker: $WORKER, batch size: $BATCH_SIZE" + + start_logstash + start_filebeat + docker ps + + echo "(0) sleep 3m" && sleep 180 + snapshot "0" + + for i in {1..8}; do + echo "($i) sleep 30s" && sleep 30 + snapshot "$i" + + # print docker log when ingestion rate is zero + # remove '.' in number and return max val + [[ $(max -g "${CURRENT/./}" "0") -eq 0 ]] && + docker logs fb0 && + docker logs ls + done + + aggregate_stats + send_summary +} + +stop_pipeline() { + echo "--- Stop Pipeline" + + for ((i = 0; i < FB_CNT; i++)); do + docker stop fb$i + docker rm fb$i + done + + docker stop ls + docker rm ls + + curl -u "$BENCHMARK_ES_USER:$BENCHMARK_ES_PW" -X DELETE $BENCHMARK_ES_HOST/_data_stream/logs-generic-default + echo " data stream deleted " + + # TODO: clean page caches, reduce memory fragmentation + # https://github.com/elastic/logstash/pull/16191#discussion_r1647050216 +} + +clean_up() { + # stop log generation if it has not done yet + [[ -n $(docker ps | grep flog) ]] && docker stop flog || true + # remove image + docker image rm docker.elastic.co/logstash/logstash:$LS_VERSION +} diff --git a/.buildkite/scripts/benchmark/main.sh b/.buildkite/scripts/benchmark/main.sh index 9534ce3fc9a..3d615fcb41b 100755 --- a/.buildkite/scripts/benchmark/main.sh +++ b/.buildkite/scripts/benchmark/main.sh @@ -15,9 +15,8 @@ set -eo pipefail # - The script sends a summary of EPS and resource usage to index `benchmark_summary` # ******************************************************* -SCRIPT_PATH="$(cd "$(dirname "$0")"; pwd)" -CONFIG_PATH="$SCRIPT_PATH/config" -source "$SCRIPT_PATH/util.sh" +SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")" +source "$SCRIPT_PATH/core.sh" ## usage: ## main.sh FB_CNT QTYPE CPU MEM @@ -39,315 +38,6 @@ source "$SCRIPT_PATH/util.sh" ## FLOG_FILE_CNT=4 # number of files to generate for ingestion ## VAULT_PATH=secret/path # vault path point to Elasticsearch credentials. The default value points to benchmark cluster. ## TAGS=test,other # tags with "," separator. -usage() { - echo "Usage: $0 [FB_CNT] [QTYPE] [CPU] [MEM]" - echo "Example: $0 4 {persisted|memory|all} 2 2" - exit 1 -} - -parse_args() { - while [[ "$#" -gt 0 ]]; do - if [ -z "$FB_CNT" ]; then - FB_CNT=$1 - elif [ -z "$QTYPE" ]; then - case $1 in - all | persisted | memory) - QTYPE=$1 - ;; - *) - echo "Error: wrong queue type $1" - usage - ;; - esac - elif [ -z "$CPU" ]; then - CPU=$1 - elif [ -z "$MEM" ]; then - MEM=$1 - else - echo "Error: Too many arguments" - usage - fi - shift - done - - # set default value - # number of filebeat - FB_CNT=${FB_CNT:-4} - # all | persisted | memory - QTYPE=${QTYPE:-all} - CPU=${CPU:-4} - MEM=${MEM:-4} - XMX=$((MEM / 2)) - - IFS=',' - # worker multiplier: 1,2,4 - MULTIPLIERS="${MULTIPLIERS:-1,2,4}" - read -ra MULTIPLIERS <<< "$MULTIPLIERS" - BATCH_SIZES="${BATCH_SIZES:-500}" - read -ra BATCH_SIZES <<< "$BATCH_SIZES" - # tags to json array - read -ra TAG_ARRAY <<< "$TAGS" - JSON_TAGS=$(printf '"%s",' "${TAG_ARRAY[@]}" | sed 's/,$//') - JSON_TAGS="[$JSON_TAGS]" - - IFS=' ' - echo "filebeats: $FB_CNT, cpu: $CPU, mem: $MEM, Queue: $QTYPE, worker multiplier: ${MULTIPLIERS[@]}, batch size: ${BATCH_SIZES[@]}" -} - -get_secret() { - VAULT_PATH=${VAULT_PATH:-secret/ci/elastic-logstash/benchmark} - VAULT_DATA=$(vault kv get -format json $VAULT_PATH) - BENCHMARK_ES_HOST=$(echo $VAULT_DATA | jq -r '.data.es_host') - BENCHMARK_ES_USER=$(echo $VAULT_DATA | jq -r '.data.es_user') - BENCHMARK_ES_PW=$(echo $VAULT_DATA | jq -r '.data.es_pw') - - MONITOR_ES_HOST=$(echo $VAULT_DATA | jq -r '.data.monitor_es_host') - MONITOR_ES_USER=$(echo $VAULT_DATA | jq -r '.data.monitor_es_user') - MONITOR_ES_PW=$(echo $VAULT_DATA | jq -r '.data.monitor_es_pw') -} - -pull_images() { - echo "--- Pull docker images" - - if [[ -n "$LS_VERSION" ]]; then - # pull image if it doesn't exist in local - [[ -z $(docker images -q docker.elastic.co/logstash/logstash:$LS_VERSION) ]] && docker pull "docker.elastic.co/logstash/logstash:$LS_VERSION" - else - # pull the latest snapshot logstash image - # select the SNAPSHOT artifact with the highest semantic version number - LS_VERSION=$( curl --retry-all-errors --retry 5 --retry-delay 1 -s https://artifacts-api.elastic.co/v1/versions | jq -r '.versions | map(select(endswith("-SNAPSHOT"))) | max_by(rtrimstr("-SNAPSHOT")|split(".")|map(tonumber))' ) - BUILD_ID=$(curl --retry-all-errors --retry 5 --retry-delay 1 -s "https://artifacts-api.elastic.co/v1/versions/${LS_VERSION}/builds/latest" | jq -re '.build.build_id') - ARCH=$(arch) - IMAGE_URL="https://snapshots.elastic.co/${BUILD_ID}/downloads/logstash/logstash-$LS_VERSION-docker-image-$ARCH.tar.gz" - IMAGE_FILENAME="$LS_VERSION.tar.gz" - - echo "Download $LS_VERSION from $IMAGE_URL" - [[ ! -e $IMAGE_FILENAME ]] && curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o "$IMAGE_FILENAME" "$IMAGE_URL" - [[ -z $(docker images -q docker.elastic.co/logstash/logstash:$LS_VERSION) ]] && docker load -i "$IMAGE_FILENAME" - fi - - # pull filebeat image - FB_DEFAULT_VERSION="8.13.4" - FB_VERSION=${FB_VERSION:-$FB_DEFAULT_VERSION} - docker pull "docker.elastic.co/beats/filebeat:$FB_VERSION" -} - -generate_logs() { - FLOG_FILE_CNT=${FLOG_FILE_CNT:-4} - SINGLE_SIZE=524288000 - TOTAL_SIZE="$((FLOG_FILE_CNT * SINGLE_SIZE))" - FLOG_PATH="$SCRIPT_PATH/flog" - mkdir -p $FLOG_PATH - - if [[ ! -e "$FLOG_PATH/log${FLOG_FILE_CNT}.log" ]]; then - echo "--- Generate logs in background. log: ${FLOG_FILE_CNT}, each size: 500mb" - docker run -d --name=flog --rm -v $FLOG_PATH:/go/src/data mingrammer/flog -t log -w -o "/go/src/data/log.log" -b $TOTAL_SIZE -p $SINGLE_SIZE - fi -} - -check_logs() { - echo "--- Check log generation" - - local cnt=0 - until [[ -e "$FLOG_PATH/log${FLOG_FILE_CNT}.log" || $cnt -gt 600 ]]; do - echo "wait 30s" && sleep 30 - cnt=$((cnt + 30)) - done - - ls -lah $FLOG_PATH -} - -start_logstash() { - LS_CONFIG_PATH=$SCRIPT_PATH/ls/config - mkdir -p $LS_CONFIG_PATH - - cp $CONFIG_PATH/pipelines.yml $LS_CONFIG_PATH/pipelines.yml - cp $CONFIG_PATH/logstash.yml $LS_CONFIG_PATH/logstash.yml - cp $CONFIG_PATH/uuid $LS_CONFIG_PATH/uuid - - LS_JAVA_OPTS=${LS_JAVA_OPTS:--Xmx${XMX}g} - docker run -d --name=ls --net=host --cpus=$CPU --memory=${MEM}g -e LS_JAVA_OPTS="$LS_JAVA_OPTS" \ - -e QTYPE="$QTYPE" -e WORKER="$WORKER" -e BATCH_SIZE="$BATCH_SIZE" \ - -e BENCHMARK_ES_HOST="$BENCHMARK_ES_HOST" -e BENCHMARK_ES_USER="$BENCHMARK_ES_USER" -e BENCHMARK_ES_PW="$BENCHMARK_ES_PW" \ - -e MONITOR_ES_HOST="$MONITOR_ES_HOST" -e MONITOR_ES_USER="$MONITOR_ES_USER" -e MONITOR_ES_PW="$MONITOR_ES_PW" \ - -v $LS_CONFIG_PATH/logstash.yml:/usr/share/logstash/config/logstash.yml:ro \ - -v $LS_CONFIG_PATH/pipelines.yml:/usr/share/logstash/config/pipelines.yml:ro \ - -v $LS_CONFIG_PATH/uuid:/usr/share/logstash/data/uuid:ro \ - docker.elastic.co/logstash/logstash:$LS_VERSION -} - -start_filebeat() { - for ((i = 0; i < FB_CNT; i++)); do - FB_PATH="$SCRIPT_PATH/fb${i}" - mkdir -p $FB_PATH - - cp $CONFIG_PATH/filebeat.yml $FB_PATH/filebeat.yml - - docker run -d --name=fb$i --net=host --user=root \ - -v $FB_PATH/filebeat.yml:/usr/share/filebeat/filebeat.yml \ - -v $SCRIPT_PATH/flog:/usr/share/filebeat/flog \ - docker.elastic.co/beats/filebeat:$FB_VERSION filebeat -e --strict.perms=false - done -} - -capture_stats() { - CURRENT=$(jq -r '.flow.output_throughput.current' $NS_JSON) - local eps_1m=$(jq -r '.flow.output_throughput.last_1_minute' $NS_JSON) - local eps_5m=$(jq -r '.flow.output_throughput.last_5_minutes' $NS_JSON) - local worker_util=$(jq -r '.pipelines.main.flow.worker_utilization.last_1_minute' $NS_JSON) - local worker_concurr=$(jq -r '.pipelines.main.flow.worker_concurrency.last_1_minute' $NS_JSON) - local cpu_percent=$(jq -r '.process.cpu.percent' $NS_JSON) - local heap=$(jq -r '.jvm.mem.heap_used_in_bytes' $NS_JSON) - local non_heap=$(jq -r '.jvm.mem.non_heap_used_in_bytes' $NS_JSON) - local q_event_cnt=$(jq -r '.pipelines.main.queue.events_count' $NS_JSON) - local q_size=$(jq -r '.pipelines.main.queue.queue_size_in_bytes' $NS_JSON) - TOTAL_EVENTS_OUT=$(jq -r '.pipelines.main.events.out' $NS_JSON) - printf "current: %s, 1m: %s, 5m: %s, worker_utilization: %s, worker_concurrency: %s, cpu: %s, heap: %s, non-heap: %s, q_events: %s, q_size: %s, total_events_out: %s\n" \ - $CURRENT $eps_1m $eps_5m $worker_util $worker_concurr $cpu_percent $heap $non_heap $q_event_cnt $q_size $TOTAL_EVENTS_OUT -} - -aggregate_stats() { - local file_glob="$SCRIPT_PATH/$NS_DIR/${QTYPE:0:1}_w${WORKER}b${BATCH_SIZE}_*.json" - MAX_EPS_1M=$( jqmax '.flow.output_throughput.last_1_minute' "$file_glob" ) - MAX_EPS_5M=$( jqmax '.flow.output_throughput.last_5_minutes' "$file_glob" ) - MAX_WORKER_UTIL=$( jqmax '.pipelines.main.flow.worker_utilization.last_1_minute' "$file_glob" ) - MAX_WORKER_CONCURR=$( jqmax '.pipelines.main.flow.worker_concurrency.last_1_minute' "$file_glob" ) - MAX_Q_EVENT_CNT=$( jqmax '.pipelines.main.queue.events_count' "$file_glob" ) - MAX_Q_SIZE=$( jqmax '.pipelines.main.queue.queue_size_in_bytes' "$file_glob" ) - - AVG_CPU_PERCENT=$( jqavg '.process.cpu.percent' "$file_glob" ) - AVG_VIRTUAL_MEM=$( jqavg '.process.mem.total_virtual_in_bytes' "$file_glob" ) - AVG_HEAP=$( jqavg '.jvm.mem.heap_used_in_bytes' "$file_glob" ) - AVG_NON_HEAP=$( jqavg '.jvm.mem.non_heap_used_in_bytes' "$file_glob" ) -} - -send_summary() { - echo "--- Send summary to Elasticsearch" - - # build json - local timestamp - timestamp=$(date -u +"%Y-%m-%dT%H:%M:%S") - SUMMARY="{\"timestamp\": \"$timestamp\", \"version\": \"$LS_VERSION\", \"cpu\": \"$CPU\", \"mem\": \"$MEM\", \"workers\": \"$WORKER\", \"batch_size\": \"$BATCH_SIZE\", \"queue_type\": \"$QTYPE\"" - not_empty "$TOTAL_EVENTS_OUT" && SUMMARY="$SUMMARY, \"total_events_out\": \"$TOTAL_EVENTS_OUT\"" - not_empty "$MAX_EPS_1M" && SUMMARY="$SUMMARY, \"max_eps_1m\": \"$MAX_EPS_1M\"" - not_empty "$MAX_EPS_5M" && SUMMARY="$SUMMARY, \"max_eps_5m\": \"$MAX_EPS_5M\"" - not_empty "$MAX_WORKER_UTIL" && SUMMARY="$SUMMARY, \"max_worker_utilization\": \"$MAX_WORKER_UTIL\"" - not_empty "$MAX_WORKER_CONCURR" && SUMMARY="$SUMMARY, \"max_worker_concurrency\": \"$MAX_WORKER_CONCURR\"" - not_empty "$AVG_CPU_PERCENT" && SUMMARY="$SUMMARY, \"avg_cpu_percentage\": \"$AVG_CPU_PERCENT\"" - not_empty "$AVG_HEAP" && SUMMARY="$SUMMARY, \"avg_heap\": \"$AVG_HEAP\"" - not_empty "$AVG_NON_HEAP" && SUMMARY="$SUMMARY, \"avg_non_heap\": \"$AVG_NON_HEAP\"" - not_empty "$AVG_VIRTUAL_MEM" && SUMMARY="$SUMMARY, \"avg_virtual_memory\": \"$AVG_VIRTUAL_MEM\"" - not_empty "$MAX_Q_EVENT_CNT" && SUMMARY="$SUMMARY, \"max_queue_events\": \"$MAX_Q_EVENT_CNT\"" - not_empty "$MAX_Q_SIZE" && SUMMARY="$SUMMARY, \"max_queue_bytes_size\": \"$MAX_Q_SIZE\"" - not_empty "$TAGS" && SUMMARY="$SUMMARY, \"tags\": $JSON_TAGS" - SUMMARY="$SUMMARY}" - - tee summary.json << EOF -{"index": {}} -$SUMMARY -EOF - - # send to ES - local resp - local err_status - resp=$(curl -s -X POST -u "$BENCHMARK_ES_USER:$BENCHMARK_ES_PW" "$BENCHMARK_ES_HOST/benchmark_summary/_bulk" -H 'Content-Type: application/json' --data-binary @"summary.json") - echo "$resp" - err_status=$(echo "$resp" | jq -r ".errors") - if [[ "$err_status" == "true" ]]; then - echo "Failed to send summary" - exit 1 - fi -} - -# $1: snapshot index -node_stats() { - NS_JSON="$SCRIPT_PATH/$NS_DIR/${QTYPE:0:1}_w${WORKER}b${BATCH_SIZE}_$1.json" # m_w8b1000_0.json - - # curl inside container because docker on mac cannot resolve localhost to host network interface - docker exec -i ls curl localhost:9600/_node/stats > "$NS_JSON" 2> /dev/null -} - -# $1: index -snapshot() { - node_stats $1 - capture_stats -} - -create_directory() { - NS_DIR="fb${FB_CNT}c${CPU}m${MEM}" # fb4c4m4 - mkdir -p "$SCRIPT_PATH/$NS_DIR" -} - -queue() { - for QTYPE in "persisted" "memory"; do - worker - done -} - -worker() { - for m in "${MULTIPLIERS[@]}"; do - WORKER=$((CPU * m)) - batch - done -} - -batch() { - for BATCH_SIZE in "${BATCH_SIZES[@]}"; do - run_pipeline - stop_pipeline - done -} - -run_pipeline() { - echo "--- Run pipeline. queue type: $QTYPE, worker: $WORKER, batch size: $BATCH_SIZE" - - start_logstash - start_filebeat - docker ps - - echo "(0) sleep 3m" && sleep 180 - snapshot "0" - - for i in {1..8}; do - echo "($i) sleep 30s" && sleep 30 - snapshot "$i" - - # print docker log when ingestion rate is zero - # remove '.' in number and return max val - [[ $(max -g "${CURRENT/./}" "0") -eq 0 ]] && - docker logs fb0 && - docker logs ls - done - - aggregate_stats - send_summary -} - -stop_pipeline() { - echo "--- Stop Pipeline" - - for ((i = 0; i < FB_CNT; i++)); do - docker stop fb$i - docker rm fb$i - done - - docker stop ls - docker rm ls - - curl -u "$BENCHMARK_ES_USER:$BENCHMARK_ES_PW" -X DELETE $BENCHMARK_ES_HOST/_data_stream/logs-generic-default - echo " data stream deleted " - - # TODO: clean page caches, reduce memory fragmentation - # https://github.com/elastic/logstash/pull/16191#discussion_r1647050216 -} - -clean_up() { - # stop log generation if it has not done yet - [[ -n $(docker ps | grep flog) ]] && docker stop flog || true - # remove image - docker image rm docker.elastic.co/logstash/logstash:$LS_VERSION -} - main() { parse_args "$@" get_secret @@ -366,4 +56,4 @@ main() { clean_up } -main "$@" +main "$@" \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/marathon.sh b/.buildkite/scripts/benchmark/marathon.sh new file mode 100755 index 00000000000..4878a2d3959 --- /dev/null +++ b/.buildkite/scripts/benchmark/marathon.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -eo pipefail + +# ******************************************************* +# Run benchmark for versions that have flow metrics +# When the hardware changes, run the marathon task to establish a new baseline. +# Usage: +# nohup bash -x all.sh > log.log 2>&1 & +# Accept env vars: +# STACK_VERSIONS=8.15.0,8.15.1,8.16.0-SNAPSHOT # versions to test. It is comma separator string +# ******************************************************* + +SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")" +source "$SCRIPT_PATH/core.sh" + +parse_stack_versions() { + IFS=',' + STACK_VERSIONS="${STACK_VERSIONS:-8.6.0,8.7.0,8.8.0,8.9.0,8.10.0,8.11.0,8.12.0,8.13.0,8.14.0,8.15.0}" + read -ra STACK_VERSIONS <<< "$STACK_VERSIONS" +} + +main() { + parse_stack_versions + parse_args "$@" + get_secret + generate_logs + check_logs + + for V in "${STACK_VERSIONS[@]}" ; do + LS_VERSION="$V" + pull_images + create_directory + if [[ $QTYPE == "all" ]]; then + queue + else + worker + fi + done +} + +main "$@" \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/save-objects/CHANGELOG.md b/.buildkite/scripts/benchmark/save-objects/CHANGELOG.md new file mode 100644 index 00000000000..335acb0ac0c --- /dev/null +++ b/.buildkite/scripts/benchmark/save-objects/CHANGELOG.md @@ -0,0 +1,5 @@ +## 20240912 +Updated runtime field `release` to return `true` when `version` contains "SNAPSHOT" + +## 20240912 +Initial dashboards \ No newline at end of file diff --git a/.buildkite/scripts/benchmark/save-objects/benchmark_objects.ndjson b/.buildkite/scripts/benchmark/save-objects/benchmark_objects.ndjson index e68a8fc77a4..ef4511357d3 100644 --- a/.buildkite/scripts/benchmark/save-objects/benchmark_objects.ndjson +++ b/.buildkite/scripts/benchmark/save-objects/benchmark_objects.ndjson @@ -1,4 +1,4 @@ -{"attributes":{"allowHidden":false,"fieldAttrs":"{\"5m_num\":{\"customLabel\":\"\"},\"avg_heap_num\":{\"customLabel\":\"\"},\"avg_non_heap_num\":{\"customLabel\":\"\"},\"max_worker_utilization_num\":{\"customLabel\":\"\"},\"max_worker_concurrency_num\":{\"customLabel\":\"\"},\"max_queue_events\":{\"customLabel\":\"\"},\"max_queue_events_num\":{\"customLabel\":\"\"},\"total_events_out_num\":{\"customLabel\":\"\"},\"avg_cpu_percentage_num\":{\"customLabel\":\"\"},\"release\":{},\"version_num\":{}}","fieldFormatMap":"{\"5m_num\":{\"id\":\"number\",\"params\":{\"pattern\":\"0\"}},\"avg_heap_num\":{\"id\":\"number\"},\"avg_non_heap_num\":{\"id\":\"number\"},\"max_worker_utilization_num\":{\"id\":\"number\"},\"max_worker_concurrency_num\":{\"id\":\"number\"},\"max_queue_events\":{\"id\":\"number\"},\"max_queue_events_num\":{\"id\":\"number\"},\"avg_cpu_percentage_num\":{\"id\":\"number\"}}","fields":"[{\"name\":\"5m_num\",\"script\":\"return Integer.parseInt(doc['max_eps_5m.keyword'].value)\",\"lang\":\"painless\",\"type\":\"number\",\"scripted\":true,\"customLabel\":\"\"}]","name":"benchmark","runtimeFieldMap":"{\"release\":{\"type\":\"boolean\",\"script\":{\"source\":\"def version = doc['version.keyword'].value;\\nif (version.endsWith(\\\"-SNAPSHOT\\\")) {\\n\\temit(false);\\n}\\nelse {\\n\\temit(true);\\n}\"}},\"version_num\":{\"type\":\"long\",\"script\":{\"source\":\"if (doc.containsKey('version.keyword') && !doc['version.keyword'].empty) {\\n // Get the version string\\n String version = doc['version.keyword'].value;\\n \\n // Remove any suffix after the patch number (e.g., -SNAPSHOT)\\n int dashIndex = version.indexOf('-');\\n if (dashIndex != -1) {\\n version = version.substring(0, dashIndex);\\n }\\n \\n // Find positions of dots\\n int firstDot = version.indexOf('.');\\n int secondDot = version.indexOf('.', firstDot + 1);\\n \\n // Extract major, minor, and patch parts of the version\\n int major = Integer.parseInt(version.substring(0, firstDot));\\n int minor = Integer.parseInt(version.substring(firstDot + 1, secondDot));\\n int patch = Integer.parseInt(version.substring(secondDot + 1));\\n \\n // Combine into a sortable numeric value\\n emit(major * 10000 + minor * 100 + patch);\\n}\\n\\nemit(0);\"}}}","sourceFilters":"[]","timeFieldName":"timestamp","title":"benchmark_*"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"c5493557-3fd6-4929-8dc2-40d248303942","managed":false,"references":[],"type":"index-pattern","typeMigrationVersion":"8.0.0","updated_at":"2024-09-05T11:52:42.820Z","version":"WzcyNiw3XQ=="} +{"attributes":{"allowHidden":false,"fieldAttrs":"{\"5m_num\":{\"customLabel\":\"\"},\"avg_heap_num\":{\"customLabel\":\"\"},\"avg_non_heap_num\":{\"customLabel\":\"\"},\"max_worker_utilization_num\":{\"customLabel\":\"\"},\"max_worker_concurrency_num\":{\"customLabel\":\"\"},\"max_queue_events\":{\"customLabel\":\"\"},\"max_queue_events_num\":{\"customLabel\":\"\"},\"total_events_out_num\":{\"customLabel\":\"\"},\"avg_cpu_percentage_num\":{\"customLabel\":\"\"},\"release\":{},\"version_num\":{}}","fieldFormatMap":"{\"5m_num\":{\"id\":\"number\",\"params\":{\"pattern\":\"0\"}},\"avg_heap_num\":{\"id\":\"number\"},\"avg_non_heap_num\":{\"id\":\"number\"},\"max_worker_utilization_num\":{\"id\":\"number\"},\"max_worker_concurrency_num\":{\"id\":\"number\"},\"max_queue_events\":{\"id\":\"number\"},\"max_queue_events_num\":{\"id\":\"number\"},\"avg_cpu_percentage_num\":{\"id\":\"number\"}}","fields":"[{\"name\":\"5m_num\",\"script\":\"return Integer.parseInt(doc['max_eps_5m.keyword'].value)\",\"lang\":\"painless\",\"type\":\"number\",\"scripted\":true,\"customLabel\":\"\"}]","name":"benchmark","runtimeFieldMap":"{\"release\":{\"type\":\"boolean\",\"script\":{\"source\":\"def version = doc['version.keyword'].value;\\nif (version.contains(\\\"SNAPSHOT\\\")) {\\n\\temit(false);\\n}\\nelse {\\n\\temit(true);\\n}\"}},\"version_num\":{\"type\":\"long\",\"script\":{\"source\":\"if (doc.containsKey('version.keyword') && !doc['version.keyword'].empty) {\\n // Get the version string\\n String version = doc['version.keyword'].value;\\n \\n // Remove any suffix after the patch number (e.g., -SNAPSHOT)\\n int dashIndex = version.indexOf('-');\\n if (dashIndex != -1) {\\n version = version.substring(0, dashIndex);\\n }\\n \\n // Find positions of dots\\n int firstDot = version.indexOf('.');\\n int secondDot = version.indexOf('.', firstDot + 1);\\n \\n // Extract major, minor, and patch parts of the version\\n int major = Integer.parseInt(version.substring(0, firstDot));\\n int minor = Integer.parseInt(version.substring(firstDot + 1, secondDot));\\n int patch = Integer.parseInt(version.substring(secondDot + 1));\\n \\n // Combine into a sortable numeric value\\n emit(major * 10000 + minor * 100 + patch);\\n}\\n\\nemit(0);\"}}}","sourceFilters":"[]","timeFieldName":"timestamp","title":"benchmark_*"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"c5493557-3fd6-4929-8dc2-40d248303942","managed":false,"references":[],"type":"index-pattern","typeMigrationVersion":"8.0.0","updated_at":"2024-09-12T20:34:35.037Z","updated_by":"u_4132377203_cloud","version":"WzE1NjYsOF0="} {"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"999c9564-cba1-4f3b-af99-ccbacb8e034f":{"columnOrder":["a58f8c48-c2a7-4fc6-b54e-716bf4ca001e","9ddc0f02-5530-4930-99bd-2a914ada1b8f","be033d18-2c04-44f2-b403-0b88d888143d"],"columns":{"9ddc0f02-5530-4930-99bd-2a914ada1b8f":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e":{"dataType":"string","isBucketed":true,"label":"Top values of workers.keyword + 1 other","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"be033d18-2c04-44f2-b403-0b88d888143d","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"multi_terms"},"secondaryFields":["queue_type.keyword"],"size":100},"scale":"ordinal","sourceField":"workers.keyword"},"be033d18-2c04-44f2-b403-0b88d888143d":{"dataType":"number","isBucketed":false,"label":"Median of 5m_num","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"5m_num"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["be033d18-2c04-44f2-b403-0b88d888143d"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["4","memory"]]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["8","memory"]]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["16","memory"]]},"touched":false},{"color":{"colorCode":"#5e6c94","type":"colorCode"},"rule":{"type":"matchExactly","values":[["8","persisted"]]},"touched":true},{"color":{"colorIndex":4,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":[["16","persisted"]]},"touched":false},{"color":{"colorCode":"#a0df7b","type":"colorCode"},"rule":{"type":"matchExactly","values":[["4","persisted"]]},"touched":true}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"999c9564-cba1-4f3b-af99-ccbacb8e034f","layerType":"data","seriesType":"line","splitAccessor":"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e","xAccessor":"9ddc0f02-5530-4930-99bd-2a914ada1b8f"}],"legend":{"isVisible":true,"position":"right"},"preferredSeriesType":"line","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","yTitle":"5m eps"}},"title":"Release EPS by worker & queue","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:53:29.581Z","id":"2c017667-7403-402c-b027-3cc790945a27","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-999c9564-cba1-4f3b-af99-ccbacb8e034f","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:53:29.581Z","version":"WzEzOTMsN10="} {"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_concurrency","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_concurrency"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"7edb6dc9-8b2b-47ec-be64-bae2a4958cc9","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_concurrency"}},"title":"[MQ][R] Worker Concurrency by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:53:13.413Z","id":"9546a73b-5937-4d3d-850f-30625d6e1405","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"7edb6dc9-8b2b-47ec-be64-bae2a4958cc9","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:53:13.413Z","version":"WzEzOTEsN10="} {"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"version","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of total_events_out","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"total_events_out"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"2786f33f-2ad1-4722-8706-9cba9613662e","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"total events out"}},"title":"[MQ][R] total events out by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-11T09:53:00.945Z","id":"6fba7dff-2564-450b-b885-8c6cf97fa727","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"2786f33f-2ad1-4722-8706-9cba9613662e","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-11T09:53:00.945Z","version":"WzEzODksN10="} @@ -17,8 +17,8 @@ {"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","16dcf8b0-378f-48df-8bb3-f537324eea46","cc8c1413-0ccc-4109-a6c3-0c910eff776e","0ad565fc-0a55-492b-ba69-1be57f501e13"],"columns":{"0ad565fc-0a55-492b-ba69-1be57f501e13":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"non_heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_non_heap"},"16dcf8b0-378f-48df-8bb3-f537324eea46":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"customLabel":true,"dataType":"number","isBucketed":false,"label":"heap","operationType":"median","params":{"emptyAsNull":true,"format":{"id":"bytes","params":{"decimals":2}}},"scale":"ratio","sourceField":"avg_heap"},"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3":{"dataType":"number","isBucketed":true,"label":"Top 10 values of workers","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"secondaryFields":[],"size":10},"scale":"ordinal","sourceField":"workers"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"962f0beb-0dd8-4449-a652-878d0d2c63e8","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e","0ad565fc-0a55-492b-ba69-1be57f501e13"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"e7fa90f9-5d12-46d7-a62d-d6a77e6b2bd3","xAccessor":"16dcf8b0-378f-48df-8bb3-f537324eea46"}],"legend":{"horizontalAlignment":"left","isInside":false,"isVisible":true,"position":"right","verticalAlignment":"bottom"},"preferredSeriesType":"line","tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","yTitle":"heap / non heap"}},"title":"[PQ] memory by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"4d1f7bef-c184-4457-b159-465128d2d70b","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"962f0beb-0dd8-4449-a652-878d0d2c63e8","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"WzczOSw3XQ=="} {"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_concurrency","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_concurrency"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"80e5b5e1-481a-478e-b5cd-3ad74106ff85","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_concurrency"}},"title":"[PQ] Worker Concurrency by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"f233b0e6-8ca5-4c29-a6b3-1f1bc83509fd","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"80e5b5e1-481a-478e-b5cd-3ad74106ff85","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0Miw3XQ=="} {"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of total_events_out","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"total_events_out"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"dd6c6d40-b512-4c67-94f0-02cb3e9e65bd","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"total events out"}},"title":"[MQ] total events out by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"b0a3d8ea-030a-46e5-95b4-9f73b625f6ee","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"dd6c6d40-b512-4c67-94f0-02cb3e9e65bd","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0Niw3XQ=="} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"disabled\":false,\"negate\":false,\"alias\":null,\"key\":\"release\",\"field\":\"release\",\"params\":{\"query\":false},\"type\":\"phrase\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"release\":false}},\"$state\":{\"store\":\"appState\"}}]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":14,\"i\":\"c0ef3cf2-9dbe-4a88-a176-c50422d9041f\"},\"panelIndex\":\"c0ef3cf2-9dbe-4a88-a176-c50422d9041f\",\"embeddableConfig\":{\"enhancements\":{},\"attributes\":{\"title\":\"Daily EPS by queue & worker\",\"description\":\"\",\"visualizationType\":\"lnsXY\",\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"Linear\",\"axisTitlesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"line\",\"layers\":[{\"layerId\":\"999c9564-cba1-4f3b-af99-ccbacb8e034f\",\"seriesType\":\"line\",\"xAccessor\":\"9ddc0f02-5530-4930-99bd-2a914ada1b8f\",\"splitAccessor\":\"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e\",\"accessors\":[\"be033d18-2c04-44f2-b403-0b88d888143d\"],\"layerType\":\"data\",\"colorMapping\":{\"assignments\":[{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"8\",\"memory\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#207f8c\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"16\",\"memory\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#0055fc\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"4\",\"memory\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#99e6e8\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"8\",\"persisted\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#c3513a\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"16\",\"persisted\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#fd1900\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"4\",\"persisted\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#e88888\"},\"touched\":true}],\"specialAssignments\":[{\"rule\":{\"type\":\"other\"},\"color\":{\"type\":\"loop\"},\"touched\":false}],\"paletteId\":\"eui_amsterdam_color_blind\",\"colorMode\":{\"type\":\"categorical\"}}}],\"yTitle\":\"5m eps\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"999c9564-cba1-4f3b-af99-ccbacb8e034f\":{\"columns\":{\"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e\":{\"label\":\"Top values of workers.keyword + 1 other\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"workers.keyword\",\"isBucketed\":true,\"params\":{\"size\":6,\"orderBy\":{\"type\":\"column\",\"columnId\":\"be033d18-2c04-44f2-b403-0b88d888143d\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"multi_terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"secondaryFields\":[\"queue_type.keyword\"]}},\"9ddc0f02-5530-4930-99bd-2a914ada1b8f\":{\"label\":\"timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"be033d18-2c04-44f2-b403-0b88d888143d\":{\"label\":\"Median of 5m_num\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"5m_num\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e\",\"9ddc0f02-5530-4930-99bd-2a914ada1b8f\",\"be033d18-2c04-44f2-b403-0b88d888143d\"],\"incompleteColumns\":{},\"sampling\":1,\"indexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"}},\"currentIndexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"},\"indexpattern\":{\"layers\":{}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}},\"references\":[{\"id\":\"c5493557-3fd6-4929-8dc2-40d248303942\",\"name\":\"indexpattern-datasource-layer-999c9564-cba1-4f3b-af99-ccbacb8e034f\",\"type\":\"index-pattern\"}],\"type\":\"lens\"}},\"panelRefName\":\"panel_c0ef3cf2-9dbe-4a88-a176-c50422d9041f\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":14,\"w\":16,\"h\":14,\"i\":\"3fc59b3e-512e-4a47-baa4-e89e4978b685\"},\"panelIndex\":\"3fc59b3e-512e-4a47-baa4-e89e4978b685\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3fc59b3e-512e-4a47-baa4-e89e4978b685\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":14,\"w\":16,\"h\":14,\"i\":\"6294036c-7c92-4e51-990d-ace6af0de939\"},\"panelIndex\":\"6294036c-7c92-4e51-990d-ace6af0de939\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6294036c-7c92-4e51-990d-ace6af0de939\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":28,\"w\":16,\"h\":8,\"i\":\"77d40890-d175-436f-aac3-7bcc9f7beacc\"},\"panelIndex\":\"77d40890-d175-436f-aac3-7bcc9f7beacc\",\"embeddableConfig\":{\"attributes\":{\"title\":\"PQ max q events by worker\",\"description\":\"\",\"visualizationType\":\"lnsXY\",\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\",\"isInside\":true,\"showSingleSeries\":true,\"legendSize\":\"small\",\"shouldTruncate\":true,\"verticalAlignment\":\"bottom\",\"horizontalAlignment\":\"left\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"Linear\",\"yTitle\":\"events in queue\",\"hideEndzones\":false,\"showCurrentTimeMarker\":false,\"valuesInLegend\":false,\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"line\",\"layers\":[{\"layerId\":\"eb818eba-fa75-45d1-a9ee-6619eecc9280\",\"seriesType\":\"line\",\"splitAccessor\":\"0da92b64-fe48-439e-9b3a-47775ce1a149\",\"accessors\":[\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\"],\"layerType\":\"data\",\"colorMapping\":{\"assignments\":[{\"rule\":{\"type\":\"matchExactly\",\"values\":[\"4\"]},\"color\":{\"type\":\"categorical\",\"paletteId\":\"eui_amsterdam_color_blind\",\"colorIndex\":0},\"touched\":false},{\"rule\":{\"type\":\"matchExactly\",\"values\":[\"8\"]},\"color\":{\"type\":\"categorical\",\"paletteId\":\"eui_amsterdam_color_blind\",\"colorIndex\":1},\"touched\":false},{\"rule\":{\"type\":\"matchExactly\",\"values\":[\"16\"]},\"color\":{\"type\":\"categorical\",\"paletteId\":\"eui_amsterdam_color_blind\",\"colorIndex\":2},\"touched\":false}],\"specialAssignments\":[{\"rule\":{\"type\":\"other\"},\"color\":{\"type\":\"loop\"},\"touched\":false}],\"paletteId\":\"eui_amsterdam_color_blind\",\"colorMode\":{\"type\":\"categorical\"}},\"xAccessor\":\"9542ed03-9691-42a3-b5c2-407bcaab221c\"}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[{\"meta\":{\"disabled\":false,\"negate\":false,\"alias\":null,\"index\":\"599131c3-0cd9-4a1f-b9da-9894538d58cc\",\"key\":\"queue_type.keyword\",\"field\":\"queue_type.keyword\",\"params\":{\"query\":\"persisted\"},\"type\":\"phrase\"},\"query\":{\"match_phrase\":{\"queue_type.keyword\":\"persisted\"}},\"$state\":{\"store\":\"appState\"}}],\"datasourceStates\":{\"formBased\":{\"layers\":{\"eb818eba-fa75-45d1-a9ee-6619eecc9280\":{\"columns\":{\"0da92b64-fe48-439e-9b3a-47775ce1a149\":{\"label\":\"workers\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"workers.keyword\",\"isBucketed\":true,\"params\":{\"size\":3,\"orderBy\":{\"type\":\"column\",\"columnId\":\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"accuracyMode\":false},\"customLabel\":true},\"9542ed03-9691-42a3-b5c2-407bcaab221c\":{\"label\":\"timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\":{\"label\":\"Median of max_queue_events_num\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"max_queue_events_num\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"0da92b64-fe48-439e-9b3a-47775ce1a149\",\"9542ed03-9691-42a3-b5c2-407bcaab221c\",\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\"],\"incompleteColumns\":{},\"sampling\":1,\"indexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"}},\"currentIndexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"},\"indexpattern\":{\"layers\":{}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}},\"references\":[{\"id\":\"c5493557-3fd6-4929-8dc2-40d248303942\",\"name\":\"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280\",\"type\":\"index-pattern\"}],\"type\":\"lens\"},\"enhancements\":{}},\"panelRefName\":\"panel_77d40890-d175-436f-aac3-7bcc9f7beacc\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":36,\"w\":16,\"h\":8,\"i\":\"8bd920f9-5be7-4481-b733-ba83bdd127d5\"},\"panelIndex\":\"8bd920f9-5be7-4481-b733-ba83bdd127d5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8bd920f9-5be7-4481-b733-ba83bdd127d5\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":28,\"w\":16,\"h\":16,\"i\":\"23581637-2ff1-4cae-a839-345620c3d180\"},\"panelIndex\":\"23581637-2ff1-4cae-a839-345620c3d180\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_23581637-2ff1-4cae-a839-345620c3d180\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":28,\"w\":16,\"h\":16,\"i\":\"c7ab44aa-4607-40b7-b268-d69136bfdcde\"},\"panelIndex\":\"c7ab44aa-4607-40b7-b268-d69136bfdcde\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_c7ab44aa-4607-40b7-b268-d69136bfdcde\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":14,\"w\":16,\"h\":14,\"i\":\"01628fc9-19c2-4c65-bc9f-3f284f77c309\"},\"panelIndex\":\"01628fc9-19c2-4c65-bc9f-3f284f77c309\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_01628fc9-19c2-4c65-bc9f-3f284f77c309\"}]","refreshInterval":{"pause":true,"value":60000},"timeFrom":"now-2M","timeRestore":true,"timeTo":"now","title":"Logstash daily benchmark","version":1},"coreMigrationVersion":"8.8.0","created_at":"2024-09-09T13:44:20.764Z","id":"9ef273e3-bee2-48bb-9976-7edc19454b93","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"94ae4c78-8004-4abb-80c9-271f37c94cdc","name":"c0ef3cf2-9dbe-4a88-a176-c50422d9041f:panel_c0ef3cf2-9dbe-4a88-a176-c50422d9041f","type":"lens"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"c0ef3cf2-9dbe-4a88-a176-c50422d9041f:indexpattern-datasource-layer-999c9564-cba1-4f3b-af99-ccbacb8e034f","type":"index-pattern"},{"id":"f5104988-461c-4f65-b31d-ed22d04cb6b8","name":"3fc59b3e-512e-4a47-baa4-e89e4978b685:panel_3fc59b3e-512e-4a47-baa4-e89e4978b685","type":"lens"},{"id":"8cbdba7b-e921-42b9-b55b-da567717e259","name":"6294036c-7c92-4e51-990d-ace6af0de939:panel_6294036c-7c92-4e51-990d-ace6af0de939","type":"lens"},{"id":"7a2c9e96-8e48-4546-b736-065757a52c47","name":"77d40890-d175-436f-aac3-7bcc9f7beacc:panel_77d40890-d175-436f-aac3-7bcc9f7beacc","type":"lens"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"77d40890-d175-436f-aac3-7bcc9f7beacc:indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"daa3722a-cab7-49ba-b5bb-033e81133096","name":"8bd920f9-5be7-4481-b733-ba83bdd127d5:panel_8bd920f9-5be7-4481-b733-ba83bdd127d5","type":"lens"},{"id":"4d1f7bef-c184-4457-b159-465128d2d70b","name":"23581637-2ff1-4cae-a839-345620c3d180:panel_23581637-2ff1-4cae-a839-345620c3d180","type":"lens"},{"id":"f233b0e6-8ca5-4c29-a6b3-1f1bc83509fd","name":"c7ab44aa-4607-40b7-b268-d69136bfdcde:panel_c7ab44aa-4607-40b7-b268-d69136bfdcde","type":"lens"},{"id":"b0a3d8ea-030a-46e5-95b4-9f73b625f6ee","name":"01628fc9-19c2-4c65-bc9f-3f284f77c309:panel_01628fc9-19c2-4c65-bc9f-3f284f77c309","type":"lens"}],"type":"dashboard","typeMigrationVersion":"8.9.0","updated_at":"2024-09-09T13:44:20.764Z","version":"WzEyNTEsN10="} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"disabled\":false,\"negate\":false,\"alias\":null,\"key\":\"release\",\"field\":\"release\",\"params\":{\"query\":true},\"type\":\"phrase\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"release\":true}},\"$state\":{\"store\":\"appState\"}}]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":19,\"i\":\"329684df-6ec4-4973-8c96-bc91eaa23fe9\"},\"panelIndex\":\"329684df-6ec4-4973-8c96-bc91eaa23fe9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_329684df-6ec4-4973-8c96-bc91eaa23fe9\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":19,\"w\":16,\"h\":17,\"i\":\"68db206a-6de8-40a8-bd6c-b82d78a41ac7\"},\"panelIndex\":\"68db206a-6de8-40a8-bd6c-b82d78a41ac7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_68db206a-6de8-40a8-bd6c-b82d78a41ac7\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":19,\"w\":16,\"h\":17,\"i\":\"485385cb-cd2b-408b-a119-5e7b15c15eeb\"},\"panelIndex\":\"485385cb-cd2b-408b-a119-5e7b15c15eeb\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_485385cb-cd2b-408b-a119-5e7b15c15eeb\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":19,\"w\":16,\"h\":17,\"i\":\"f29c5ed5-423f-44bc-8ded-2bdf8844f1ce\"},\"panelIndex\":\"f29c5ed5-423f-44bc-8ded-2bdf8844f1ce\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_f29c5ed5-423f-44bc-8ded-2bdf8844f1ce\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":36,\"w\":16,\"h\":16,\"i\":\"2934659f-462a-41b1-bdc0-344a7a4a2267\"},\"panelIndex\":\"2934659f-462a-41b1-bdc0-344a7a4a2267\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2934659f-462a-41b1-bdc0-344a7a4a2267\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":36,\"w\":16,\"h\":16,\"i\":\"76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2\"},\"panelIndex\":\"76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":36,\"w\":16,\"h\":8,\"i\":\"99c4094f-8906-4ded-b56f-8e20107a8f3b\"},\"panelIndex\":\"99c4094f-8906-4ded-b56f-8e20107a8f3b\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_99c4094f-8906-4ded-b56f-8e20107a8f3b\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":44,\"w\":16,\"h\":8,\"i\":\"a8dd132b-9212-4c33-83ef-19ec5152448e\"},\"panelIndex\":\"a8dd132b-9212-4c33-83ef-19ec5152448e\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_a8dd132b-9212-4c33-83ef-19ec5152448e\"}]","refreshInterval":{"pause":true,"value":60000},"timeFrom":"now-1y","timeRestore":true,"timeTo":"now","title":"Logstash release benchmark","version":1},"coreMigrationVersion":"8.8.0","created_at":"2024-09-09T13:43:43.023Z","id":"901317cf-f97a-4d20-8420-53f0bf67c606","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"2c017667-7403-402c-b027-3cc790945a27","name":"329684df-6ec4-4973-8c96-bc91eaa23fe9:panel_329684df-6ec4-4973-8c96-bc91eaa23fe9","type":"lens"},{"id":"a577558f-f9a6-4c86-b562-7618b2c71db0","name":"68db206a-6de8-40a8-bd6c-b82d78a41ac7:panel_68db206a-6de8-40a8-bd6c-b82d78a41ac7","type":"lens"},{"id":"9546a73b-5937-4d3d-850f-30625d6e1405","name":"485385cb-cd2b-408b-a119-5e7b15c15eeb:panel_485385cb-cd2b-408b-a119-5e7b15c15eeb","type":"lens"},{"id":"6fba7dff-2564-450b-b885-8c6cf97fa727","name":"f29c5ed5-423f-44bc-8ded-2bdf8844f1ce:panel_f29c5ed5-423f-44bc-8ded-2bdf8844f1ce","type":"lens"},{"id":"72ae6cc9-47cd-4b51-b441-86905c2b92bd","name":"2934659f-462a-41b1-bdc0-344a7a4a2267:panel_2934659f-462a-41b1-bdc0-344a7a4a2267","type":"lens"},{"id":"661cb795-bb69-4599-8c4b-a4798ef69dc6","name":"76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2:panel_76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2","type":"lens"},{"id":"d79b8742-6284-4842-ac44-65640d090179","name":"99c4094f-8906-4ded-b56f-8e20107a8f3b:panel_99c4094f-8906-4ded-b56f-8e20107a8f3b","type":"lens"},{"id":"1872e84f-56f2-4a17-9426-3d25d6e03931","name":"a8dd132b-9212-4c33-83ef-19ec5152448e:panel_a8dd132b-9212-4c33-83ef-19ec5152448e","type":"lens"}],"type":"dashboard","typeMigrationVersion":"8.9.0","updated_at":"2024-09-09T13:43:43.023Z","version":"WzEyNDUsN10="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"disabled\":false,\"negate\":false,\"alias\":null,\"key\":\"release\",\"field\":\"release\",\"params\":{\"query\":false},\"type\":\"phrase\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"release\":false}},\"$state\":{\"store\":\"appState\"}}]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":14,\"i\":\"c0ef3cf2-9dbe-4a88-a176-c50422d9041f\"},\"panelIndex\":\"c0ef3cf2-9dbe-4a88-a176-c50422d9041f\",\"embeddableConfig\":{\"enhancements\":{},\"attributes\":{\"title\":\"Daily EPS by queue & worker\",\"description\":\"\",\"visualizationType\":\"lnsXY\",\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"Linear\",\"axisTitlesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"line\",\"layers\":[{\"layerId\":\"999c9564-cba1-4f3b-af99-ccbacb8e034f\",\"seriesType\":\"line\",\"xAccessor\":\"9ddc0f02-5530-4930-99bd-2a914ada1b8f\",\"splitAccessor\":\"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e\",\"accessors\":[\"be033d18-2c04-44f2-b403-0b88d888143d\"],\"layerType\":\"data\",\"colorMapping\":{\"assignments\":[{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"8\",\"memory\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#207f8c\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"16\",\"memory\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#0055fc\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"4\",\"memory\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#99e6e8\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"8\",\"persisted\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#c3513a\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"16\",\"persisted\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#fd1900\"},\"touched\":true},{\"rule\":{\"type\":\"matchExactly\",\"values\":[[\"4\",\"persisted\"]]},\"color\":{\"type\":\"colorCode\",\"colorCode\":\"#e88888\"},\"touched\":true}],\"specialAssignments\":[{\"rule\":{\"type\":\"other\"},\"color\":{\"type\":\"loop\"},\"touched\":false}],\"paletteId\":\"eui_amsterdam_color_blind\",\"colorMode\":{\"type\":\"categorical\"}}}],\"yTitle\":\"5m eps\"},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[],\"datasourceStates\":{\"formBased\":{\"layers\":{\"999c9564-cba1-4f3b-af99-ccbacb8e034f\":{\"columns\":{\"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e\":{\"label\":\"Top values of workers.keyword + 1 other\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"workers.keyword\",\"isBucketed\":true,\"params\":{\"size\":6,\"orderBy\":{\"type\":\"column\",\"columnId\":\"be033d18-2c04-44f2-b403-0b88d888143d\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"multi_terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"secondaryFields\":[\"queue_type.keyword\"]}},\"9ddc0f02-5530-4930-99bd-2a914ada1b8f\":{\"label\":\"timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"be033d18-2c04-44f2-b403-0b88d888143d\":{\"label\":\"Median of 5m_num\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"5m_num\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"a58f8c48-c2a7-4fc6-b54e-716bf4ca001e\",\"9ddc0f02-5530-4930-99bd-2a914ada1b8f\",\"be033d18-2c04-44f2-b403-0b88d888143d\"],\"incompleteColumns\":{},\"sampling\":1,\"indexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"}},\"currentIndexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"},\"indexpattern\":{\"layers\":{}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}},\"references\":[{\"id\":\"c5493557-3fd6-4929-8dc2-40d248303942\",\"name\":\"indexpattern-datasource-layer-999c9564-cba1-4f3b-af99-ccbacb8e034f\",\"type\":\"index-pattern\"}],\"type\":\"lens\"}},\"panelRefName\":\"panel_c0ef3cf2-9dbe-4a88-a176-c50422d9041f\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":14,\"w\":16,\"h\":14,\"i\":\"3fc59b3e-512e-4a47-baa4-e89e4978b685\"},\"panelIndex\":\"3fc59b3e-512e-4a47-baa4-e89e4978b685\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3fc59b3e-512e-4a47-baa4-e89e4978b685\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":14,\"w\":16,\"h\":14,\"i\":\"6294036c-7c92-4e51-990d-ace6af0de939\"},\"panelIndex\":\"6294036c-7c92-4e51-990d-ace6af0de939\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6294036c-7c92-4e51-990d-ace6af0de939\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":28,\"w\":16,\"h\":8,\"i\":\"77d40890-d175-436f-aac3-7bcc9f7beacc\"},\"panelIndex\":\"77d40890-d175-436f-aac3-7bcc9f7beacc\",\"embeddableConfig\":{\"attributes\":{\"title\":\"PQ max q events by worker\",\"description\":\"\",\"visualizationType\":\"lnsXY\",\"state\":{\"visualization\":{\"legend\":{\"isVisible\":true,\"position\":\"right\",\"isInside\":true,\"showSingleSeries\":true,\"legendSize\":\"small\",\"shouldTruncate\":true,\"verticalAlignment\":\"bottom\",\"horizontalAlignment\":\"left\"},\"valueLabels\":\"hide\",\"fittingFunction\":\"Linear\",\"yTitle\":\"events in queue\",\"hideEndzones\":false,\"showCurrentTimeMarker\":false,\"valuesInLegend\":false,\"axisTitlesVisibilitySettings\":{\"x\":false,\"yLeft\":true,\"yRight\":true},\"tickLabelsVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"labelsOrientation\":{\"x\":0,\"yLeft\":0,\"yRight\":0},\"gridlinesVisibilitySettings\":{\"x\":true,\"yLeft\":true,\"yRight\":true},\"preferredSeriesType\":\"line\",\"layers\":[{\"layerId\":\"eb818eba-fa75-45d1-a9ee-6619eecc9280\",\"seriesType\":\"line\",\"splitAccessor\":\"0da92b64-fe48-439e-9b3a-47775ce1a149\",\"accessors\":[\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\"],\"layerType\":\"data\",\"colorMapping\":{\"assignments\":[{\"rule\":{\"type\":\"matchExactly\",\"values\":[\"4\"]},\"color\":{\"type\":\"categorical\",\"paletteId\":\"eui_amsterdam_color_blind\",\"colorIndex\":0},\"touched\":false},{\"rule\":{\"type\":\"matchExactly\",\"values\":[\"8\"]},\"color\":{\"type\":\"categorical\",\"paletteId\":\"eui_amsterdam_color_blind\",\"colorIndex\":1},\"touched\":false},{\"rule\":{\"type\":\"matchExactly\",\"values\":[\"16\"]},\"color\":{\"type\":\"categorical\",\"paletteId\":\"eui_amsterdam_color_blind\",\"colorIndex\":2},\"touched\":false}],\"specialAssignments\":[{\"rule\":{\"type\":\"other\"},\"color\":{\"type\":\"loop\"},\"touched\":false}],\"paletteId\":\"eui_amsterdam_color_blind\",\"colorMode\":{\"type\":\"categorical\"}},\"xAccessor\":\"9542ed03-9691-42a3-b5c2-407bcaab221c\"}]},\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filters\":[{\"meta\":{\"disabled\":false,\"negate\":false,\"alias\":null,\"index\":\"599131c3-0cd9-4a1f-b9da-9894538d58cc\",\"key\":\"queue_type.keyword\",\"field\":\"queue_type.keyword\",\"params\":{\"query\":\"persisted\"},\"type\":\"phrase\"},\"query\":{\"match_phrase\":{\"queue_type.keyword\":\"persisted\"}},\"$state\":{\"store\":\"appState\"}}],\"datasourceStates\":{\"formBased\":{\"layers\":{\"eb818eba-fa75-45d1-a9ee-6619eecc9280\":{\"columns\":{\"0da92b64-fe48-439e-9b3a-47775ce1a149\":{\"label\":\"workers\",\"dataType\":\"string\",\"operationType\":\"terms\",\"scale\":\"ordinal\",\"sourceField\":\"workers.keyword\",\"isBucketed\":true,\"params\":{\"size\":3,\"orderBy\":{\"type\":\"column\",\"columnId\":\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\"},\"orderDirection\":\"desc\",\"otherBucket\":true,\"missingBucket\":false,\"parentFormat\":{\"id\":\"terms\"},\"include\":[],\"exclude\":[],\"includeIsRegex\":false,\"excludeIsRegex\":false,\"accuracyMode\":false},\"customLabel\":true},\"9542ed03-9691-42a3-b5c2-407bcaab221c\":{\"label\":\"timestamp\",\"dataType\":\"date\",\"operationType\":\"date_histogram\",\"sourceField\":\"timestamp\",\"isBucketed\":true,\"scale\":\"interval\",\"params\":{\"interval\":\"auto\",\"includeEmptyRows\":true,\"dropPartials\":false}},\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\":{\"label\":\"Median of max_queue_events_num\",\"dataType\":\"number\",\"operationType\":\"median\",\"sourceField\":\"max_queue_events_num\",\"isBucketed\":false,\"scale\":\"ratio\",\"params\":{\"emptyAsNull\":true}}},\"columnOrder\":[\"0da92b64-fe48-439e-9b3a-47775ce1a149\",\"9542ed03-9691-42a3-b5c2-407bcaab221c\",\"cc8c1413-0ccc-4109-a6c3-0c910eff776e\"],\"incompleteColumns\":{},\"sampling\":1,\"indexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"}},\"currentIndexPatternId\":\"c5493557-3fd6-4929-8dc2-40d248303942\"},\"indexpattern\":{\"layers\":{}},\"textBased\":{\"layers\":{}}},\"internalReferences\":[],\"adHocDataViews\":{}},\"references\":[{\"id\":\"c5493557-3fd6-4929-8dc2-40d248303942\",\"name\":\"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280\",\"type\":\"index-pattern\"}],\"type\":\"lens\"},\"enhancements\":{}},\"panelRefName\":\"panel_77d40890-d175-436f-aac3-7bcc9f7beacc\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":36,\"w\":16,\"h\":8,\"i\":\"8bd920f9-5be7-4481-b733-ba83bdd127d5\"},\"panelIndex\":\"8bd920f9-5be7-4481-b733-ba83bdd127d5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_8bd920f9-5be7-4481-b733-ba83bdd127d5\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":28,\"w\":16,\"h\":16,\"i\":\"23581637-2ff1-4cae-a839-345620c3d180\"},\"panelIndex\":\"23581637-2ff1-4cae-a839-345620c3d180\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_23581637-2ff1-4cae-a839-345620c3d180\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":28,\"w\":16,\"h\":16,\"i\":\"c7ab44aa-4607-40b7-b268-d69136bfdcde\"},\"panelIndex\":\"c7ab44aa-4607-40b7-b268-d69136bfdcde\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_c7ab44aa-4607-40b7-b268-d69136bfdcde\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":14,\"w\":16,\"h\":14,\"i\":\"01628fc9-19c2-4c65-bc9f-3f284f77c309\"},\"panelIndex\":\"01628fc9-19c2-4c65-bc9f-3f284f77c309\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_01628fc9-19c2-4c65-bc9f-3f284f77c309\"}]","refreshInterval":{"pause":true,"value":60000},"timeFrom":"now-2M","timeRestore":true,"timeTo":"now","title":"Logstash daily benchmark","version":1},"coreMigrationVersion":"8.8.0","created_at":"2024-09-09T13:44:20.764Z","id":"9ef273e3-bee2-48bb-9976-7edc19454b93","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"94ae4c78-8004-4abb-80c9-271f37c94cdc","name":"c0ef3cf2-9dbe-4a88-a176-c50422d9041f:panel_c0ef3cf2-9dbe-4a88-a176-c50422d9041f","type":"lens"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"c0ef3cf2-9dbe-4a88-a176-c50422d9041f:indexpattern-datasource-layer-999c9564-cba1-4f3b-af99-ccbacb8e034f","type":"index-pattern"},{"id":"f5104988-461c-4f65-b31d-ed22d04cb6b8","name":"3fc59b3e-512e-4a47-baa4-e89e4978b685:panel_3fc59b3e-512e-4a47-baa4-e89e4978b685","type":"lens"},{"id":"8cbdba7b-e921-42b9-b55b-da567717e259","name":"6294036c-7c92-4e51-990d-ace6af0de939:panel_6294036c-7c92-4e51-990d-ace6af0de939","type":"lens"},{"id":"7a2c9e96-8e48-4546-b736-065757a52c47","name":"77d40890-d175-436f-aac3-7bcc9f7beacc:panel_77d40890-d175-436f-aac3-7bcc9f7beacc","type":"lens"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"77d40890-d175-436f-aac3-7bcc9f7beacc:indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"daa3722a-cab7-49ba-b5bb-033e81133096","name":"8bd920f9-5be7-4481-b733-ba83bdd127d5:panel_8bd920f9-5be7-4481-b733-ba83bdd127d5","type":"lens"},{"id":"4d1f7bef-c184-4457-b159-465128d2d70b","name":"23581637-2ff1-4cae-a839-345620c3d180:panel_23581637-2ff1-4cae-a839-345620c3d180","type":"lens"},{"id":"f233b0e6-8ca5-4c29-a6b3-1f1bc83509fd","name":"c7ab44aa-4607-40b7-b268-d69136bfdcde:panel_c7ab44aa-4607-40b7-b268-d69136bfdcde","type":"lens"},{"id":"b0a3d8ea-030a-46e5-95b4-9f73b625f6ee","name":"01628fc9-19c2-4c65-bc9f-3f284f77c309:panel_01628fc9-19c2-4c65-bc9f-3f284f77c309","type":"lens"}],"type":"dashboard","typeMigrationVersion":"10.2.0","updated_at":"2024-09-09T13:44:20.764Z","version":"WzE0ODksOF0="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"disabled\":false,\"negate\":false,\"alias\":null,\"key\":\"release\",\"field\":\"release\",\"params\":{\"query\":true},\"type\":\"phrase\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match_phrase\":{\"release\":true}},\"$state\":{\"store\":\"appState\"}}]}"},"optionsJSON":"{\"useMargins\":true,\"syncColors\":false,\"syncCursor\":true,\"syncTooltips\":false,\"hidePanelTitles\":false}","panelsJSON":"[{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":19,\"i\":\"329684df-6ec4-4973-8c96-bc91eaa23fe9\"},\"panelIndex\":\"329684df-6ec4-4973-8c96-bc91eaa23fe9\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_329684df-6ec4-4973-8c96-bc91eaa23fe9\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":19,\"w\":16,\"h\":17,\"i\":\"68db206a-6de8-40a8-bd6c-b82d78a41ac7\"},\"panelIndex\":\"68db206a-6de8-40a8-bd6c-b82d78a41ac7\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_68db206a-6de8-40a8-bd6c-b82d78a41ac7\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":19,\"w\":16,\"h\":17,\"i\":\"485385cb-cd2b-408b-a119-5e7b15c15eeb\"},\"panelIndex\":\"485385cb-cd2b-408b-a119-5e7b15c15eeb\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_485385cb-cd2b-408b-a119-5e7b15c15eeb\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":19,\"w\":16,\"h\":17,\"i\":\"f29c5ed5-423f-44bc-8ded-2bdf8844f1ce\"},\"panelIndex\":\"f29c5ed5-423f-44bc-8ded-2bdf8844f1ce\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_f29c5ed5-423f-44bc-8ded-2bdf8844f1ce\"},{\"type\":\"lens\",\"gridData\":{\"x\":0,\"y\":36,\"w\":16,\"h\":16,\"i\":\"2934659f-462a-41b1-bdc0-344a7a4a2267\"},\"panelIndex\":\"2934659f-462a-41b1-bdc0-344a7a4a2267\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2934659f-462a-41b1-bdc0-344a7a4a2267\"},{\"type\":\"lens\",\"gridData\":{\"x\":16,\"y\":36,\"w\":16,\"h\":16,\"i\":\"76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2\"},\"panelIndex\":\"76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":36,\"w\":16,\"h\":8,\"i\":\"99c4094f-8906-4ded-b56f-8e20107a8f3b\"},\"panelIndex\":\"99c4094f-8906-4ded-b56f-8e20107a8f3b\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_99c4094f-8906-4ded-b56f-8e20107a8f3b\"},{\"type\":\"lens\",\"gridData\":{\"x\":32,\"y\":44,\"w\":16,\"h\":8,\"i\":\"a8dd132b-9212-4c33-83ef-19ec5152448e\"},\"panelIndex\":\"a8dd132b-9212-4c33-83ef-19ec5152448e\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_a8dd132b-9212-4c33-83ef-19ec5152448e\"}]","refreshInterval":{"pause":true,"value":60000},"timeFrom":"now-1y","timeRestore":true,"timeTo":"now","title":"Logstash release benchmark","version":1},"coreMigrationVersion":"8.8.0","created_at":"2024-09-09T13:43:43.023Z","id":"901317cf-f97a-4d20-8420-53f0bf67c606","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"2c017667-7403-402c-b027-3cc790945a27","name":"329684df-6ec4-4973-8c96-bc91eaa23fe9:panel_329684df-6ec4-4973-8c96-bc91eaa23fe9","type":"lens"},{"id":"a577558f-f9a6-4c86-b562-7618b2c71db0","name":"68db206a-6de8-40a8-bd6c-b82d78a41ac7:panel_68db206a-6de8-40a8-bd6c-b82d78a41ac7","type":"lens"},{"id":"9546a73b-5937-4d3d-850f-30625d6e1405","name":"485385cb-cd2b-408b-a119-5e7b15c15eeb:panel_485385cb-cd2b-408b-a119-5e7b15c15eeb","type":"lens"},{"id":"6fba7dff-2564-450b-b885-8c6cf97fa727","name":"f29c5ed5-423f-44bc-8ded-2bdf8844f1ce:panel_f29c5ed5-423f-44bc-8ded-2bdf8844f1ce","type":"lens"},{"id":"72ae6cc9-47cd-4b51-b441-86905c2b92bd","name":"2934659f-462a-41b1-bdc0-344a7a4a2267:panel_2934659f-462a-41b1-bdc0-344a7a4a2267","type":"lens"},{"id":"661cb795-bb69-4599-8c4b-a4798ef69dc6","name":"76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2:panel_76f881d0-d2d3-4eaa-b71e-8cf8b7c1c5c2","type":"lens"},{"id":"d79b8742-6284-4842-ac44-65640d090179","name":"99c4094f-8906-4ded-b56f-8e20107a8f3b:panel_99c4094f-8906-4ded-b56f-8e20107a8f3b","type":"lens"},{"id":"1872e84f-56f2-4a17-9426-3d25d6e03931","name":"a8dd132b-9212-4c33-83ef-19ec5152448e:panel_a8dd132b-9212-4c33-83ef-19ec5152448e","type":"lens"}],"type":"dashboard","typeMigrationVersion":"10.2.0","updated_at":"2024-09-09T13:43:43.023Z","version":"WzE0OTAsOF0="} {"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_utilization","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_utilization"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"9aee27f1-97bc-46c4-b369-81b7cd5fe38c","key":"queue_type.keyword","negate":false,"params":{"query":"persisted"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"persisted"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"endValue":"None","fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"position":"top","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_utilization (%)"}},"title":"[PQ] Worker Utilization (%) by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"07700ebe-b8c9-498f-9a91-30efb0c48784","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"9aee27f1-97bc-46c4-b369-81b7cd5fe38c","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0MCw3XQ=="} {"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"date","isBucketed":true,"label":"timestamp","operationType":"date_histogram","params":{"dropPartials":false,"includeEmptyRows":true,"interval":"auto"},"scale":"interval","sourceField":"timestamp"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of max_worker_utilization","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"max_worker_utilization"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"4e0f2401-369e-4818-8a1f-52b64ff175df","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"worker_utilization (%)"}},"title":"[MQ] Worker Utilization (%) by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"29ebb6b1-bee6-4f15-b3dc-aac083d5d2f5","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"4e0f2401-369e-4818-8a1f-52b64ff175df","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0NSw3XQ=="} {"attributes":{"description":"","state":{"adHocDataViews":{},"datasourceStates":{"formBased":{"layers":{"eb818eba-fa75-45d1-a9ee-6619eecc9280":{"columnOrder":["0da92b64-fe48-439e-9b3a-47775ce1a149","9542ed03-9691-42a3-b5c2-407bcaab221c","cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"columns":{"0da92b64-fe48-439e-9b3a-47775ce1a149":{"customLabel":true,"dataType":"string","isBucketed":true,"label":"workers","operationType":"terms","params":{"accuracyMode":false,"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderBy":{"columnId":"cc8c1413-0ccc-4109-a6c3-0c910eff776e","type":"column"},"orderDirection":"desc","otherBucket":true,"parentFormat":{"id":"terms"},"size":10},"scale":"ordinal","sourceField":"workers.keyword"},"9542ed03-9691-42a3-b5c2-407bcaab221c":{"dataType":"string","isBucketed":true,"label":"Top 100 values of version.keyword","operationType":"terms","params":{"exclude":[],"excludeIsRegex":false,"include":[],"includeIsRegex":false,"missingBucket":false,"orderAgg":{"dataType":"number","isBucketed":false,"label":"Maximum of version_num","operationType":"max","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"version_num"},"orderBy":{"type":"custom"},"orderDirection":"asc","otherBucket":true,"parentFormat":{"id":"terms"},"size":100},"scale":"ordinal","sourceField":"version.keyword"},"cc8c1413-0ccc-4109-a6c3-0c910eff776e":{"dataType":"number","isBucketed":false,"label":"Median of avg_cpu_percentage","operationType":"median","params":{"emptyAsNull":true},"scale":"ratio","sourceField":"avg_cpu_percentage"}},"incompleteColumns":{},"sampling":1}}},"indexpattern":{"layers":{}},"textBased":{"layers":{}}},"filters":[{"$state":{"store":"appState"},"meta":{"alias":null,"disabled":false,"index":"f4c5b868-7b2c-4d12-b989-9a057dd128e4","negate":false,"params":[{"meta":{"alias":null,"disabled":false,"field":"queue_type.keyword","index":"c5493557-3fd6-4929-8dc2-40d248303942","key":"queue_type.keyword","negate":false,"params":{"query":"memory"},"type":"phrase"},"query":{"match_phrase":{"queue_type.keyword":"memory"}}},{"meta":{"alias":null,"disabled":false,"field":"release","index":"c5493557-3fd6-4929-8dc2-40d248303942","key":"release","negate":false,"params":{"query":true},"type":"phrase"},"query":{"match_phrase":{"release":true}}}],"relation":"AND","type":"combined"},"query":{}}],"internalReferences":[],"query":{"language":"kuery","query":""},"visualization":{"axisTitlesVisibilitySettings":{"x":false,"yLeft":true,"yRight":true},"fittingFunction":"Linear","gridlinesVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"hideEndzones":false,"labelsOrientation":{"x":0,"yLeft":0,"yRight":0},"layers":[{"accessors":["cc8c1413-0ccc-4109-a6c3-0c910eff776e"],"colorMapping":{"assignments":[{"color":{"colorIndex":0,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["4"]},"touched":false},{"color":{"colorIndex":1,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["8"]},"touched":false},{"color":{"colorIndex":2,"paletteId":"eui_amsterdam_color_blind","type":"categorical"},"rule":{"type":"matchExactly","values":["16"]},"touched":false}],"colorMode":{"type":"categorical"},"paletteId":"eui_amsterdam_color_blind","specialAssignments":[{"color":{"type":"loop"},"rule":{"type":"other"},"touched":false}]},"layerId":"eb818eba-fa75-45d1-a9ee-6619eecc9280","layerType":"data","seriesType":"line","splitAccessor":"0da92b64-fe48-439e-9b3a-47775ce1a149","xAccessor":"9542ed03-9691-42a3-b5c2-407bcaab221c"}],"legend":{"horizontalAlignment":"left","isInside":true,"isVisible":true,"legendSize":"small","position":"right","shouldTruncate":true,"showSingleSeries":true,"verticalAlignment":"bottom"},"preferredSeriesType":"line","showCurrentTimeMarker":false,"tickLabelsVisibilitySettings":{"x":true,"yLeft":true,"yRight":true},"valueLabels":"hide","valuesInLegend":false,"yTitle":"avg cpu (%)"}},"title":"[MQ][R] cpu % by worker","visualizationType":"lnsXY"},"coreMigrationVersion":"8.8.0","created_at":"2024-09-05T11:52:42.820Z","id":"26ec195e-e4a9-4417-be97-6d1fbc6b63c0","managed":false,"references":[{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"indexpattern-datasource-layer-eb818eba-fa75-45d1-a9ee-6619eecc9280","type":"index-pattern"},{"id":"c5493557-3fd6-4929-8dc2-40d248303942","name":"f4c5b868-7b2c-4d12-b989-9a057dd128e4","type":"index-pattern"}],"type":"lens","typeMigrationVersion":"8.9.0","updated_at":"2024-09-05T11:52:42.820Z","version":"Wzc0OSw3XQ=="} diff --git a/catalog-info.yaml b/catalog-info.yaml index 9f1dfa94dd6..c5367ea6691 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -614,7 +614,7 @@ spec: kind: Pipeline metadata: name: logstash-benchmark-pipeline - description: ':logstash: The Benchmark pipeline for snapshot version' + description: ':running: The Benchmark pipeline for snapshot version' spec: repository: elastic/logstash pipeline_file: ".buildkite/benchmark_pipeline.yml" @@ -646,18 +646,18 @@ spec: # ******************************* # *********************************** -# SECTION START: Benchmark multiple Logstash versions +# SECTION START: Benchmark Marathon # *********************************** --- # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json apiVersion: backstage.io/v1alpha1 kind: Resource metadata: - name: logstash-benchmark-versions-pipeline - description: Buildkite pipeline for benchmarking multiple versions + name: logstash-benchmark-marathon-pipeline + description: Buildkite pipeline for benchmarking multi-version links: - - title: 'Logstash benchmark pipeline for multiple versions' - url: https://buildkite.com/elastic/logstash-benchmark-versions-pipeline + - title: 'Logstash Benchmark Marathon' + url: https://buildkite.com/elastic/logstash-benchmark-marathon-pipeline spec: type: buildkite-pipeline owner: group:logstash @@ -666,11 +666,11 @@ spec: apiVersion: buildkite.elastic.dev/v1 kind: Pipeline metadata: - name: logstash-benchmark-versions-pipeline - description: ':logstash: The Benchmark pipeline for multiple versions' + name: logstash-benchmark-marathon-pipeline + description: ':running: The Benchmark Marathon for multi-version' spec: repository: elastic/logstash - pipeline_file: ".buildkite/benchmark_versions_pipeline.yml" + pipeline_file: ".buildkite/benchmark_marathon_pipeline.yml" maximum_timeout_in_minutes: 480 provider_settings: trigger_mode: none # don't trigger jobs from github activity @@ -690,7 +690,7 @@ spec: access_level: READ_ONLY # ******************************* -# SECTION END: Benchmark multiple Logstash versions +# SECTION END: Benchmark Marathon # ******************************* # ***********************************