diff --git a/.github/workflows/run_comparative_benchmark.yml b/.github/workflows/run_comparative_benchmark.yml index 4c77c803..95885840 100644 --- a/.github/workflows/run_comparative_benchmark.yml +++ b/.github/workflows/run_comparative_benchmark.yml @@ -13,6 +13,7 @@ on: # Scheduled to run at 09:00 UTC and 21:00 UTC. - cron: '0 09,21 * * *' workflow_dispatch: + pull_request: concurrency: # A PR number if a pull request and otherwise the commit hash. This cancels @@ -100,93 +101,169 @@ jobs: gcloud storage cp "${XLA_TOOLS_DIR_ARCHIVE}" "${XLA_TOOLS_DIR_GCS_ARTIFACT}" echo "xla-tools-dir-gcs-artifact=${XLA_TOOLS_DIR_GCS_ARTIFACT}" >> "${GITHUB_OUTPUT}" - benchmark_on_a2-highgpu-1g: - needs: [setup, build_xla_tools] - runs-on: - - self-hosted # must come first - - runner-group=${{ needs.setup.outputs.runner-group }} - - environment=prod - - machine-type=a2-highgpu-1g - env: - BENCHMARK_GCS_DIR: ${{ needs.setup.outputs.benchmark-gcs-dir }} - RESULTS_DIR: results-dir - TARGET_DEVICE: a2-highgpu-1g - XLA_TOOLS_DIR: ${{ needs.build_xla_tools.outputs.xla-tools-dir }} - XLA_TOOLS_DIR_ARCHIVE: ${{ needs.build_xla_tools.outputs.xla-tools-dir-archive }} - XLA_TOOLS_DIR_GCS_ARTIFACT: ${{ needs.build_xla_tools.outputs.xla-tools-dir-gcs-artifact }} - steps: - - name: "Checking out PR repository" - uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 - - name: "Setup" - id: setup - run: | - echo "results-gcs-dir=${BENCHMARK_GCS_DIR}/${TARGET_DEVICE}-results" >> "${GITHUB_OUTPUT}" - mkdir "${RESULTS_DIR}" - - name: "Downloading and unpacking XLA tools" - run: | - gcloud storage cp "${XLA_TOOLS_DIR_GCS_ARTIFACT}" "${XLA_TOOLS_DIR_ARCHIVE}" - tar -xvf "${XLA_TOOLS_DIR_ARCHIVE}" - - name: "Benchmarking XLA-HLO:GPU" - env: - XLA_HLO_RESULTS_JSON: xla-hlo.json - RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} - run: | - RESULTS_PATH="${RESULTS_DIR}/${XLA_HLO_RESULTS_JSON}" - docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ - --env "OOBI_XLA_TOOLS_DIR=${XLA_TOOLS_DIR}" \ - "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \ - ./comparative_benchmark/xla_hlo/benchmark_all.sh \ - "${TARGET_DEVICE}"\ - "${RESULTS_PATH}" - gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" - - name: "Benchmarking JAX-XLA:GPU" - env: - JAX_XLA_RESULTS_JSON: jax-xla.json - RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} - run: | - RESULTS_PATH="${RESULTS_DIR}/${JAX_XLA_RESULTS_JSON}" - docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ - "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \ - ./comparative_benchmark/jax_xla/benchmark_all.sh \ - "${TARGET_DEVICE}"\ - "${RESULTS_PATH}" - gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" - - name: "Benchmarking TF-XLA:GPU" - env: - TF_XLA_RESULTS_JSON: tf-xla.json - RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} - run: | - RESULTS_PATH="${RESULTS_DIR}/${TF_XLA_RESULTS_JSON}" - docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ - "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \ - ./comparative_benchmark/tf_xla/benchmark_all.sh \ - "${TARGET_DEVICE}"\ - "${RESULTS_PATH}" - gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" - - name: "Benchmarking PT-Inductor:GPU" - env: - PT_INDUCTOR_RESULTS_JSON: pt-inductor.json - RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} - run: | - RESULTS_PATH="${RESULTS_DIR}/${PT_INDUCTOR_RESULTS_JSON}" - docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ - "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \ - ./comparative_benchmark/pt_inductor/benchmark_all.sh \ - "${TARGET_DEVICE}"\ - "${RESULTS_PATH}" - gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" + # benchmark_on_a2-highgpu-1g: + # needs: [setup, build_xla_tools] + # runs-on: + # - self-hosted # must come first + # - runner-group=${{ needs.setup.outputs.runner-group }} + # - environment=prod + # - machine-type=a2-highgpu-1g + # env: + # BENCHMARK_GCS_DIR: ${{ needs.setup.outputs.benchmark-gcs-dir }} + # RESULTS_DIR: results-dir + # TARGET_DEVICE: a2-highgpu-1g + # XLA_TOOLS_DIR: ${{ needs.build_xla_tools.outputs.xla-tools-dir }} + # XLA_TOOLS_DIR_ARCHIVE: ${{ needs.build_xla_tools.outputs.xla-tools-dir-archive }} + # XLA_TOOLS_DIR_GCS_ARTIFACT: ${{ needs.build_xla_tools.outputs.xla-tools-dir-gcs-artifact }} + # steps: + # - name: "Checking out PR repository" + # uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 + # - name: "Setup" + # id: setup + # run: | + # echo "results-gcs-dir=${BENCHMARK_GCS_DIR}/${TARGET_DEVICE}-results" >> "${GITHUB_OUTPUT}" + # mkdir "${RESULTS_DIR}" + # - name: "Downloading and unpacking XLA tools" + # run: | + # gcloud storage cp "${XLA_TOOLS_DIR_GCS_ARTIFACT}" "${XLA_TOOLS_DIR_ARCHIVE}" + # tar -xvf "${XLA_TOOLS_DIR_ARCHIVE}" + # - name: "Benchmarking XLA-HLO:GPU" + # env: + # XLA_HLO_RESULTS_JSON: xla-hlo.json + # RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} + # run: | + # RESULTS_PATH="${RESULTS_DIR}/${XLA_HLO_RESULTS_JSON}" + # docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ + # --env "OOBI_XLA_TOOLS_DIR=${XLA_TOOLS_DIR}" \ + # "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \ + # ./comparative_benchmark/xla_hlo/benchmark_all.sh \ + # "${TARGET_DEVICE}"\ + # "${RESULTS_PATH}" + # gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" + # - name: "Benchmarking JAX-XLA:GPU" + # env: + # JAX_XLA_RESULTS_JSON: jax-xla.json + # RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} + # run: | + # RESULTS_PATH="${RESULTS_DIR}/${JAX_XLA_RESULTS_JSON}" + # docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ + # "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \ + # ./comparative_benchmark/jax_xla/benchmark_all.sh \ + # "${TARGET_DEVICE}"\ + # "${RESULTS_PATH}" + # gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" + # - name: "Benchmarking TF-XLA:GPU" + # env: + # TF_XLA_RESULTS_JSON: tf-xla.json + # RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} + # run: | + # RESULTS_PATH="${RESULTS_DIR}/${TF_XLA_RESULTS_JSON}" + # docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ + # "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \ + # ./comparative_benchmark/tf_xla/benchmark_all.sh \ + # "${TARGET_DEVICE}"\ + # "${RESULTS_PATH}" + # gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" + # - name: "Benchmarking PT-Inductor:GPU" + # env: + # PT_INDUCTOR_RESULTS_JSON: pt-inductor.json + # RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} + # run: | + # RESULTS_PATH="${RESULTS_DIR}/${PT_INDUCTOR_RESULTS_JSON}" + # docker run --gpus all --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ + # "gcr.io/iree-oss/openxla-benchmark/cuda11.8-cudnn8.9@sha256:c39107c4160e749b7c4bac18862c6c1b6d56e1aa60644a4fe323e315ffba0a0b" \ + # ./comparative_benchmark/pt_inductor/benchmark_all.sh \ + # "${TARGET_DEVICE}"\ + # "${RESULTS_PATH}" + # gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" + + # benchmark_on_c2-standard-16: + # needs: [setup, build_xla_tools] + # runs-on: + # - self-hosted # must come first + # - runner-group=${{ needs.setup.outputs.runner-group }} + # - environment=prod + # - machine-type=c2-standard-16 + # env: + # BENCHMARK_GCS_DIR: ${{ needs.setup.outputs.benchmark-gcs-dir }} + # RESULTS_DIR: results-dir + # TARGET_DEVICE: c2-standard-16 + # XLA_TOOLS_DIR: ${{ needs.build_xla_tools.outputs.xla-tools-dir }} + # XLA_TOOLS_DIR_ARCHIVE: ${{ needs.build_xla_tools.outputs.xla-tools-dir-archive }} + # XLA_TOOLS_DIR_GCS_ARTIFACT: ${{ needs.build_xla_tools.outputs.xla-tools-dir-gcs-artifact }} + # steps: + # - name: "Checking out PR repository" + # uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0 + # - name: "Setup" + # id: setup + # run: | + # echo "results-gcs-dir=${BENCHMARK_GCS_DIR}/${TARGET_DEVICE}-results" >> "${GITHUB_OUTPUT}" + # mkdir "${RESULTS_DIR}" + # - name: "Downloading and unpacking XLA tools" + # run: | + # gcloud storage cp "${XLA_TOOLS_DIR_GCS_ARTIFACT}" "${XLA_TOOLS_DIR_ARCHIVE}" + # tar -xvf "${XLA_TOOLS_DIR_ARCHIVE}" + # - name: "Benchmarking XLA-HLO:CPU" + # env: + # XLA_HLO_RESULTS_JSON: xla-hlo.json + # RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} + # run: | + # RESULTS_PATH="${RESULTS_DIR}/${XLA_HLO_RESULTS_JSON}" + # docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ + # --env "OOBI_XLA_TOOLS_DIR=${XLA_TOOLS_DIR}" \ + # "gcr.io/iree-oss/openxla-benchmark/base@sha256:692bb3ca648c474e89bad21ef89f4b33a9741e3d1cc477fa61844caddd798ba0" \ + # ./comparative_benchmark/xla_hlo/benchmark_all.sh \ + # "${TARGET_DEVICE}"\ + # "${RESULTS_PATH}" + # gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" + # - name: "Benchmarking JAX-XLA:CPU" + # env: + # JAX_XLA_RESULTS_JSON: jax-xla.json + # RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} + # run: | + # RESULTS_PATH="${RESULTS_DIR}/${JAX_XLA_RESULTS_JSON}" + # docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ + # "gcr.io/iree-oss/openxla-benchmark/base@sha256:692bb3ca648c474e89bad21ef89f4b33a9741e3d1cc477fa61844caddd798ba0" \ + # ./comparative_benchmark/jax_xla/benchmark_all.sh \ + # "${TARGET_DEVICE}"\ + # "${RESULTS_PATH}" + # gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" + # - name: "Benchmarking TF-XLA:CPU" + # env: + # TF_XLA_RESULTS_JSON: tf-xla.json + # RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} + # run: | + # RESULTS_PATH="${RESULTS_DIR}/${TF_XLA_RESULTS_JSON}" + # docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ + # "gcr.io/iree-oss/openxla-benchmark/base@sha256:692bb3ca648c474e89bad21ef89f4b33a9741e3d1cc477fa61844caddd798ba0" \ + # ./comparative_benchmark/tf_xla/benchmark_all.sh \ + # "${TARGET_DEVICE}"\ + # "${RESULTS_PATH}" + # gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" + # - name: "Benchmarking PT-Inductor:CPU" + # env: + # PT_INDUCTOR_RESULTS_JSON: pt-inductor.json + # RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} + # run: | + # RESULTS_PATH="${RESULTS_DIR}/${PT_INDUCTOR_RESULTS_JSON}" + # docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ + # "gcr.io/iree-oss/openxla-benchmark/base@sha256:692bb3ca648c474e89bad21ef89f4b33a9741e3d1cc477fa61844caddd798ba0" \ + # ./comparative_benchmark/pt_inductor/benchmark_all.sh \ + # "${TARGET_DEVICE}"\ + # "${RESULTS_PATH}" + # gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" - benchmark_on_c2-standard-16: + benchmark_on_c2-standard-60: needs: [setup, build_xla_tools] runs-on: - self-hosted # must come first - runner-group=${{ needs.setup.outputs.runner-group }} - environment=prod - - machine-type=c2-standard-16 + - machine-type=c2-standard-60 env: BENCHMARK_GCS_DIR: ${{ needs.setup.outputs.benchmark-gcs-dir }} RESULTS_DIR: results-dir - TARGET_DEVICE: c2-standard-16 + TARGET_DEVICE: c2-standard-60 XLA_TOOLS_DIR: ${{ needs.build_xla_tools.outputs.xla-tools-dir }} XLA_TOOLS_DIR_ARCHIVE: ${{ needs.build_xla_tools.outputs.xla-tools-dir-archive }} XLA_TOOLS_DIR_GCS_ARTIFACT: ${{ needs.build_xla_tools.outputs.xla-tools-dir-gcs-artifact }} @@ -227,27 +304,3 @@ jobs: "${TARGET_DEVICE}"\ "${RESULTS_PATH}" gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" - - name: "Benchmarking TF-XLA:CPU" - env: - TF_XLA_RESULTS_JSON: tf-xla.json - RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} - run: | - RESULTS_PATH="${RESULTS_DIR}/${TF_XLA_RESULTS_JSON}" - docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ - "gcr.io/iree-oss/openxla-benchmark/base@sha256:692bb3ca648c474e89bad21ef89f4b33a9741e3d1cc477fa61844caddd798ba0" \ - ./comparative_benchmark/tf_xla/benchmark_all.sh \ - "${TARGET_DEVICE}"\ - "${RESULTS_PATH}" - gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" - - name: "Benchmarking PT-Inductor:CPU" - env: - PT_INDUCTOR_RESULTS_JSON: pt-inductor.json - RESULTS_GCS_DIR: ${{ steps.setup.outputs.results-gcs-dir }} - run: | - RESULTS_PATH="${RESULTS_DIR}/${PT_INDUCTOR_RESULTS_JSON}" - docker run --mount="type=bind,src="${PWD}",target=/work" --workdir="/work" \ - "gcr.io/iree-oss/openxla-benchmark/base@sha256:692bb3ca648c474e89bad21ef89f4b33a9741e3d1cc477fa61844caddd798ba0" \ - ./comparative_benchmark/pt_inductor/benchmark_all.sh \ - "${TARGET_DEVICE}"\ - "${RESULTS_PATH}" - gcloud storage cp "${RESULTS_PATH}" "${RESULTS_GCS_DIR}/" diff --git a/common_benchmark_suite/openxla/benchmark/devices/gcp_devices.py b/common_benchmark_suite/openxla/benchmark/devices/gcp_devices.py index 837a9d48..60b92688 100644 --- a/common_benchmark_suite/openxla/benchmark/devices/gcp_devices.py +++ b/common_benchmark_suite/openxla/benchmark/devices/gcp_devices.py @@ -34,4 +34,19 @@ }, ) -ALL_DEVICES = [GCP_A2_HIGHGPU_1G, GCP_C2_STANDARD_16] +GCP_C2_STANDARD_60 = def_types.DeviceSpec( + id=unique_ids.DEVICE_SPEC_GCP_C2_STANDARD_60, + name="c2-standard-60", + host_type="gcp", + host_model="c2-standard-60", + host_environment="linux-x86_64", + accelerator_type="cpu", + accelerator_model="intel-cascadelake", + accelerator_architecture="x86_64-cascadelake", + accelerator_attributes={ + "num_of_cores": 30, + "hyper-threading": False, + }, +) + +ALL_DEVICES = [GCP_A2_HIGHGPU_1G, GCP_C2_STANDARD_16, GCP_C2_STANDARD_60] diff --git a/common_benchmark_suite/openxla/benchmark/unique_ids.py b/common_benchmark_suite/openxla/benchmark/unique_ids.py index 86349466..73b72aa8 100644 --- a/common_benchmark_suite/openxla/benchmark/unique_ids.py +++ b/common_benchmark_suite/openxla/benchmark/unique_ids.py @@ -168,3 +168,4 @@ ################################################################################ DEVICE_SPEC_GCP_C2_STANDARD_16 = "9a4804f1-b1b9-46cd-b251-7f16a655f782" DEVICE_SPEC_GCP_A2_HIGHGPU_1G = "78c56b95-2d7d-44b5-b5fd-8e47aa961108" +DEVICE_SPEC_GCP_C2_STANDARD_60 = "c416243f-90ba-45b8-9933-405e1078a71b" diff --git a/comparative_benchmark/jax_xla/benchmark_all.sh b/comparative_benchmark/jax_xla/benchmark_all.sh index 0a53a992..f0bbbc34 100755 --- a/comparative_benchmark/jax_xla/benchmark_all.sh +++ b/comparative_benchmark/jax_xla/benchmark_all.sh @@ -61,6 +61,9 @@ if [ "${TARGET_DEVICE}" = "a2-highgpu-1g" ]; then elif [ "${TARGET_DEVICE}" = "c2-standard-16" ]; then BENCHMARK_NAMES=("${CPU_BENCHMARK_NAMES[@]}") ITERATIONS=20 +elif [ "${TARGET_DEVICE}" = "c2-standard-60" ]; then + BENCHMARK_NAMES=("${CPU_BENCHMARK_NAMES[@]}") + ITERATIONS=20 else echo "Unsupported target device ${TARGET_DEVICE}." exit 1 diff --git a/comparative_benchmark/pt_inductor/benchmark_all.sh b/comparative_benchmark/pt_inductor/benchmark_all.sh index cb17c2d1..2672f38f 100755 --- a/comparative_benchmark/pt_inductor/benchmark_all.sh +++ b/comparative_benchmark/pt_inductor/benchmark_all.sh @@ -50,6 +50,9 @@ if [ "${TARGET_DEVICE}" = "a2-highgpu-1g" ]; then elif [ "${TARGET_DEVICE}" = "c2-standard-16" ]; then BENCHMARK_NAMES=("${CPU_BENCHMARK_NAMES[@]}") ITERATIONS=20 +elif [ "${TARGET_DEVICE}" = "c2-standard-60" ]; then + BENCHMARK_NAMES=("${CPU_BENCHMARK_NAMES[@]}") + ITERATIONS=20 else echo "Unsupported target device ${TARGET_DEVICE}." exit 1 diff --git a/comparative_benchmark/tf_xla/benchmark_all.sh b/comparative_benchmark/tf_xla/benchmark_all.sh index bf455376..16e478e8 100755 --- a/comparative_benchmark/tf_xla/benchmark_all.sh +++ b/comparative_benchmark/tf_xla/benchmark_all.sh @@ -57,6 +57,9 @@ if [ "${TARGET_DEVICE}" = "a2-highgpu-1g" ]; then elif [ "${TARGET_DEVICE}" = "c2-standard-16" ]; then BENCHMARK_NAMES=("${CPU_BENCHMARK_NAMES[@]}") ITERATIONS=20 +elif [ "${TARGET_DEVICE}" = "c2-standard-60" ]; then + BENCHMARK_NAMES=("${CPU_BENCHMARK_NAMES[@]}") + ITERATIONS=20 else echo "Unsupported target device ${TARGET_DEVICE}." exit 1 diff --git a/comparative_benchmark/xla_hlo/benchmark_all.sh b/comparative_benchmark/xla_hlo/benchmark_all.sh index 86138250..a083ea55 100755 --- a/comparative_benchmark/xla_hlo/benchmark_all.sh +++ b/comparative_benchmark/xla_hlo/benchmark_all.sh @@ -69,6 +69,13 @@ elif [ "${TARGET_DEVICE}" = "c2-standard-16" ]; then BENCHMARK_NAMES=("${CPU_BENCHMARK_NAMES[@]}") HLO_TOOL="run_hlo_module" ITERATIONS=5 +elif [ "${TARGET_DEVICE}" = "c2-standard-60" ]; then + # Since each iteration includes both compilation and inference, we keep the + # total iterations small because of the amount of time it takes to do both. + # Std deviation is <1ms. + BENCHMARK_NAMES=("${CPU_BENCHMARK_NAMES[@]}") + HLO_TOOL="run_hlo_module" + ITERATIONS=5 else echo "Unsupported target device ${TARGET_DEVICE}." exit 1