benchmarks.yml

name: API Performance Benchmarks

on:
  push:
    branches:
      - master

permissions:
  contents: read

jobs:
  macos:
    if: github.repository == 'tensorflow/io' # Don't do this in forks
    name: macOS ${{ matrix.python }} + ${{ matrix.version }}
    runs-on: macos-latest
    strategy:
      fail-fast: false
      matrix:
        python: ['3.8']
        version: ['tensorflow:tensorflow-io-nightly', 'tensorflow:tensorflow-io']
    steps:
      - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
        with:
          python-version: ${{ matrix.python }}
      - name: Setup macOS
        run: |
          set -x -e
          python -m pip install -U wheel setuptools
          python --version
      - name: Benchmark on macOS
        run: |
          set -x -e
          python --version
          df -h
          rm -rf tensorflow_io
          rm -rf tensorflow_io_gcs_filesystem
          export TF_USE_MODULAR_FILESYSTEM=1
          echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
          echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
          python -m pip install tensorflow-io-gcs-filesystem-nightly
          python -m pip install -q scikit-image pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==2.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 google-cloud-bigtable==1.6.0 azure-storage-blob==12.7.1
          python -m pip freeze
          python -c 'import tensorflow as tf; print(tf.version.VERSION)'
          python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)'
          python -c 'from tensorflow_io_gcs_filesystem.core.python.ops import plugin_gs; print(plugin_gs)'
          python -m pytest --benchmark-only -v --import-mode=append $(find . -type f \( -iname "test_*.py" ! \( -iname "test_*_v1.py" -o -iname "test_bigquery.py" \) \))

  linux:
    if: github.repository == 'tensorflow/io' # Don't do this in forks
    name: Linux ${{ matrix.python }} + ${{ matrix.version }}
    runs-on: ubuntu-20.04
    permissions:
      contents: write # to allow sending a commit comment for the benchmark action
    strategy:
      fail-fast: false
      matrix:
        python: ['3.8']
        version: ['tensorflow:tensorflow-io-nightly', 'tensorflow:tensorflow-io']
    steps:
      - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
        with:
          fetch-depth: 0
      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
        with:
          python-version: ${{ matrix.python }}
      - name: Setup Linux
        run: |
          set -x -e
          bash -x -e .github/workflows/build.space.sh
          bash -x -e tests/test_sql/sql_test.sh
      - name: Benchmark on Linux
        run: |
          set -x -e
          python --version
          df -h
          rm -rf tensorflow_io
          rm -rf tensorflow_io_gcs_filesystem
          export TF_USE_MODULAR_FILESYSTEM=1
          echo ${{ matrix.version }} | awk -F: '{print $1}' | xargs python -m pip install -U
          echo ${{ matrix.version }} | awk -F: '{print $2}' | xargs python -m pip install --no-deps -U
          python -m pip install tensorflow-io-gcs-filesystem-nightly
          python -m pip install -q scikit-image pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==2.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 google-cloud-bigtable==1.6.0 azure-storage-blob==12.7.1
          python -m pip freeze
          python -c 'import tensorflow as tf; print(tf.version.VERSION)'
          python -c 'import tensorflow_io as tfio; print(tfio.version.VERSION)'
          python -c 'from tensorflow_io_gcs_filesystem.core.python.ops import plugin_gs; print(plugin_gs)'
          python -m pytest --benchmark-only --benchmark-json benchmark.json -v --import-mode=append $(find . -type f \( -iname "test_*.py" ! \( -iname "test_*_v1.py" -o -iname "test_bigquery.py" \) \))
      - name: Store benchmark result
        uses: rhysd/github-action-benchmark@1846227a307d8c0149b960b986d46f8f4c95db0c # v1.20.1
        with:
          name: Tensorflow-IO Benchmarks
          tool: 'pytest'
          output-file-path: benchmark.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          auto-push: true