action.yml

# action.yml
name: 'ClearML detect model degradation'
description: 'Search ClearML for a task corresponding to the current PR and compare its performance to the previous best task.'
inputs:
  CLEARML_API_ACCESS_KEY:
    description: 'ClearML api access key (can be generated in your profile page)'
    required: true
  CLEARML_API_SECRET_KEY:
    description: 'ClearML api secret key (can be generated in your profile page)'
    required: true
  CLEARML_API_HOST:
    description: 'ClearML server api host'
    required: true
  CLEARML_PROJECT:
    description: 'Which project to search in for the task.'
    required: true
  CLEARML_TASK_NAME:
    description: 'Which task name to search for.'
    required: true
  CLEARML_SCALAR_TITLE:
    description: 'Which scalar to use for comparison. Title of the scalar.'
    required: true
  CLEARML_SCALAR_SERIES:
    description: 'Which scalar to use for comparison. Series to use within given plot.'
    required: true
  CLEARML_SCALAR_MIN_MAX:
    description: 'Whether smaller is better (MIN) or larger is better (MAX). (default: "MAX")'
    default: 'MAX'
  CLEARML_BEST_TAGNAME:
    description: 'The name of tag to be given to the best task. Every task that is checked and is equal or better than the previous best will get this tag. (default: "Best Performance")'
    default: 'Best Performance'
  CLEARML_SCALAR_THRESHOLD:
    description: 'The threshold for the difference between the previous best and the current commit. This should be a percentage of the previous best. E.g. CLEARML_SCALAR_THRESHOLD=3 -> if previous best PR is metric value 100 and CLEARML_SCALAR_MIN_MAX is max, then current PR can have a minimum metric value of 97 while still passing. (default: 0)'
    default: '0'
runs:
  using: 'composite'
  steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v4
      with:
        python-version: '3.10'
    - name: Install ClearML
      shell: bash
      run: pip install -r ${{ github.action_path }}/requirements.txt
    - name: Start the task
      shell: bash
      run: python ${{ github.action_path }}/compare_models.py
      env:
        CLEARML_API_ACCESS_KEY: ${{ inputs.CLEARML_API_ACCESS_KEY }}
        CLEARML_API_SECRET_KEY: ${{ inputs.CLEARML_API_SECRET_KEY }}
        CLEARML_API_HOST: ${{ inputs.CLEARML_API_HOST }}
        CLEARML_PROJECT: ${{ inputs.CLEARML_PROJECT }}
        CLEARML_TASK_NAME: ${{ inputs.CLEARML_TASK_NAME }}
        CLEARML_SCALAR_TITLE: ${{ inputs.CLEARML_SCALAR_TITLE }}
        CLEARML_SCALAR_SERIES: ${{ inputs.CLEARML_SCALAR_SERIES }}
        CLEARML_BEST_TAGNAME: ${{ inputs.CLEARML_BEST_TAGNAME }}
        CLEARML_SCALAR_MIN_MAX: ${{ inputs.CLEARML_SCALAR_MIN_MAX }}
        CLEARML_SCALAR_THRESHOLD: ${{ inputs.CLEARML_SCALAR_THRESHOLD }}