upload_to_pypi.yaml

name: upload-to-pypi

on:
  create:
    tags:
      - "*"

env:
  WORKSPACE_PREFIX: $(echo $GITHUB_WORKSPACE |cut -d '/' -f 1-4)
  SLURM_PARTITION: llm_s
  TWINE_USERNAME: __token__
  TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}

jobs:
  build-and-upload:
    runs-on: [t_cluster]
    steps:
    - name: mask env
      run: |
        echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
        echo "::add-mask::$path_prefix"
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - name: install dependencies
      run: |
        pip install setuptools wheel twine

    - name: build and upload package
      run: |
        source activate ${evo_env_torch21_flash2}
        python_path=$(which python) && echo "Python executable is at: $python_path"
        latest_tag=$(git describe --tags --abbrev=0)
        echo "$latest_tag" > version.txt
        export PYTHONPATH=$PWD:$PYTHONPATH
        export LLMPLATFORM=/mnt/petrelfs/share_data/llm_env
        export CUDA_PATH=${LLMPLATFORM}/dep/cuda-11.8
        export GCC_HOME=${LLMPLATFORM}/dep/gcc-10.2.0
        export MPFR_HOME=${LLMPLATFORM}/dep/mpfr-4.1.0
        export LD_LIBRARY_PATH=${CUDA_PATH}/lib64:${CUDA_PATH}/extras/CUPTI/lib64/:$LD_LIBRARY_PATH
        export LD_LIBRARY_PATH=${GCC_HOME}/lib64:$LD_LIBRARY_PATH
        export LD_LIBRARY_PATH=${MPFR_HOME}/lib:$LD_LIBRARY_PATH
        export CC=${GCC_HOME}/bin/gcc
        export CXX=${GCC_HOME}/bin/c++
        jobname=${GITHUB_RUN_ID}-${GITHUB_JOB}-${GITHUB_RUN_ATTEMPT}
        srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=$jobname -N 1 -n 1 --gres=gpu:1 python setup.py sdist bdist_wheel
        cd csrc/rotary/
        srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=$jobname -N 1 -n 1 --gres=gpu:1 python setup.py sdist bdist_wheel
        cd ../xentropy/
        srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=$jobname -N 1 -n 1 --gres=gpu:1 python setup.py sdist bdist_wheel
        cd ../../
        exit_code=$?
        twine upload -u __token__ -p ${{ secrets.PYPI_API_TOKEN }} dist/*
        twine upload -u __token__ -p ${{ secrets.PYPI_API_TOKEN }} csrc/rotary/dist/*
        twine upload -u __token__ -p ${{ secrets.PYPI_API_TOKEN }} csrc/xentropy/dist/*
        sh ./ci_scripts/common/check_slurm_cancled.sh $exit_code $jobname