upload-to-pypi #35
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: upload-to-pypi | |
| on: | |
| create: | |
| tags: | |
| - "*" | |
| env: | |
| WORKSPACE_PREFIX: $(echo $GITHUB_WORKSPACE |cut -d '/' -f 1-4) | |
| SLURM_PARTITION: llm_s | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} | |
| jobs: | |
| build-and-upload: | |
| runs-on: [t_cluster] | |
| steps: | |
| - name: mask env | |
| run: | | |
| echo "::add-mask::${{env.WORKSPACE_PREFIX}}" | |
| echo "::add-mask::$path_prefix" | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: install dependencies | |
| run: | | |
| pip install setuptools wheel twine | |
| - name: build and upload package | |
| run: | | |
| source activate ${evo_env_torch21_flash2} | |
| python_path=$(which python) && echo "Python executable is at: $python_path" | |
| latest_tag=$(git describe --tags --abbrev=0) | |
| echo "$latest_tag" > version.txt | |
| export PYTHONPATH=$PWD:$PYTHONPATH | |
| export LLMPLATFORM=/mnt/petrelfs/share_data/llm_env | |
| export CUDA_PATH=${LLMPLATFORM}/dep/cuda-11.8 | |
| export GCC_HOME=${LLMPLATFORM}/dep/gcc-10.2.0 | |
| export MPFR_HOME=${LLMPLATFORM}/dep/mpfr-4.1.0 | |
| export LD_LIBRARY_PATH=${CUDA_PATH}/lib64:${CUDA_PATH}/extras/CUPTI/lib64/:$LD_LIBRARY_PATH | |
| export LD_LIBRARY_PATH=${GCC_HOME}/lib64:$LD_LIBRARY_PATH | |
| export LD_LIBRARY_PATH=${MPFR_HOME}/lib:$LD_LIBRARY_PATH | |
| export CC=${GCC_HOME}/bin/gcc | |
| export CXX=${GCC_HOME}/bin/c++ | |
| jobname=${GITHUB_RUN_ID}-${GITHUB_JOB}-${GITHUB_RUN_ATTEMPT} | |
| srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=$jobname -N 1 -n 1 --gres=gpu:1 python setup.py sdist bdist_wheel | |
| cd csrc/rotary/ | |
| srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=$jobname -N 1 -n 1 --gres=gpu:1 python setup.py sdist bdist_wheel | |
| cd ../xentropy/ | |
| srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=$jobname -N 1 -n 1 --gres=gpu:1 python setup.py sdist bdist_wheel | |
| cd ../../ | |
| exit_code=$? | |
| twine upload -u __token__ -p ${{ secrets.PYPI_API_TOKEN }} dist/* | |
| twine upload -u __token__ -p ${{ secrets.PYPI_API_TOKEN }} csrc/rotary/dist/* | |
| twine upload -u __token__ -p ${{ secrets.PYPI_API_TOKEN }} csrc/xentropy/dist/* | |
| sh ./ci_scripts/common/check_slurm_cancled.sh $exit_code $jobname |