From 7ad9bd14d887c55f9786247e5a5bbda4ac2565a2 Mon Sep 17 00:00:00 2001 From: Frank Lee Date: Wed, 23 Nov 2022 15:52:42 +0800 Subject: [PATCH] [workflow] added conda cache and fixed no-compilation bug in release (#2005) --- .github/workflows/release_bdist.yml | 12 ++++++++++++ .github/workflows/scripts/build_colossalai_wheel.sh | 7 ++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_bdist.yml b/.github/workflows/release_bdist.yml index aeac3e327..ef75cef4b 100644 --- a/.github/workflows/release_bdist.yml +++ b/.github/workflows/release_bdist.yml @@ -64,9 +64,21 @@ jobs: - name: Copy scripts and checkout run: | cp -r ./.github/workflows/scripts/* ./ + + # link the cache diretories to current path + ln -s /github/home/conda_pkgs ./conda_pkgs ln -s /github/home/pip_wheels ./pip_wheels + + # set the conda package path + echo pkgs_dirs:\n\t- $PWD/conda_pkgs > ~/.condarc + + # set safe directory git config --global --add safe.directory /__w/ColossalAI/ColossalAI + + # check out git checkout $git_ref + + # get cub package for cuda 10.2 wget https://github.com/NVIDIA/cub/archive/refs/tags/1.8.0.zip unzip 1.8.0.zip env: diff --git a/.github/workflows/scripts/build_colossalai_wheel.sh b/.github/workflows/scripts/build_colossalai_wheel.sh index 55a87d956..c0d40fd2c 100644 --- a/.github/workflows/scripts/build_colossalai_wheel.sh +++ b/.github/workflows/scripts/build_colossalai_wheel.sh @@ -18,7 +18,7 @@ if [ $1 == "pip" ] then wget -nc -q -O ./pip_wheels/$filename $url pip install ./pip_wheels/$filename - + elif [ $1 == 'conda' ] then conda install pytorch==$torch_version cudatoolkit=$cuda_version $flags @@ -34,8 +34,9 @@ fi python setup.py bdist_wheel mv ./dist/* ./all_dist +# must remove build to enable compilation for +# cuda extension in the next build +rm -rf ./build python setup.py clean conda deactivate conda env remove -n $python_version - -