mirror of https://github.com/hpcaitech/ColossalAI
[ci] added wheel build scripts (#910)
* [ci] added wheel build scripts * polish code and workflow * polish code and workflow * polish code and workflow * polish code and workflow * polish code and workflow * polish code and workflow * polish code and workflow * polish code and workflow * polish code and workflow * polish code and workflow * polish code and workflow * [ci] polish wheel build scriptspull/912/head
Frank Lee
GitHub
parent
150b1a7453
commit
f0f35216f1
|
|
@ -3,18 +3,21 @@ name: Release bdist wheel
|
|||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
cuda_version:
|
||||
type: choice
|
||||
description: version for testing
|
||||
description: CUDA Version
|
||||
default: 'all'
|
||||
required: true
|
||||
options:
|
||||
- all
|
||||
- pytorch-cuda:1.9.0-11.1.1 # python 3.8
|
||||
- pytorch-cuda:1.8.1-11.1.1 # python 3.8
|
||||
- pytorch-cuda:1.7.1-11.0.3 # python 3.8
|
||||
- pytorch-cuda:1.6.0-10.2 # python 3.6
|
||||
|
||||
- "11.3"
|
||||
- "11.1"
|
||||
- "10.2"
|
||||
github_ref:
|
||||
type: string
|
||||
description: Branch or Tag
|
||||
default: 'main'
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
matrix_preparation:
|
||||
|
|
@ -25,9 +28,9 @@ jobs:
|
|||
steps:
|
||||
- id: set-matrix
|
||||
run: |
|
||||
[ "${{github.event.inputs.version}}" != "" ] && matrix="[\"frankleeeee/${{github.event.inputs.version}}\"]"
|
||||
[ "${{github.event.inputs.version}}" == "" ] || [ "${{github.event.inputs.version}}" == "all" ] && \
|
||||
matrix="[\"frankleeeee/pytorch-cuda:1.9.0-11.1.1\", \"frankleeeee/pytorch-cuda:1.8.1-11.1.1\", \"frankleeeee/pytorch-cuda:1.7.1-11.0.3\", \"frankleeeee/pytorch-cuda:1.6.0-10.2\"]"
|
||||
[ "${{github.event.inputs.cuda_version}}" != "" ] && matrix="[\"frankleeeee/cuda-conda:${{github.event.inputs.cuda_version}}\"]"
|
||||
[ "${{github.event.inputs.cuda_version}}" == "" ] || [ "${{github.event.inputs.version}}" == "all" ] && \
|
||||
matrix="[\"frankleeeee/cuda-conda:11.3\", \"frankleeeee/cuda-conda:11.1\", \"frankleeeee/cuda-conda:10.2\"]"
|
||||
echo $matrix
|
||||
echo "::set-output name=matrix::{\"container\":$(echo $matrix)}"
|
||||
|
||||
|
|
@ -42,20 +45,28 @@ jobs:
|
|||
container:
|
||||
image: ${{ matrix.container }}
|
||||
options: --gpus all --rm
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
|
||||
pip install -U pip setuptools wheel --user
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
# cub is for cuda 10.2
|
||||
- name: Copy scripts and checkout
|
||||
run: |
|
||||
cp -r ./.github/workflows/scripts/* ./
|
||||
ln -s /github/home/pip_wheels ./pip_wheels
|
||||
git checkout $git_ref
|
||||
wget https://github.com/NVIDIA/cub/archive/refs/tags/1.8.0.zip
|
||||
unzip 1.8.0.zip
|
||||
env:
|
||||
git_ref: ${{ github.event.inputs.github_ref }}
|
||||
- name: Build bdist wheel
|
||||
run: |
|
||||
python setup.py bdist_wheel
|
||||
pip install beautifulsoup4 requests
|
||||
python ./build_colossalai_wheel.py
|
||||
- name: 🚀 Deploy
|
||||
uses: garygrossgarten/github-action-scp@release
|
||||
with:
|
||||
local: dist
|
||||
local: all_dist
|
||||
remote: ${{ secrets.PRIVATE_PYPI_DIR }}
|
||||
host: ${{ secrets.PRIVATE_PYPI_HOST }}
|
||||
username: ${{ secrets.PRIVATE_PYPI_USER }}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,82 @@
|
|||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
import re
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
|
||||
WHEEL_TEXT_ROOT_URL = 'https://github.com/hpcaitech/public_assets/tree/main/colossalai/torch_build/torch_wheels'
|
||||
RAW_TEXT_FILE_PREFIX = 'https://raw.githubusercontent.com/hpcaitech/public_assets/main/colossalai/torch_build/torch_wheels'
|
||||
CUDA_HOME = os.environ['CUDA_HOME']
|
||||
|
||||
def get_cuda_bare_metal_version():
|
||||
raw_output = subprocess.check_output([CUDA_HOME + "/bin/nvcc", "-V"], universal_newlines=True)
|
||||
output = raw_output.split()
|
||||
release_idx = output.index("release") + 1
|
||||
release = output[release_idx].split(".")
|
||||
bare_metal_major = release[0]
|
||||
bare_metal_minor = release[1][0]
|
||||
|
||||
return bare_metal_major, bare_metal_minor
|
||||
|
||||
def all_wheel_info():
|
||||
page_text = requests.get(WHEEL_TEXT_ROOT_URL).text
|
||||
soup = BeautifulSoup(page_text)
|
||||
|
||||
all_a_links = soup.find_all('a')
|
||||
|
||||
wheel_info = dict()
|
||||
|
||||
for a_link in all_a_links:
|
||||
if 'cuda' in a_link.text and '.txt' in a_link.text:
|
||||
filename = a_link.text
|
||||
torch_version, cuda_version = filename.rstrip('.txt').split('-')
|
||||
cuda_version = cuda_version.lstrip('cuda')
|
||||
|
||||
if torch_version not in wheel_info:
|
||||
wheel_info[torch_version] = dict()
|
||||
wheel_info[torch_version][cuda_version] = dict()
|
||||
|
||||
file_text = requests.get(f'{RAW_TEXT_FILE_PREFIX}/{filename}').text
|
||||
lines = file_text.strip().split('\n')
|
||||
|
||||
for line in lines:
|
||||
parts = line.split('\t')
|
||||
method, url, python_version = parts[:3]
|
||||
|
||||
if len(parts) > 3:
|
||||
flags = parts[3]
|
||||
flags = ' '.join(flags.split('+'))
|
||||
else:
|
||||
flags = ''
|
||||
wheel_info[torch_version][cuda_version][python_version] = dict(method=method, url=url, flags=flags)
|
||||
return wheel_info
|
||||
|
||||
|
||||
def build_colossalai(wheel_info):
|
||||
cuda_version_major, cuda_version_minor = get_cuda_bare_metal_version()
|
||||
cuda_version_on_host = f'{cuda_version_major}.{cuda_version_minor}'
|
||||
|
||||
for torch_version, cuda_versioned_wheel_info in wheel_info.items():
|
||||
for cuda_version, python_versioned_wheel_info in cuda_versioned_wheel_info.items():
|
||||
if cuda_version_on_host == cuda_version:
|
||||
for python_version, wheel_info in python_versioned_wheel_info.items():
|
||||
url = wheel_info['url']
|
||||
method = wheel_info['method']
|
||||
flags = wheel_info['flags']
|
||||
filename = url.split('/')[-1].replace('%2B', '+')
|
||||
cmd = f'bash ./build_colossalai_wheel.sh {method} {url} {filename} {cuda_version} {python_version} {torch_version} {flags}'
|
||||
os.system(cmd)
|
||||
|
||||
def main():
|
||||
wheel_info = all_wheel_info()
|
||||
build_colossalai(wheel_info)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
method=${1}
|
||||
url=${2}
|
||||
filename=${3}
|
||||
cuda_version=${4}
|
||||
python_version=${5}
|
||||
torch_version=${6}
|
||||
flags=${@:7}
|
||||
|
||||
git reset --hard HEAD
|
||||
mkdir -p ./all_dist
|
||||
source activate base
|
||||
conda create -n $python_version -y python=$python_version
|
||||
source activate $python_version
|
||||
|
||||
if [ $1 == "pip" ]
|
||||
then
|
||||
wget -nc -q -O ./pip_wheels/$filename $url
|
||||
pip install ./pip_wheels/$filename
|
||||
|
||||
elif [ $1 == 'conda' ]
|
||||
then
|
||||
conda install pytorch==$torch_version cudatoolkit=$cuda_version $flags
|
||||
else
|
||||
echo Invalid installation method
|
||||
exit
|
||||
fi
|
||||
|
||||
if [ $cuda_version == "10.2" ]
|
||||
then
|
||||
cp -r cub-1.8.0/cub/ colossalai/kernel/cuda_native/csrc/kernels/include/
|
||||
fi
|
||||
|
||||
python setup.py bdist_wheel
|
||||
mv ./dist/* ./all_dist
|
||||
python setup.py clean
|
||||
conda env remove -n $python_version
|
||||
|
||||
|
||||
Loading…
Reference in New Issue