name: 'e2e test'
on:
workflow_call:
inputs:
vllm:
required: true
type: string
image:
required: true
type: string
image_a3:
required: false
type: string
default: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-a3-ubuntu22.04-py3.11
image_310p:
required: false
type: string
default: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:9.0.0-310p-ubuntu22.04-py3.11
type:
required: true
type: string
contains_310:
required: false
type: boolean
default: false
continue_on_error:
required: false
type: boolean
default: false
ref:
required: false
type: string
default: ''
singlecard_tests:
required: false
type: string
default: ''
multicard_2_tests:
required: false
type: string
default: ''
multicard_4_tests:
required: false
type: string
default: ''
p310_tests:
required: false
type: string
default: ''
env:
UV_INDEX_URL: http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple
UV_EXTRA_INDEX_URL: "https://repo.huaweicloud.com/ascend/repos/pypi http://cache-service.nginx-pypi-cache.svc.cluster.local/whl/cpu/"
UV_INDEX_STRATEGY: unsafe-best-match
UV_INSECURE_HOST: cache-service.nginx-pypi-cache.svc.cluster.local
UV_HTTP_TIMEOUT: 120
UV_NO_CACHE: 1
UV_SYSTEM_PYTHON: 1
jobs:
e2e-light:
name: singlecard-light
if: ${{ inputs.type == 'light' }}
runs-on: linux-aarch64-a2b3-1
strategy:
fail-fast: false
matrix:
part: [0]
container:
image: ${{ inputs.image }}
env:
VLLM_LOGGING_LEVEL: ERROR
VLLM_USE_MODELSCOPE: True
HF_HUB_OFFLINE: 1
steps:
- name: Checkout vllm-project/vllm-ascend repo
uses: actions/checkout@v6
- name: Check npu and CANN info
run: |
cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
npu-smi info
- name: Config mirrors
run: |
sed -Ei 's@(ports|archive).ubuntu.com@cache-service.nginx-pypi-cache.svc.cluster.local:8081@g' /etc/apt/sources.list
pip config set global.index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple
pip config set global.trusted-host cache-service.nginx-pypi-cache.svc.cluster.local
apt-get update -y
apt install git -y
git config --global --add safe.directory /__w/vllm-ascend/vllm-ascend
- name: Install system dependencies
run: |
apt-get -y install `cat packages.txt`
apt-get -y install gcc g++ cmake libnuma-dev clang-15
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 20
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-15 20
pip install uv
- name: Checkout vllm-project/vllm repo
uses: actions/checkout@v6
with:
repository: vllm-project/vllm
ref: ${{ inputs.vllm }}
path: ./vllm-empty
fetch-depth: 1
- name: Install vllm-project/vllm from source
working-directory: ./vllm-empty
run: |
VLLM_TARGET_DEVICE=empty uv pip install -e .
pip uninstall -y triton
- name: Get csrc hash
id: get_csrc_hash
run: |
CSRC_HASH=$(find ./csrc ./setup.py ./CMakeLists.txt ./cmake \
-type f -not -path '*/.*' | sort | xargs sha256sum | sha256sum | awk '{print $1}')
echo "CSRC_HASH=$CSRC_HASH" >> $GITHUB_OUTPUT
- name: Cache vllm-ascend csrc
uses: runs-on/cache@v4
with:
path: |
vllm_ascend/_cann_ops_custom
vllm_ascend/*.so
vllm_ascend/lib
vllm_ascend/include
key: vllm-ascend-build-v1-${{ runner.os }}-${{ inputs.image }}-${{ steps.get_csrc_hash.outputs.CSRC_HASH }}
restore-keys: |
vllm-ascend-build-v1-${{ runner.os }}-${{ inputs.image }}-${{ steps.get_csrc_hash.outputs.CSRC_HASH }}
- name: Install vllm-project/vllm-ascend
env:
MAX_JOBS: "23"
run: |
pip install uc-manager
uv pip install -r requirements-dev.txt
uv pip install --force-reinstall --no-deps triton-ascend==3.2.1
if find vllm_ascend -maxdepth 1 -name '*.so' -type f 2>/dev/null | grep -q .; then
COMPILE_CUSTOM_KERNELS=0 uv pip install -e .
else
uv pip install -e .
fi
- name: Run vllm-project/vllm-ascend test
env:
PYTORCH_NPU_ALLOC_CONF: max_split_size_mb:256
VLLM_WORKER_MULTIPROC_METHOD: spawn
shell: bash
run: |
set -o pipefail
if [ "${{ inputs.continue_on_error }}" = "true" ]; then
python3 .github/workflows/scripts/run_suite.py \
--suite e2e-singlecard-light \
--auto-partition-id "${{ matrix.part }}" \
--auto-partition-size 1 \
--auto-upgrade-estimated-times \
--continue-on-error \
2>&1 | tee /tmp/e2e-singlecard-light-part${{ matrix.part }}.log
else
python3 .github/workflows/scripts/run_suite.py \
--suite e2e-singlecard-light \
--auto-partition-id "${{ matrix.part }}" \
--auto-partition-size 1 \
2>&1 | tee /tmp/e2e-singlecard-light-part${{ matrix.part }}.log
fi
exit ${PIPESTATUS[0]}
- name: Summarize singlecard-light failure
if: ${{ always() }}
run: |
python3 .github/workflows/scripts/ci_log_summary.py \
--step-name "Run singlecard-light test" \
--log-file /tmp/e2e-singlecard-light-part${{ matrix.part }}.log \
--output "$GITHUB_STEP_SUMMARY"
- name: Upload timing data
uses: actions/upload-artifact@v7
if: ${{ inputs.continue_on_error == true && github.event_name != 'pull_request' }}
with:
name: timing-data-singlecard-light-part${{ matrix.part }}
path: test_timing_data.json
if-no-files-found: warn
retention-days: 5
e2e-full:
name: singlecard-full
if: ${{ inputs.type == 'full' || inputs.singlecard_tests != '' }}
runs-on: linux-aarch64-a2b3-1
strategy:
fail-fast: false
matrix:
part: ${{ fromJSON(inputs.type == 'full' && '[0, 1]' || '[0]') }}
container:
image: ${{ inputs.image }}
env:
VLLM_LOGGING_LEVEL: ERROR
VLLM_USE_MODELSCOPE: True
HF_HUB_OFFLINE: 1
MODELSCOPE_HUB_FILE_LOCK: False
steps:
- name: Checkout vllm-project/vllm-ascend repo
uses: actions/checkout@v6
with:
ref: ${{ inputs.ref }}
- name: Check npu and CANN info
run: |
npu-smi info
cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
- name: Config mirrors
run: |
sed -Ei 's@(ports|archive).ubuntu.com@cache-service.nginx-pypi-cache.svc.cluster.local:8081@g' /etc/apt/sources.list
pip config set global.index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple
pip config set global.trusted-host cache-service.nginx-pypi-cache.svc.cluster.local
apt-get update -y
apt install git -y
git config --global --add safe.directory /__w/vllm-ascend/vllm-ascend
- name: Install system dependencies
run: |
apt-get -y install `cat packages.txt`
apt-get -y install gcc g++ cmake libnuma-dev clang-15
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 20
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-15 20
pip install uv
- name: Checkout vllm-project/vllm repo
uses: actions/checkout@v6
with:
repository: vllm-project/vllm
ref: ${{ inputs.vllm }}
path: ./vllm-empty
fetch-depth: 1
- name: Install vllm-project/vllm from source
working-directory: ./vllm-empty
run: |
VLLM_TARGET_DEVICE=empty uv pip install -e .
pip uninstall -y triton
- name: Get csrc hash
id: get_csrc_hash
run: |
CSRC_HASH=$(find ./csrc ./setup.py ./CMakeLists.txt ./cmake \
-type f -not -path '*/.*' | sort | xargs sha256sum | sha256sum | awk '{print $1}')
echo "CSRC_HASH=$CSRC_HASH" >> $GITHUB_OUTPUT
- name: Cache vllm-ascend csrc
uses: runs-on/cache@v4
with:
path: |
vllm_ascend/_cann_ops_custom
vllm_ascend/*.so
vllm_ascend/lib
vllm_ascend/include
key: vllm-ascend-build-v1-${{ runner.os }}-${{ inputs.image }}-${{ steps.get_csrc_hash.outputs.CSRC_HASH }}
restore-keys: |
vllm-ascend-build-v1-${{ runner.os }}-${{ inputs.image }}-${{ steps.get_csrc_hash.outputs.CSRC_HASH }}
- name: Install vllm-project/vllm-ascend
env:
MAX_JOBS: "23"
run: |
pip install uc-manager
uv pip install -r requirements-dev.txt
uv pip install --force-reinstall --no-deps triton-ascend==3.2.1
if find vllm_ascend -maxdepth 1 -name '*.so' -type f 2>/dev/null | grep -q .; then
COMPILE_CUSTOM_KERNELS=0 uv pip install -e .
else
uv pip install -e .
fi
- name: Run e2e test
if: ${{ inputs.type == 'full' }}
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
PYTORCH_NPU_ALLOC_CONF: max_split_size_mb:256
shell: bash
run: |
set -o pipefail
python -c "from triton.backends import backends"
if [ "${{ inputs.continue_on_error }}" = "true" ]; then
python3 .github/workflows/scripts/run_suite.py \
--suite e2e-singlecard \
--auto-partition-id "${{ matrix.part }}" \
--auto-partition-size 2 \
--auto-upgrade-estimated-times \
--continue-on-error \
2>&1 | tee /tmp/e2e-singlecard-full-part${{ matrix.part }}.log
else
python3 .github/workflows/scripts/run_suite.py \
--suite e2e-singlecard \
--auto-partition-id "${{ matrix.part }}" \
--auto-partition-size 2 \
2>&1 | tee /tmp/e2e-singlecard-full-part${{ matrix.part }}.log
fi
exit ${PIPESTATUS[0]}
- name: Summarize singlecard-full failure
if: ${{ always() && inputs.type == 'full' }}
run: |
python3 .github/workflows/scripts/ci_log_summary.py \
--step-name "Run singlecard-full test" \
--log-file /tmp/e2e-singlecard-full-part${{ matrix.part }}.log \
--output "$GITHUB_STEP_SUMMARY"
- name: Upload timing data
uses: actions/upload-artifact@v7
if: ${{ inputs.continue_on_error == true && github.event_name != 'pull_request' }}
with:
name: timing-data-singlecard-full-part${{ matrix.part }}
path: test_timing_data.json
if-no-files-found: warn
retention-days: 5
- name: Run E2E tests (comment-singlecard)
if: ${{ inputs.singlecard_tests != '' }}
env:
PYTORCH_NPU_ALLOC_CONF: max_split_size_mb:256
VLLM_WORKER_MULTIPROC_METHOD: spawn
run: |
pytest -sv --durations=0 ${{ inputs.singlecard_tests }}
e2e-2-cards-light:
name: multicard-2-light
if: ${{ inputs.type == 'light' }}
runs-on: linux-aarch64-a3-2
strategy:
fail-fast: false
matrix:
part: [0]
container:
image: ${{ inputs.image_a3 }}
env:
VLLM_LOGGING_LEVEL: ERROR
VLLM_USE_MODELSCOPE: True
HCCL_BUFFSIZE: 1024
HF_HUB_OFFLINE: 1
steps:
- name: Checkout vllm-project/vllm-ascend repo
uses: actions/checkout@v6
- name: Check npu and CANN info
run: |
npu-smi info
cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
- name: Config mirrors
run: |
sed -Ei 's@(ports|archive).ubuntu.com@cache-service.nginx-pypi-cache.svc.cluster.local:8081@g' /etc/apt/sources.list
pip config set global.index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple
pip config set global.trusted-host cache-service.nginx-pypi-cache.svc.cluster.local
apt-get update -y
apt install git -y
git config --global --add safe.directory /__w/vllm-ascend/vllm-ascend
- name: Install system dependencies
run: |
apt-get -y install `cat packages.txt`
apt-get -y install gcc g++ cmake libnuma-dev clang-15
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 20
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-15 20
pip install uv
- name: Checkout vllm-project/vllm repo
uses: actions/checkout@v6
with:
repository: vllm-project/vllm
ref: ${{ inputs.vllm }}
path: ./vllm-empty
fetch-depth: 1
- name: Install vllm-project/vllm from source
working-directory: ./vllm-empty
run: |
VLLM_TARGET_DEVICE=empty uv pip install -e .
pip uninstall -y triton
- name: Get csrc hash
id: get_csrc_hash
run: |
CSRC_HASH=$(find ./csrc ./setup.py ./CMakeLists.txt ./cmake \
-type f -not -path '*/.*' | sort | xargs sha256sum | sha256sum | awk '{print $1}')
echo "CSRC_HASH=$CSRC_HASH" >> $GITHUB_OUTPUT
- name: Cache vllm-ascend csrc
uses: runs-on/cache@v4
with:
path: |
vllm_ascend/_cann_ops_custom
vllm_ascend/*.so
vllm_ascend/lib
vllm_ascend/include
key: vllm-ascend-build-v1-${{ runner.os }}-${{ inputs.image_a3 }}-${{ steps.get_csrc_hash.outputs.CSRC_HASH }}
restore-keys: |
vllm-ascend-build-v1-${{ runner.os }}-${{ inputs.image_a3 }}-${{ steps.get_csrc_hash.outputs.CSRC_HASH }}
- name: Install vllm-project/vllm-ascend
env:
MAX_JOBS: "46"
run: |
pip install uc-manager
uv pip install -r requirements-dev.txt
uv pip install --force-reinstall --no-deps triton-ascend==3.2.1
if find vllm_ascend -maxdepth 1 -name '*.so' -type f 2>/dev/null | grep -q .; then
COMPILE_CUSTOM_KERNELS=0 uv pip install -e .
else
uv pip install -e .
fi
- name: Run vllm-project/vllm-ascend test (light)
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
shell: bash
run: |
set -o pipefail
if [ "${{ inputs.continue_on_error }}" = "true" ]; then
python3 .github/workflows/scripts/run_suite.py \
--suite e2e-2card-light \
--auto-partition-id "${{ matrix.part }}" \
--auto-partition-size 1 \
--auto-upgrade-estimated-times \
--continue-on-error \
2>&1 | tee /tmp/e2e-2card-light-part${{ matrix.part }}.log
else
python3 .github/workflows/scripts/run_suite.py \
--suite e2e-2card-light \
--auto-partition-id "${{ matrix.part }}" \
--auto-partition-size 1 \
2>&1 | tee /tmp/e2e-2card-light-part${{ matrix.part }}.log
fi
exit ${PIPESTATUS[0]}
- name: Summarize multicard-2-light failure
if: ${{ always() }}
run: |
python3 .github/workflows/scripts/ci_log_summary.py \
--step-name "Run multicard-2-light test" \
--log-file /tmp/e2e-2card-light-part${{ matrix.part }}.log \
--output "$GITHUB_STEP_SUMMARY"
- name: Upload timing data
uses: actions/upload-artifact@v7
if: ${{ inputs.continue_on_error == true && github.event_name != 'pull_request' }}
with:
name: timing-data-2card-light-part${{ matrix.part }}
path: test_timing_data.json
if-no-files-found: warn
retention-days: 5
e2e-2-cards-full:
name: multicard-2-full
if: ${{ inputs.type == 'full' || inputs.multicard_2_tests != '' }}
runs-on: linux-aarch64-a3-2
strategy:
fail-fast: false
matrix:
part: [0]
container:
image: ${{ inputs.image_a3 }}
env:
VLLM_LOGGING_LEVEL: ERROR
VLLM_USE_MODELSCOPE: True
HCCL_BUFFSIZE: 1024
HF_HUB_OFFLINE: 1
steps:
- name: Checkout vllm-project/vllm-ascend repo
uses: actions/checkout@v6
with:
ref: ${{ inputs.ref }}
- name: Check npu and CANN info
run: |
npu-smi info
cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
- name: Config mirrors
run: |
sed -Ei 's@(ports|archive).ubuntu.com@cache-service.nginx-pypi-cache.svc.cluster.local:8081@g' /etc/apt/sources.list
pip config set global.index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple
pip config set global.trusted-host cache-service.nginx-pypi-cache.svc.cluster.local
apt-get update -y
apt install git -y
git config --global --add safe.directory /__w/vllm-ascend/vllm-ascend
- name: Install system dependencies
run: |
apt-get -y install `cat packages.txt`
apt-get -y install gcc g++ cmake libnuma-dev clang-15
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 20
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-15 20
pip install uv
- name: Checkout vllm-project/vllm repo
uses: actions/checkout@v6
with:
repository: vllm-project/vllm
ref: ${{ inputs.vllm }}
path: ./vllm-empty
fetch-depth: 1
- name: Install vllm-project/vllm from source
working-directory: ./vllm-empty
run: |
VLLM_TARGET_DEVICE=empty uv pip install -e .
pip uninstall -y triton
- name: Get csrc hash
id: get_csrc_hash
run: |
CSRC_HASH=$(find ./csrc ./setup.py ./CMakeLists.txt ./cmake \
-type f -not -path '*/.*' | sort | xargs sha256sum | sha256sum | awk '{print $1}')
echo "CSRC_HASH=$CSRC_HASH" >> $GITHUB_OUTPUT
- name: Cache vllm-ascend csrc
uses: runs-on/cache@v4
with:
path: |
vllm_ascend/_cann_ops_custom
vllm_ascend/*.so
vllm_ascend/lib
vllm_ascend/include
key: vllm-ascend-build-v1-${{ runner.os }}-${{ inputs.image_a3 }}-${{ steps.get_csrc_hash.outputs.CSRC_HASH }}
restore-keys: |
vllm-ascend-build-v1-${{ runner.os }}-${{ inputs.image_a3 }}-${{ steps.get_csrc_hash.outputs.CSRC_HASH }}
- name: Install vllm-project/vllm-ascend
env:
MAX_JOBS: "46"
run: |
pip install uc-manager
uv pip install -r requirements-dev.txt
uv pip install --force-reinstall --no-deps triton-ascend==3.2.1
if find vllm_ascend -maxdepth 1 -name '*.so' -type f 2>/dev/null | grep -q .; then
COMPILE_CUSTOM_KERNELS=0 uv pip install -e .
else
uv pip install -e .
fi
- name: Run vllm-project/vllm-ascend test (full)
if: ${{ inputs.type == 'full' }}
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
shell: bash
run: |
set -o pipefail
if [ "${{ inputs.continue_on_error }}" = "true" ]; then
python3 .github/workflows/scripts/run_suite.py \
--suite e2e-multicard-2-cards \
--auto-partition-id "${{ matrix.part }}" \
--auto-partition-size 1 \
--auto-upgrade-estimated-times \
--continue-on-error \
2>&1 | tee /tmp/e2e-2card-full-part${{ matrix.part }}.log
else
python3 .github/workflows/scripts/run_suite.py \
--suite e2e-multicard-2-cards \
--auto-partition-id "${{ matrix.part }}" \
--auto-partition-size 1 \
2>&1 | tee /tmp/e2e-2card-full-part${{ matrix.part }}.log
fi
exit ${PIPESTATUS[0]}
- name: Summarize multicard-2-full failure
if: ${{ always() && inputs.type == 'full' }}
run: |
python3 .github/workflows/scripts/ci_log_summary.py \
--step-name "Run multicard-2-full test " \
--log-file /tmp/e2e-2card-full-part${{ matrix.part }}.log \
--output "$GITHUB_STEP_SUMMARY"
- name: Upload timing data
uses: actions/upload-artifact@v7
if: ${{ inputs.continue_on_error == true && github.event_name != 'pull_request' }}
with:
name: timing-data-2card-full-part${{ matrix.part }}
path: test_timing_data.json
if-no-files-found: warn
retention-days: 5
- name: Run E2E tests (comment-multicard-2)
if: ${{ inputs.multicard_2_tests != '' }}
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
run: |
pytest -sv --durations=0 ${{ inputs.multicard_2_tests }}
- name: Run vllm-project/vllm-ascend test (non triton)
if: ${{ inputs.type == 'full' && matrix.part == 0 }}
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
shell: bash
run: |
set -o pipefail
# we must uninstall triton-ascend first, then triton. Otherwise, the triton folder will be left behind.
python3 -m pip uninstall -y triton-ascend triton
python3 -m pip install regex
pytest -sv --durations=0 tests/e2e/multicard/2-cards/test_aclgraph_capture_replay.py \
2>&1 | tee /tmp/e2e-non-triton.log
exit ${PIPESTATUS[0]}
- name: Summarize non-triton failure
if: ${{ always() && inputs.type == 'full' && matrix.part == 0 }}
run: |
python3 .github/workflows/scripts/ci_log_summary.py \
--step-name "Run multicard-2-full test (non triton)" \
--log-file /tmp/e2e-non-triton.log \
--output "$GITHUB_STEP_SUMMARY"
e2e-4-cards-light:
name: multicard-4-light
if: ${{ inputs.type == 'light' }}
runs-on: linux-aarch64-a3-4
strategy:
fail-fast: false
matrix:
part: [0]
container:
image: ${{ inputs.image_a3 }}
env:
VLLM_LOGGING_LEVEL: ERROR
VLLM_USE_MODELSCOPE: True
HCCL_BUFFSIZE: 1024
HF_HUB_OFFLINE: 1
steps:
- name: Checkout vllm-project/vllm-ascend repo
uses: actions/checkout@v6
with:
ref: ${{ inputs.ref }}
- name: Set global envs
run: |
set -a
. /usr/local/Ascend/ascend-toolkit/set_env.sh
set +a
env >> $GITHUB_ENV
- name: Check npu and CANN info
run: |
npu-smi info
cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
- name: Config mirrors
run: |
sed -Ei 's@(ports|archive).ubuntu.com@cache-service.nginx-pypi-cache.svc.cluster.local:8081@g' /etc/apt/sources.list
pip config set global.index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple
pip config set global.trusted-host cache-service.nginx-pypi-cache.svc.cluster.local
apt-get update -y
apt install git -y
git config --global --add safe.directory /__w/vllm-ascend/vllm-ascend
- name: Install system dependencies
run: |
apt-get -y install `cat packages.txt`
apt-get -y install gcc g++ cmake libnuma-dev clang-15
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 20
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-15 20
pip install uv
- name: Checkout vllm-project/vllm repo
uses: actions/checkout@v6
with:
repository: vllm-project/vllm
ref: ${{ inputs.vllm }}
path: ./vllm-empty
fetch-depth: 1
- name: Install vllm-project/vllm from source
working-directory: ./vllm-empty
run: |
VLLM_TARGET_DEVICE=empty uv pip install -e .
pip uninstall -y triton
- name: Get csrc hash
id: get_csrc_hash
run: |
CSRC_HASH=$(find ./csrc ./setup.py ./CMakeLists.txt ./cmake \
-type f -not -path '*/.*' | sort | xargs sha256sum | sha256sum | awk '{print $1}')
echo "CSRC_HASH=$CSRC_HASH" >> $GITHUB_OUTPUT
- name: Cache vllm-ascend csrc
uses: runs-on/cache@v4
with:
path: |
vllm_ascend/_cann_ops_custom
vllm_ascend/*.so
vllm_ascend/lib
vllm_ascend/include
key: vllm-ascend-build-v1-${{ runner.os }}-${{ inputs.image_a3 }}-${{ steps.get_csrc_hash.outputs.CSRC_HASH }}
restore-keys: |
vllm-ascend-build-v1-${{ runner.os }}-${{ inputs.image_a3 }}-${{ steps.get_csrc_hash.outputs.CSRC_HASH }}
- name: Install Mooncake wheel
shell: bash
run: |
set -euxo pipefail
apt-get update -y
apt-get install -y --no-install-recommends \
libibverbs1 \
ibverbs-providers \
librdmacm1 \
libnuma1 \
libcurl4
ldconfig
MOONCAKE_WHEEL="mooncake_transfer_engine_ascend-0.3.8.post1-cp311-cp311-manylinux_2_17_aarch64.manylinux_2_35_aarch64.whl"
pip install --no-cache-dir --no-deps \
"https://vllm-ascend.obs.cn-north-4.myhuaweicloud.com/vllm-ascend/${MOONCAKE_WHEEL}"
pip show mooncake-transfer-engine-ascend || true
- name: Install vllm-project/vllm-ascend
env:
MAX_JOBS: "92"
run: |
pip install uc-manager
uv pip install -r requirements-dev.txt
uv pip install --force-reinstall --no-deps triton-ascend==3.2.1
if find vllm_ascend -maxdepth 1 -name '*.so' -type f 2>/dev/null | grep -q .; then
COMPILE_CUSTOM_KERNELS=0 uv pip install -e .
else
uv pip install -e .
fi
- name: Run vllm-project/vllm-ascend test (light)
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
shell: bash
run: |
set -o pipefail
if [ "${{ inputs.continue_on_error }}" = "true" ]; then
python3 .github/workflows/scripts/run_suite.py \
--suite e2e-4card-light \
--auto-partition-id "${{ matrix.part }}" \
--auto-partition-size 1 \
--auto-upgrade-estimated-times \
--continue-on-error \
2>&1 | tee /tmp/e2e-4card-light-part${{ matrix.part }}.log
else
python3 .github/workflows/scripts/run_suite.py \
--suite e2e-4card-light \
--auto-partition-id "${{ matrix.part }}" \
--auto-partition-size 1 \
2>&1 | tee /tmp/e2e-4card-light-part${{ matrix.part }}.log
fi
exit ${PIPESTATUS[0]}
- name: Summarize multicard-4-light failure
if: ${{ always() }}
run: |
python3 .github/workflows/scripts/ci_log_summary.py \
--step-name "Run multicard-4-light test" \
--log-file /tmp/e2e-4card-light-part${{ matrix.part }}.log \
--output "$GITHUB_STEP_SUMMARY"
- name: Upload timing data
uses: actions/upload-artifact@v7
if: ${{ inputs.continue_on_error == true && github.event_name != 'pull_request' }}
with:
name: timing-data-4card-light-part${{ matrix.part }}
path: test_timing_data.json
if-no-files-found: warn
retention-days: 5
e2e-4-cards-full:
name: multicard-4-full
if: ${{ inputs.type == 'full' || inputs.multicard_4_tests != '' }}
runs-on: linux-aarch64-a3-4
strategy:
fail-fast: false
matrix:
part: [0]
container:
image: ${{ inputs.image_a3 }}
env:
VLLM_LOGGING_LEVEL: ERROR
VLLM_USE_MODELSCOPE: True
HF_HUB_OFFLINE: 1
steps:
- name: Checkout vllm-project/vllm-ascend repo
uses: actions/checkout@v6
with:
ref: ${{ inputs.ref }}
- name: Check npu and CANN info
run: |
npu-smi info
cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
- name: Config mirrors
run: |
sed -Ei 's@(ports|archive).ubuntu.com@cache-service.nginx-pypi-cache.svc.cluster.local:8081@g' /etc/apt/sources.list
pip config set global.index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple
pip config set global.trusted-host cache-service.nginx-pypi-cache.svc.cluster.local
apt-get update -y
apt install git -y
git config --global --add safe.directory /__w/vllm-ascend/vllm-ascend
- name: Install system dependencies
run: |
apt-get -y install `cat packages.txt`
apt-get -y install gcc g++ cmake libnuma-dev clang-15
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 20
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-15 20
pip install uv
- name: Checkout vllm-project/vllm repo
uses: actions/checkout@v6
with:
repository: vllm-project/vllm
ref: ${{ inputs.vllm }}
path: ./vllm-empty
fetch-depth: 1
- name: Install vllm-project/vllm from source
working-directory: ./vllm-empty
run: |
VLLM_TARGET_DEVICE=empty uv pip install -e .
pip uninstall -y triton
- name: Get csrc hash
id: get_csrc_hash
run: |
CSRC_HASH=$(find ./csrc ./setup.py ./CMakeLists.txt ./cmake \
-type f -not -path '*/.*' | sort | xargs sha256sum | sha256sum | awk '{print $1}')
echo "CSRC_HASH=$CSRC_HASH" >> $GITHUB_OUTPUT
- name: Cache vllm-ascend csrc
uses: runs-on/cache@v4
with:
path: |
vllm_ascend/_cann_ops_custom
vllm_ascend/*.so
vllm_ascend/lib
vllm_ascend/include
key: vllm-ascend-build-v1-${{ runner.os }}-${{ inputs.image_a3 }}-${{ steps.get_csrc_hash.outputs.CSRC_HASH }}
restore-keys: |
vllm-ascend-build-v1-${{ runner.os }}-${{ inputs.image_a3 }}-${{ steps.get_csrc_hash.outputs.CSRC_HASH }}
- name: Install vllm-project/vllm-ascend
env:
MAX_JOBS: "92"
run: |
pip install uc-manager
uv pip install -r requirements-dev.txt
uv pip install --force-reinstall --no-deps triton-ascend==3.2.1
if find vllm_ascend -maxdepth 1 -name '*.so' -type f 2>/dev/null | grep -q .; then
COMPILE_CUSTOM_KERNELS=0 uv pip install -e .
else
uv pip install -e .
fi
- name: Run vllm-project/vllm-ascend test for V1 Engine
if: ${{ inputs.type == 'full' }}
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
shell: bash
run: |
set -o pipefail
if [ "${{ inputs.continue_on_error }}" = "true" ]; then
python3 .github/workflows/scripts/run_suite.py \
--suite e2e-multicard-4-cards \
--auto-partition-id "${{ matrix.part }}" \
--auto-partition-size 1 \
--auto-upgrade-estimated-times \
--continue-on-error \
2>&1 | tee /tmp/e2e-4card-full-part${{ matrix.part }}.log
else
python3 .github/workflows/scripts/run_suite.py \
--suite e2e-multicard-4-cards \
--auto-partition-id "${{ matrix.part }}" \
--auto-partition-size 1 \
2>&1 | tee /tmp/e2e-4card-full-part${{ matrix.part }}.log
fi
exit ${PIPESTATUS[0]}
- name: Summarize multicard-4-full failure
if: ${{ always() && inputs.type == 'full' }}
run: |
python3 .github/workflows/scripts/ci_log_summary.py \
--step-name "Run vllm-project/vllm-ascend test for V1 Engine" \
--log-file /tmp/e2e-4card-full-part${{ matrix.part }}.log \
--output "$GITHUB_STEP_SUMMARY"
- name: Upload timing data
uses: actions/upload-artifact@v7
if: ${{ inputs.continue_on_error == true && github.event_name != 'pull_request' }}
with:
name: timing-data-4card-full-part${{ matrix.part }}
path: test_timing_data.json
if-no-files-found: warn
retention-days: 5
- name: Run E2E tests (comment-multicard-4)
if: ${{ inputs.multicard_4_tests != '' }}
env:
VLLM_WORKER_MULTIPROC_METHOD: spawn
run: |
pytest -sv --durations=0 ${{ inputs.multicard_4_tests }}
e2e_310p:
name: 310p singlecard
runs-on: linux-aarch64-310p-1
if: ${{ inputs.contains_310 || inputs.p310_tests != '' }}
container:
image: ${{ inputs.image_310p }}
env:
VLLM_LOGGING_LEVEL: ERROR
VLLM_USE_MODELSCOPE: True
HF_HUB_OFFLINE: 1
steps:
- name: Check npu and CANN info
run: |
npu-smi info
cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
- name: Config mirrors
run: |
sed -Ei 's@(ports|archive).ubuntu.com@cache-service.nginx-pypi-cache.svc.cluster.local:8081@g' /etc/apt/sources.list
pip config set global.index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple
pip config set global.trusted-host cache-service.nginx-pypi-cache.svc.cluster.local
apt-get update -y
apt install git -y
git config --global --add safe.directory /__w/vllm-ascend/vllm-ascend
- name: Checkout vllm-project/vllm-ascend repo
uses: actions/checkout@v6
with:
ref: ${{ inputs.ref }}
- name: Install system dependencies
run: |
apt-get -y install `cat packages.txt`
apt-get -y install gcc g++ cmake libnuma-dev
pip install uv
- name: Checkout vllm-project/vllm repo
uses: actions/checkout@v6
with:
repository: vllm-project/vllm
ref: ${{ inputs.vllm }}
path: ./vllm-empty
fetch-depth: 1
- name: Install vllm-project/vllm from source
working-directory: ./vllm-empty
run: |
VLLM_TARGET_DEVICE=empty uv pip install -e .
- name: Get csrc hash
id: get_csrc_hash
run: |
CSRC_HASH=$(find ./csrc ./setup.py ./CMakeLists.txt ./cmake \
-type f -not -path '*/.*' | sort | xargs sha256sum | sha256sum | awk '{print $1}')
echo "CSRC_HASH=$CSRC_HASH" >> $GITHUB_OUTPUT
- name: Cache vllm-ascend csrc
uses: runs-on/cache@v4
with:
path: |
vllm_ascend/_cann_ops_custom
vllm_ascend/*.so
vllm_ascend/lib
vllm_ascend/include
key: vllm-ascend-build-v1-${{ runner.os }}-${{ inputs.image_310p }}-${{ steps.get_csrc_hash.outputs.CSRC_HASH }}
restore-keys: |
vllm-ascend-build-v1-${{ runner.os }}-${{ inputs.image_310p }}-${{ steps.get_csrc_hash.outputs.CSRC_HASH }}
- name: Install vllm-project/vllm-ascend
env:
MAX_JOBS: "23"
run: |
pip install uc-manager
uv pip install -r requirements-dev.txt
uv pip install --force-reinstall --no-deps triton-ascend==3.2.1
if find vllm_ascend -maxdepth 1 -name '*.so' -type f 2>/dev/null | grep -q .; then
COMPILE_CUSTOM_KERNELS=0 uv pip install -e .
else
uv pip install -e .
fi
pip uninstall -y triton-ascend triton
- name: Uninstall triton-ascend (310p e2e-light)
if: ${{ inputs.type == 'light' }}
run: pip uninstall -y triton-ascend triton
- name: Run vllm-project/vllm-ascend test
if: ${{ inputs.contains_310 }}
env:
PYTORCH_NPU_ALLOC_CONF: max_split_size_mb:256
VLLM_WORKER_MULTIPROC_METHOD: spawn
shell: bash
run: |
set -o pipefail
pytest -sv --durations=0 tests/e2e/310p/singlecard/test_dense_model_singlecard.py \
tests/e2e/310p/singlecard/test_vl_model_singlecard.py \
2>&1 | tee /tmp/e2e-310p-singlecard.log
exit ${PIPESTATUS[0]}
- name: Summarize 310p singlecard failure
if: ${{ always() && inputs.contains_310 }}
run: |
python3 .github/workflows/scripts/ci_log_summary.py \
--step-name "Run vllm-project/vllm-ascend test" \
--log-file /tmp/e2e-310p-singlecard.log \
--output "$GITHUB_STEP_SUMMARY"
- name: Run E2E tests (comment-310p)
if: ${{ inputs.p310_tests != '' }}
env:
PYTORCH_NPU_ALLOC_CONF: max_split_size_mb:256
VLLM_WORKER_MULTIPROC_METHOD: spawn
run: |
pytest -sv --durations=0 ${{ inputs.p310_tests }}
e2e_310p-4cards:
name: 310p multicards 4cards
runs-on: linux-aarch64-310p-4
if: ${{ inputs.contains_310 }}
container:
image: ${{ inputs.image_310p }}
env:
VLLM_LOGGING_LEVEL: ERROR
VLLM_USE_MODELSCOPE: True
HF_HUB_OFFLINE: 1
steps:
- name: Check npu and CANN info
run: |
npu-smi info
cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
- name: Config mirrors
run: |
sed -Ei 's@(ports|archive).ubuntu.com@cache-service.nginx-pypi-cache.svc.cluster.local:8081@g' /etc/apt/sources.list
pip config set global.index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple
pip config set global.trusted-host cache-service.nginx-pypi-cache.svc.cluster.local
apt-get update -y
apt install git -y
git config --global --add safe.directory /__w/vllm-ascend/vllm-ascend
- name: Checkout vllm-project/vllm-ascend repo
uses: actions/checkout@v6
- name: Install system dependencies
run: |
apt-get -y install `cat packages.txt`
apt-get -y install gcc g++ cmake libnuma-dev
pip install uv
- name: Checkout vllm-project/vllm repo
uses: actions/checkout@v6
with:
repository: vllm-project/vllm
ref: ${{ inputs.vllm }}
path: ./vllm-empty
fetch-depth: 1
- name: Install vllm-project/vllm from source
working-directory: ./vllm-empty
run: |
VLLM_TARGET_DEVICE=empty uv pip install -e .
- name: Get csrc hash
id: get_csrc_hash
run: |
CSRC_HASH=$(find ./csrc ./setup.py ./CMakeLists.txt ./cmake \
-type f -not -path '*/.*' | sort | xargs sha256sum | sha256sum | awk '{print $1}')
echo "CSRC_HASH=$CSRC_HASH" >> $GITHUB_OUTPUT
- name: Cache vllm-ascend csrc
uses: runs-on/cache@v4
with:
path: |
vllm_ascend/_cann_ops_custom
vllm_ascend/*.so
vllm_ascend/lib
vllm_ascend/include
key: vllm-ascend-build-v1-${{ runner.os }}-${{ inputs.image_310p }}-${{ steps.get_csrc_hash.outputs.CSRC_HASH }}
restore-keys: |
vllm-ascend-build-v1-${{ runner.os }}-${{ inputs.image_310p }}-${{ steps.get_csrc_hash.outputs.CSRC_HASH }}
- name: Install vllm-project/vllm-ascend
env:
MAX_JOBS: "92"
run: |
pip install uc-manager
uv pip install -r requirements-dev.txt
uv pip install --force-reinstall --no-deps triton-ascend==3.2.1
if find vllm_ascend -maxdepth 1 -name '*.so' -type f 2>/dev/null | grep -q .; then
COMPILE_CUSTOM_KERNELS=0 uv pip install -e .
else
uv pip install -e .
fi
pip uninstall -y triton-ascend triton
- name: Uninstall triton-ascend (310p e2e-light)
if: ${{ inputs.type == 'light' }}
run: pip uninstall -y triton-ascend triton
- name: Run vllm-project/vllm-ascend test
env:
PYTORCH_NPU_ALLOC_CONF: max_split_size_mb:256
VLLM_WORKER_MULTIPROC_METHOD: spawn
shell: bash
run: |
set -o pipefail
pytest -sv --durations=0 \
tests/e2e/310p/multicard/test_dense_model_multicard.py::test_qwen3_dense_tp4_w8a8 \
tests/e2e/310p/multicard/test_moe_model_multicard.py::test_qwen3_moe_tp4_fp16 \
tests/e2e/310p/multicard/test_moe_model_multicard.py::test_qwen3_5_moe_tp4_fp16 \
tests/e2e/310p/multicard/test_vl_model_multicard.py::test_qwen3_vl_8b_tp2_fp16 \
2>&1 | tee /tmp/e2e-310p-4cards.log
exit ${PIPESTATUS[0]}
- name: Summarize 310p multicards failure
if: ${{ always() && inputs.contains_310 }}
run: |
python3 .github/workflows/scripts/ci_log_summary.py \
--step-name "Run vllm-project/vllm-ascend test" \
--log-file /tmp/e2e-310p-4cards.log \
--output "$GITHUB_STEP_SUMMARY"