已合并
Optimize github action for ascend/pytorch and fix some tiny problems at a time. #7146
AtomGit-Bot创建于 2023年11月7日
Optimize github action for ascend/pytorch and fix some tiny problems at a time. #7146
已合并
从refs/pull/7146/head合入到master
共 5 个文件变更+104-38
| @@ -0,0 +1,47 @@ | |||
| 1 | +name: Fetch and Rebase | ||
| 2 | + | ||
| 3 | +description: Fetch and rebase for repository | ||
| 4 | + | ||
| 5 | +inputs: | ||
| 6 | + repo_path: | ||
| 7 | + description: the path the repository locate on | ||
| 8 | + required: true | ||
| 9 | + remote_branch: | ||
| 10 | + description: the branch rebased from | ||
| 11 | + required: true | ||
| 12 | + loop: | ||
| 13 | + description: loop times when fails | ||
| 14 | + required: true | ||
| 15 | + | ||
| 16 | +runs: | ||
| 17 | + using: composite | ||
| 18 | + steps: | ||
| 19 | + - name: Fetch and rebase | ||
| 20 | + env: | ||
| 21 | + REPO_PATH: ${{ inputs.repo_path }} | ||
| 22 | + REMOTE_BRANCH: ${{ inputs.remote_branch }} | ||
| 23 | + LOOP: ${{ inputs.loop }} | ||
| 24 | + shell: bash | ||
| 25 | + run: | | ||
| 26 | + COUNT=0 | ||
| 27 | + for i in $(seq 1 ${LOOP}) | ||
| 28 | + do | ||
| 29 | + pushd ${REPO_PATH} && | ||
| 30 | + git fetch --all && | ||
| 31 | + git rebase ${REMOTE_BRANCH} && | ||
| 32 | + git submodule sync && | ||
| 33 | + git submodule update --init --recursive && | ||
| 34 | + git reset --hard HEAD && | ||
| 35 | + git clean -dfx && | ||
| 36 | + git submodule foreach git reset --hard HEAD && | ||
| 37 | + git submodule foreach git clean -dfx && | ||
| 38 | + popd | ||
| 39 | + if [[ $? -ne 0 ]] | ||
| 40 | + then | ||
| 41 | + let COUNT++ | ||
| 42 | + else | ||
| 43 | + break | ||
| 44 | + fi | ||
| 45 | + done | ||
| 46 | + | ||
| 47 | + [[ ${COUNT} -lt ${LOOP} ]] && true || false | ||
| @@ -1,4 +1,4 @@ | |||
| 1 | -name: build-and-test | 1 | +name: Build and Test |
| 2 | 2 | ||
| 3 | on: | 3 | on: |
| 4 | workflow_call: | 4 | workflow_call: |
| @@ -17,37 +17,8 @@ on: | |||
| 17 | description: The docker iamge which will be loaded | 17 | description: The docker iamge which will be loaded |
| 18 | 18 | ||
| 19 | jobs: | 19 | jobs: |
| 20 | - fetch-and-rebase: | 20 | + build-and-test: |
| 21 | runs-on: ${{ inputs.runner }} | 21 | runs-on: ${{ inputs.runner }} |
| 22 | - steps: | ||
| 23 | - - name: Pull latest codes for torch | ||
| 24 | - run: | | ||
| 25 | - pushd /root/codes/pytorch/pytorch && | ||
| 26 | - git fetch --all && | ||
| 27 | - git rebase upstream/main && | ||
| 28 | - git submodule sync && | ||
| 29 | - git submodule update --init --recursive && | ||
| 30 | - git reset --hard HEAD && | ||
| 31 | - git clean -dfx && | ||
| 32 | - git submodule foreach git reset --hard HEAD && | ||
| 33 | - git submodule foreach git clean -dfx && | ||
| 34 | - popd | ||
| 35 | - - name: Pull latest codes for torch_npu | ||
| 36 | - run: | | ||
| 37 | - pushd /root/codes/npu/pytorch && | ||
| 38 | - git fetch --all && | ||
| 39 | - git rebase upstream/master && | ||
| 40 | - git submodule sync && | ||
| 41 | - git submodule update --init --recursive && | ||
| 42 | - git reset --hard HEAD && | ||
| 43 | - git clean -dfx && | ||
| 44 | - git submodule foreach git reset --hard HEAD && | ||
| 45 | - git submodule foreach git clean -dfx && | ||
| 46 | - popd | ||
| 47 | - | ||
| 48 | - compile-and-test: | ||
| 49 | - runs-on: ${{ inputs.runner }} | ||
| 50 | - needs: fetch-and-rebase | ||
| 51 | defaults: | 22 | defaults: |
| 52 | run: | 23 | run: |
| 53 | shell: bash | 24 | shell: bash |
| @@ -64,10 +35,13 @@ jobs: | |||
| 64 | options: --network host ${{ inputs.devices }} --device /dev/davinci_manager --device /dev/devmm_svm --device /dev/hisi_hdc | 35 | options: --network host ${{ inputs.devices }} --device /dev/davinci_manager --device /dev/devmm_svm --device /dev/hisi_hdc |
| 65 | steps: | 36 | steps: |
| 66 | - name: Set environment | 37 | - name: Set environment |
| 67 | - run: | 38 | + run: | |
| 68 | source /root/.bashrc && conda activate torch_npu | 39 | source /root/.bashrc && conda activate torch_npu |
| 40 | + - name: Prepare the codes | ||
| 41 | + run: | | ||
| 42 | + \cp -rf /root/codes/ /root/build | ||
| 69 | - name: Compile torch | 43 | - name: Compile torch |
| 70 | - working-directory: /root/codes/pytorch/pytorch | 44 | + working-directory: /root/build/pytorch/pytorch |
| 71 | run: | | 45 | run: | |
| 72 | pip3 install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple | 46 | pip3 install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple |
| 73 | export _GLIBCXX_USE_CXX11_ABI=0 | 47 | export _GLIBCXX_USE_CXX11_ABI=0 |
| @@ -75,12 +49,13 @@ jobs: | |||
| 75 | export USE_XNNPACK=0 | 49 | export USE_XNNPACK=0 |
| 76 | python setup.py develop | 50 | python setup.py develop |
| 77 | - name: Compile and install torch_npu | 51 | - name: Compile and install torch_npu |
| 78 | - working-directory: /root/codes/npu/pytorch | 52 | + working-directory: /root/build/npu/pytorch |
| 79 | run: | | 53 | run: | |
| 80 | pip3 install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple | 54 | pip3 install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple |
| 81 | bash ci/build.sh --python=3.8 | 55 | bash ci/build.sh --python=3.8 |
| 82 | pip3 install dist/torch_npu*.whl | 56 | pip3 install dist/torch_npu*.whl |
| 83 | - name: Do the test | 57 | - name: Do the test |
| 84 | - working-directory: /root/codes | 58 | + working-directory: /root/build |
| 85 | run: | | 59 | run: | |
| 60 | + pip3 install -r npu/pytorch/test/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --no-deps | ||
| 86 | python npu/pytorch/ci/access_control_test.py | 61 | python npu/pytorch/ci/access_control_test.py |
| @@ -0,0 +1,30 @@ | |||
| 1 | +name: Fetch and Rebase | ||
| 2 | + | ||
| 3 | +on: | ||
| 4 | + workflow_call: | ||
| 5 | + inputs: | ||
| 6 | + runner: | ||
| 7 | + required: true | ||
| 8 | + type: string | ||
| 9 | + description: The runner selected to run on | ||
| 10 | + loop: | ||
| 11 | + required: true | ||
| 12 | + type: string | ||
| 13 | + description: Loop time when fails | ||
| 14 | + | ||
| 15 | +jobs: | ||
| 16 | + fetch-and-rebase: | ||
| 17 | + runs-on: ${{ inputs.runner }} | ||
| 18 | + steps: | ||
| 19 | + - name: Pull latest codes for torch | ||
| 20 | + uses: ascend/pytorch/.github/actions/fetch-and-rebase@master | ||
| 21 | + with: | ||
| 22 | + repo_path: /root/codes/pytorch/pytorch | ||
| 23 | + remote_branch: upstream/main | ||
| 24 | + loop: ${{ inputs.loop }} | ||
| 25 | + - name: Pull latest codes for torch_npu | ||
| 26 | + uses: ascend/pytorch/.github/actions/fetch-and-rebase@master | ||
| 27 | + with: | ||
| 28 | + repo_path: /root/codes/npu/pytorch | ||
| 29 | + remote_branch: upstream/master | ||
| 30 | + loop: ${{ inputs.loop }} | ||
| @@ -1,15 +1,25 @@ | |||
| 1 | +# Note: | ||
| 2 | +# Same runner only needs one job named like .*fetch-and-rebase | ||
| 3 | + | ||
| 1 | name: Ascend NPU | 4 | name: Ascend NPU |
| 2 | 5 | ||
| 3 | on: | 6 | on: |
| 4 | schedule: | 7 | schedule: |
| 5 | - cron: '0 12 * * *' | 8 | - cron: '0 12 * * *' |
| 6 | - issue_comment: | 9 | + workflow_dispatch: |
| 7 | - types: [created] | ||
| 8 | 10 | ||
| 9 | jobs: | 11 | jobs: |
| 10 | - linux-py3_8-7_0_RC1_alpha005: | 12 | + linux-py3_8-fetch-and-rebase: |
| 13 | + name: linux-py3_8-7.0.RC1.alpha005 | ||
| 14 | + uses: ./.github/workflows/_fetch_and_rebase.yml | ||
| 15 | + with: | ||
| 16 | + runner: self-hosted | ||
| 17 | + loop: 10 | ||
| 18 | + | ||
| 19 | + linux-py3_8-build-and-test: | ||
| 11 | name: linux-py3_8-7.0.RC1.alpha005 | 20 | name: linux-py3_8-7.0.RC1.alpha005 |
| 12 | uses: ./.github/workflows/_build-and-test.yml | 21 | uses: ./.github/workflows/_build-and-test.yml |
| 22 | + needs: linux-py3_8-fetch-and-rebase | ||
| 13 | with: | 23 | with: |
| 14 | runner: self-hosted | 24 | runner: self-hosted |
| 15 | devices: --device /dev/davinci6 | 25 | devices: --device /dev/davinci6 |