detectron2-dcu:基于 ROCM 生态的目标检测与分割项目

项目主要是基于国家超算郑州中心的DCU显卡进行detectron2的移植和适配

Branch1Tags0
FilesLast commitLast update
10 months ago
10 months ago
10 months ago
10 months ago
10 months ago
10 months ago
10 months ago
10 months ago
10 months ago
10 months ago
10 months ago
10 months ago
10 months ago
10 months ago
10 months ago
10 months ago
10 months ago
10 months ago
10 months ago

💡 Tips:github下载的人工智能源码,一般需要查看README查看软件安装的相关方法,查看https://detectron2.readthedocs.io/tutorials/install.html安装方法

  • 如果安装地址打不开,一般正常的包会有setup.py文件直接可以python setup.py install

安装说明

1、一般github下载的软件首先查看的是README文件

2、一般的人工智能的软件安装需要在查看setup.py文件,分析这个文件来查看怎么安装这个package,需要着重看的就是os.getenv函数出现的地方 这个需要看看安装的逻辑 然后设置相应的环境变量,需要查看的我在下面的代码中我标注出来了

#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.

import glob
import os
import shutil
from os import path
from setuptools import find_packages, setup
from typing import List
import torch
from torch.utils.cpp_extension import CUDA_HOME, CppExtension, CUDAExtension

torch_ver = [int(x) for x in torch.__version__.split(".")[:2]]
#显示pytorch最低版本
assert torch_ver >= [1, 8], "Requires PyTorch >= 1.8" 


def get_version():
    init_py_path = path.join(path.abspath(path.dirname(__file__)), "detectron2", "__init__.py")
    init_py = open(init_py_path, "r").readlines()
    version_line = [l.strip() for l in init_py if l.startswith("__version__")][0]
    version = version_line.split("=")[-1].strip().strip("'\"")

    # The following is used to build release packages.
    # Users should never use it.
    suffix = os.getenv("D2_VERSION_SUFFIX", "")
    version = version + suffix
    if os.getenv("BUILD_NIGHTLY", "0") == "1":
        from datetime import datetime

        date_str = datetime.today().strftime("%y%m%d")
        version = version + ".dev" + date_str

        new_init_py = [l for l in init_py if not l.startswith("__version__")]
        new_init_py.append('__version__ = "{}"\n'.format(version))
        with open(init_py_path, "w") as f:
            f.write("".join(new_init_py))
    return version


def get_extensions():
    this_dir = path.dirname(path.abspath(__file__))
    extensions_dir = path.join(this_dir, "detectron2", "layers", "csrc")

    main_source = path.join(extensions_dir, "vision.cpp")
    sources = glob.glob(path.join(extensions_dir, "**", "*.cpp"))

    from torch.utils.cpp_extension import ROCM_HOME
  	##尤其注意文件中的rocm和hip关键字,这种一般就是这个软件可以兼容rocm
    ##pytorch中ROCM_HOME这个关键字是获取系统中rocm的路劲就是需要如果您的系统支持并且
    ##已经安装	rocm那么就需要设置ROCM_HOME环境变量方法export ROCM_HOME=/opt/rocm-4.0.1
    is_rocm_pytorch = (
        True if ((torch.version.hip is not None) and (ROCM_HOME is not None)) else False
    )
    if is_rocm_pytorch:
        assert torch_ver >= [1, 8], "ROCM support requires PyTorch >= 1.8!"

    # common code between cuda and rocm platforms, for hipify version [1,0,0] and later.
    source_cuda = glob.glob(path.join(extensions_dir, "**", "*.cu")) + glob.glob(
        path.join(extensions_dir, "*.cu")
    )
    sources = [main_source] + sources

    extension = CppExtension

    extra_compile_args = {"cxx": []}
    define_macros = []

    if (torch.cuda.is_available() and ((CUDA_HOME is not None) or is_rocm_pytorch)) or os.getenv(
        "FORCE_CUDA", "0"
    ) == "1":
        extension = CUDAExtension
        sources += source_cuda

        if not is_rocm_pytorch:
            define_macros += [("WITH_CUDA", None)]
            extra_compile_args["nvcc"] = [
                "-O3",
                "-DCUDA_HAS_FP16=1",
                "-D__CUDA_NO_HALF_OPERATORS__",
                "-D__CUDA_NO_HALF_CONVERSIONS__",
                "-D__CUDA_NO_HALF2_OPERATORS__",
            ]
        else:
            define_macros += [("WITH_HIP", None)]
            extra_compile_args["nvcc"] = []

        if torch_ver < [1, 7]:
            # supported by https://github.com/pytorch/pytorch/pull/43931
            CC = os.environ.get("CC", None)
            if CC is not None:
                extra_compile_args["nvcc"].append("-ccbin={}".format(CC))

    include_dirs = [extensions_dir]

    ext_modules = [
        extension(
            "detectron2._C",
            sources,
            include_dirs=include_dirs,
            define_macros=define_macros,
            extra_compile_args=extra_compile_args,
        )
    ]

    return ext_modules


def get_model_zoo_configs() -> List[str]:
    """
    Return a list of configs to include in package for model zoo. Copy over these configs inside
    detectron2/model_zoo.
    """

    # Use absolute paths while symlinking.
    source_configs_dir = path.join(path.dirname(path.realpath(__file__)), "configs")
    destination = path.join(
        path.dirname(path.realpath(__file__)), "detectron2", "model_zoo", "configs"
    )
    # Symlink the config directory inside package to have a cleaner pip install.

    # Remove stale symlink/directory from a previous build.
    if path.exists(source_configs_dir):
        if path.islink(destination):
            os.unlink(destination)
        elif path.isdir(destination):
            shutil.rmtree(destination)

    if not path.exists(destination):
        try:
            os.symlink(source_configs_dir, destination)
        except OSError:
            # Fall back to copying if symlink fails: ex. on Windows.
            shutil.copytree(source_configs_dir, destination)

    config_paths = glob.glob("configs/**/*.yaml", recursive=True) + glob.glob(
        "configs/**/*.py", recursive=True
    )
    return config_paths


# For projects that are relative small and provide features that are very close
# to detectron2's core functionalities, we install them under detectron2.projects
PROJECTS = {
    "detectron2.projects.point_rend": "projects/PointRend/point_rend",
    "detectron2.projects.deeplab": "projects/DeepLab/deeplab",
    "detectron2.projects.panoptic_deeplab": "projects/Panoptic-DeepLab/panoptic_deeplab",
}

setup(
    name="detectron2",
    version=get_version(),
    author="FAIR",
    url="https://github.com/facebookresearch/detectron2",
    description="Detectron2 is FAIR's next-generation research "
    "platform for object detection and segmentation.",
    packages=find_packages(exclude=("configs", "tests*")) + list(PROJECTS.keys()),
    package_dir=PROJECTS,
    package_data={"detectron2.model_zoo": get_model_zoo_configs()},
    python_requires=">=3.7",
    install_requires=[
        # These dependencies are not pure-python.
        # In general, avoid adding more dependencies like them because they are not
        # guaranteed to be installable by `pip install` on all platforms.
        # To tell if a package is pure-python, go to https://pypi.org/project/{name}/#files
        "Pillow>=7.1",  # or use pillow-simd for better performance
        "matplotlib",  # TODO move it to optional after we add opencv visualization
        "pycocotools>=2.0.2",  # corresponds to https://github.com/ppwwyyxx/cocoapi
        # Do not add opencv here. Just like pytorch, user should install
        # opencv themselves, preferrably by OS's package manager, or by
        # choosing the proper pypi package name at https://github.com/skvark/opencv-python
        # The following are pure-python dependencies that should be easily installable
        "termcolor>=1.1",
        "yacs>=0.1.8",
        "tabulate",
        "cloudpickle",
        "tqdm>4.29.0",
        "tensorboard",
        # Lock version of fvcore/iopath because they may have breaking changes
        # NOTE: when updating fvcore/iopath version, make sure fvcore depends
        # on compatible version of iopath.
        "fvcore>=0.1.5,<0.1.6",  # required like this to make it pip installable
        "iopath>=0.1.7,<0.1.10",
        "future",  # used by caffe2
        "pydot",  # used to save caffe2 SVGs
        "dataclasses; python_version<'3.7'",
        "omegaconf>=2.1",
        "hydra-core>=1.1",
        "black==22.3.0",
        "timm",
        "fairscale",
        "packaging",
        # If a new dependency is required at import time (in addition to runtime), it
        # probably needs to exist in docs/requirements.txt, or as a mock in docs/conf.py
    ],
    extras_require={
        # optional dependencies, required by some features
        "all": [
            "scipy>1.5.1",
            "shapely",
            "pygments>=2.2",
            "psutil",
            "panopticapi @ https://github.com/cocodataset/panopticapi/archive/master.zip",
        ],
        # dev dependencies. Install them by `pip install 'detectron2[dev]'`
        "dev": [
            "flake8==3.8.1",
            "isort==4.3.21",
            "flake8-bugbear",
            "flake8-comprehensions",
        ],
    },
    ext_modules=get_extensions(),
    cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension},
)

创建conda环境

💡 Tips:下面的操作的登陆节点上面操作

module purge
module load apps/anaconda3/5.2.0
module load compiler/devtoolset/7.3.1 
module load compiler/rocm/4.0.1
conda create --name detectron_env python=3.7
source activate detectron_env
pip install ninja psutil #加速编译速度

安装torch软件

💡 Tips:下载torch-1.9.0%2Brocm4.0.1-cp37-cp37m-linux_x86_64.whl,下载地址http://download.pytorch.org/whl/torch_stable.html

wget http://download.pytorch.org/whl/rocm4.0.1/torch-1.9.0%2Brocm4.0.1-cp37-cp37m-linux_x86_64.whl
pip install torch-1.8.0+rocm4.0.1-cp37-cp37m-linux_x86_64.whl

安装编译适合torch-rocm的torchvision软件

💡 Tips:有时候如果报一些莫名其妙的错误的话,需要登陆计算节点进行安装,由于计算节点不能上网,所以需要在登陆节点上面把所有的依赖安装完之后使用salloc -N 1 -p fat --gres=dcu:4这个命令分配计算节点登陆之后在进行安装

#下载适合torch匹配的版本如果超算集群中github访问不了,可以在本地下载完后上传到超算下载相应的vision编译
git clone https://github.com/pytorch/vision.git -b v0.10.0
module purge
module load apps/anaconda3/5.2.0
module load compiler/devtoolset/7.3.1 
module load compiler/rocm/4.0.1
source activate detectron_env
cd detectron2 #进度文件夹
export ROCM_HOME=/opt/rocm-4.0.1
cd vision
python setup.py install

安装detectron软件

💡 Tips:有时候如果报一些莫名其妙的错误的话,需要登陆计算节点进行安装,由于计算节点不能上网,所以需要在登陆节点上面把所有的依赖安装完之后使用salloc -N 1 -p fat --gres=dcu:4这个命令分配计算节点登陆之后在进行安装

module purge
module load apps/anaconda3/5.2.0
module load compiler/devtoolset/7.3.1 
module load compiler/rocm/4.0.1
source activate detectron_env
cd detectron2 #进度文件夹
export ROCM_HOME=/opt/rocm-4.0.1
python setup.py install

测试detectron是否安装成功

cd  detectron2/demo
python3 demo.py --config-file ../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml  --input inputs/val2014/*.jpg --output outputs/ --opts MODEL.WEIGHTS 'model_final_f10217.pkl'

Introduction

项目主要是基于国家超算郑州中心的DCU显卡进行detectron2的移植和适配

Customize your domain