项目主要是基于国家超算郑州中心的DCU显卡进行detectron2的移植和适配
| Files | Last commit | Last update |
|---|---|---|
| 10 months ago | ||
| 10 months ago | ||
| 10 months ago | ||
| 10 months ago | ||
| 10 months ago | ||
| 10 months ago | ||
| 10 months ago | ||
| 10 months ago | ||
| 10 months ago | ||
| 10 months ago | ||
| 10 months ago | ||
| 10 months ago | ||
| 10 months ago | ||
| 10 months ago | ||
| 10 months ago | ||
| 10 months ago | ||
| 10 months ago | ||
| 10 months ago | ||
| 10 months ago |
💡 Tips:github下载的人工智能源码,一般需要查看README查看软件安装的相关方法,查看https://detectron2.readthedocs.io/tutorials/install.html安装方法
- 如果安装地址打不开,一般正常的包会有setup.py文件直接可以python setup.py install
安装说明
1、一般github下载的软件首先查看的是README文件
2、一般的人工智能的软件安装需要在查看setup.py文件,分析这个文件来查看怎么安装这个package,需要着重看的就是os.getenv函数出现的地方 这个需要看看安装的逻辑 然后设置相应的环境变量,需要查看的我在下面的代码中我标注出来了
#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
import glob
import os
import shutil
from os import path
from setuptools import find_packages, setup
from typing import List
import torch
from torch.utils.cpp_extension import CUDA_HOME, CppExtension, CUDAExtension
torch_ver = [int(x) for x in torch.__version__.split(".")[:2]]
#显示pytorch最低版本
assert torch_ver >= [1, 8], "Requires PyTorch >= 1.8"
def get_version():
init_py_path = path.join(path.abspath(path.dirname(__file__)), "detectron2", "__init__.py")
init_py = open(init_py_path, "r").readlines()
version_line = [l.strip() for l in init_py if l.startswith("__version__")][0]
version = version_line.split("=")[-1].strip().strip("'\"")
# The following is used to build release packages.
# Users should never use it.
suffix = os.getenv("D2_VERSION_SUFFIX", "")
version = version + suffix
if os.getenv("BUILD_NIGHTLY", "0") == "1":
from datetime import datetime
date_str = datetime.today().strftime("%y%m%d")
version = version + ".dev" + date_str
new_init_py = [l for l in init_py if not l.startswith("__version__")]
new_init_py.append('__version__ = "{}"\n'.format(version))
with open(init_py_path, "w") as f:
f.write("".join(new_init_py))
return version
def get_extensions():
this_dir = path.dirname(path.abspath(__file__))
extensions_dir = path.join(this_dir, "detectron2", "layers", "csrc")
main_source = path.join(extensions_dir, "vision.cpp")
sources = glob.glob(path.join(extensions_dir, "**", "*.cpp"))
from torch.utils.cpp_extension import ROCM_HOME
##尤其注意文件中的rocm和hip关键字,这种一般就是这个软件可以兼容rocm
##pytorch中ROCM_HOME这个关键字是获取系统中rocm的路劲就是需要如果您的系统支持并且
##已经安装 rocm那么就需要设置ROCM_HOME环境变量方法export ROCM_HOME=/opt/rocm-4.0.1
is_rocm_pytorch = (
True if ((torch.version.hip is not None) and (ROCM_HOME is not None)) else False
)
if is_rocm_pytorch:
assert torch_ver >= [1, 8], "ROCM support requires PyTorch >= 1.8!"
# common code between cuda and rocm platforms, for hipify version [1,0,0] and later.
source_cuda = glob.glob(path.join(extensions_dir, "**", "*.cu")) + glob.glob(
path.join(extensions_dir, "*.cu")
)
sources = [main_source] + sources
extension = CppExtension
extra_compile_args = {"cxx": []}
define_macros = []
if (torch.cuda.is_available() and ((CUDA_HOME is not None) or is_rocm_pytorch)) or os.getenv(
"FORCE_CUDA", "0"
) == "1":
extension = CUDAExtension
sources += source_cuda
if not is_rocm_pytorch:
define_macros += [("WITH_CUDA", None)]
extra_compile_args["nvcc"] = [
"-O3",
"-DCUDA_HAS_FP16=1",
"-D__CUDA_NO_HALF_OPERATORS__",
"-D__CUDA_NO_HALF_CONVERSIONS__",
"-D__CUDA_NO_HALF2_OPERATORS__",
]
else:
define_macros += [("WITH_HIP", None)]
extra_compile_args["nvcc"] = []
if torch_ver < [1, 7]:
# supported by https://github.com/pytorch/pytorch/pull/43931
CC = os.environ.get("CC", None)
if CC is not None:
extra_compile_args["nvcc"].append("-ccbin={}".format(CC))
include_dirs = [extensions_dir]
ext_modules = [
extension(
"detectron2._C",
sources,
include_dirs=include_dirs,
define_macros=define_macros,
extra_compile_args=extra_compile_args,
)
]
return ext_modules
def get_model_zoo_configs() -> List[str]:
"""
Return a list of configs to include in package for model zoo. Copy over these configs inside
detectron2/model_zoo.
"""
# Use absolute paths while symlinking.
source_configs_dir = path.join(path.dirname(path.realpath(__file__)), "configs")
destination = path.join(
path.dirname(path.realpath(__file__)), "detectron2", "model_zoo", "configs"
)
# Symlink the config directory inside package to have a cleaner pip install.
# Remove stale symlink/directory from a previous build.
if path.exists(source_configs_dir):
if path.islink(destination):
os.unlink(destination)
elif path.isdir(destination):
shutil.rmtree(destination)
if not path.exists(destination):
try:
os.symlink(source_configs_dir, destination)
except OSError:
# Fall back to copying if symlink fails: ex. on Windows.
shutil.copytree(source_configs_dir, destination)
config_paths = glob.glob("configs/**/*.yaml", recursive=True) + glob.glob(
"configs/**/*.py", recursive=True
)
return config_paths
# For projects that are relative small and provide features that are very close
# to detectron2's core functionalities, we install them under detectron2.projects
PROJECTS = {
"detectron2.projects.point_rend": "projects/PointRend/point_rend",
"detectron2.projects.deeplab": "projects/DeepLab/deeplab",
"detectron2.projects.panoptic_deeplab": "projects/Panoptic-DeepLab/panoptic_deeplab",
}
setup(
name="detectron2",
version=get_version(),
author="FAIR",
url="https://github.com/facebookresearch/detectron2",
description="Detectron2 is FAIR's next-generation research "
"platform for object detection and segmentation.",
packages=find_packages(exclude=("configs", "tests*")) + list(PROJECTS.keys()),
package_dir=PROJECTS,
package_data={"detectron2.model_zoo": get_model_zoo_configs()},
python_requires=">=3.7",
install_requires=[
# These dependencies are not pure-python.
# In general, avoid adding more dependencies like them because they are not
# guaranteed to be installable by `pip install` on all platforms.
# To tell if a package is pure-python, go to https://pypi.org/project/{name}/#files
"Pillow>=7.1", # or use pillow-simd for better performance
"matplotlib", # TODO move it to optional after we add opencv visualization
"pycocotools>=2.0.2", # corresponds to https://github.com/ppwwyyxx/cocoapi
# Do not add opencv here. Just like pytorch, user should install
# opencv themselves, preferrably by OS's package manager, or by
# choosing the proper pypi package name at https://github.com/skvark/opencv-python
# The following are pure-python dependencies that should be easily installable
"termcolor>=1.1",
"yacs>=0.1.8",
"tabulate",
"cloudpickle",
"tqdm>4.29.0",
"tensorboard",
# Lock version of fvcore/iopath because they may have breaking changes
# NOTE: when updating fvcore/iopath version, make sure fvcore depends
# on compatible version of iopath.
"fvcore>=0.1.5,<0.1.6", # required like this to make it pip installable
"iopath>=0.1.7,<0.1.10",
"future", # used by caffe2
"pydot", # used to save caffe2 SVGs
"dataclasses; python_version<'3.7'",
"omegaconf>=2.1",
"hydra-core>=1.1",
"black==22.3.0",
"timm",
"fairscale",
"packaging",
# If a new dependency is required at import time (in addition to runtime), it
# probably needs to exist in docs/requirements.txt, or as a mock in docs/conf.py
],
extras_require={
# optional dependencies, required by some features
"all": [
"scipy>1.5.1",
"shapely",
"pygments>=2.2",
"psutil",
"panopticapi @ https://github.com/cocodataset/panopticapi/archive/master.zip",
],
# dev dependencies. Install them by `pip install 'detectron2[dev]'`
"dev": [
"flake8==3.8.1",
"isort==4.3.21",
"flake8-bugbear",
"flake8-comprehensions",
],
},
ext_modules=get_extensions(),
cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension},
)
创建conda环境
💡 Tips:下面的操作的登陆节点上面操作
module purge
module load apps/anaconda3/5.2.0
module load compiler/devtoolset/7.3.1
module load compiler/rocm/4.0.1
conda create --name detectron_env python=3.7
source activate detectron_env
pip install ninja psutil #加速编译速度
安装torch软件
💡 Tips:下载torch-1.9.0%2Brocm4.0.1-cp37-cp37m-linux_x86_64.whl,下载地址http://download.pytorch.org/whl/torch_stable.html
wget http://download.pytorch.org/whl/rocm4.0.1/torch-1.9.0%2Brocm4.0.1-cp37-cp37m-linux_x86_64.whl
pip install torch-1.8.0+rocm4.0.1-cp37-cp37m-linux_x86_64.whl
安装编译适合torch-rocm的torchvision软件
💡 Tips:有时候如果报一些莫名其妙的错误的话,需要登陆计算节点进行安装,由于计算节点不能上网,所以需要在登陆节点上面把所有的依赖安装完之后使用salloc -N 1 -p fat --gres=dcu:4这个命令分配计算节点登陆之后在进行安装
#下载适合torch匹配的版本如果超算集群中github访问不了,可以在本地下载完后上传到超算下载相应的vision编译
git clone https://github.com/pytorch/vision.git -b v0.10.0
module purge
module load apps/anaconda3/5.2.0
module load compiler/devtoolset/7.3.1
module load compiler/rocm/4.0.1
source activate detectron_env
cd detectron2 #进度文件夹
export ROCM_HOME=/opt/rocm-4.0.1
cd vision
python setup.py install
安装detectron软件
💡 Tips:有时候如果报一些莫名其妙的错误的话,需要登陆计算节点进行安装,由于计算节点不能上网,所以需要在登陆节点上面把所有的依赖安装完之后使用salloc -N 1 -p fat --gres=dcu:4这个命令分配计算节点登陆之后在进行安装
module purge
module load apps/anaconda3/5.2.0
module load compiler/devtoolset/7.3.1
module load compiler/rocm/4.0.1
source activate detectron_env
cd detectron2 #进度文件夹
export ROCM_HOME=/opt/rocm-4.0.1
python setup.py install
测试detectron是否安装成功
cd detectron2/demo
python3 demo.py --config-file ../configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml --input inputs/val2014/*.jpg --output outputs/ --opts MODEL.WEIGHTS 'model_final_f10217.pkl'