"""download os dependencies"""
import json
import os
from . import logger_config
from .deb_downloader import Apt
from .download_data import DownloadData
from .download_util import get_download_path, get_obs_downloader_path
from .rpm_downloader import Yum
from ascend_deployer.module_utils.safe_file_handler import SafeFileHandler
LOG = logger_config.LOG
class OsDepDownloader:
def __init__(self, download_data: DownloadData):
self.software_mgr = download_data.software_mgr
self.project_dir = get_download_path()
self.resources_dir = os.path.join(self.project_dir, 'resources')
def download(self, os_list, dst):
print("Start analyzing OS dependency packages...")
results = {}
for os_item in os_list:
res = self.download_os(os_item, dst)
results[os_item] = res
return results
def download_os(self, os_item, dst):
"""
download os packages. debs or rpms
:param os_itme: Ubuntu_18.04_aarch64, CentOS_8.2_x86_64..
"""
docker_pkg_list = [
"docker-ce-cli",
"containerd.io",
"docker-ce",
"docker-ce-rootless-extras",
"docker-scan-plugin",
"docker-engine",
"libseccomp2",
"libseccomp"
]
extra_downloader = None
if "CentOS_7.6" in os_item:
extra_package = ['container-selinux', 'libcgroup', 'fuse-overlayfs', 'slirp4netns', 'fuse3-libs']
docker_pkg_list.extend(extra_package)
if os_item.startswith(("CTyunOS_22.06", "CTyunOS_23.01", "OpenEuler_22.03")):
extra_package = ['fuse-overlayfs', 'slirp4netns', 'container-selinux', 'libcgroup', 'fuse3']
docker_pkg_list.extend(extra_package)
if "EulerOS" in os_item:
extra_package = ['libtool-ltdl', 'device-mapper', 'libxcrypt', 'audit-libs', 'libselinux']
docker_pkg_list.extend(extra_package)
if "MTOS" in os_item or "OpenEuler_22.03LTS-" in os_item:
extra_package = ["libcgroup", "container-selinux", "fuse-overlayfs", 'slirp4netns', 'libslirp']
docker_pkg_list.extend(extra_package)
if "OpenEuler_24.03LTS-SP1_aarch64" in os_item:
extra_package = ['container-selinux', 'libcgroup', 'fuse-overlayfs', 'slirp4netns', 'libsepol-devel',
'pcre2-devel', 'fuse3', 'libslirp', 'libtool-ltdl']
docker_pkg_list.extend(extra_package)
if "BCLinux" in os_item:
extra_package = ['container-selinux', 'libcgroup', 'docker-buildx-plugin', 'docker-compose-plugin']
docker_pkg_list.extend(extra_package)
if "Ubuntu_20.04_x86_64" in os_item:
docker_pkg_list.append("libc6")
dst_dir = os.path.join(dst, os_item)
if "Kylin" in os_item:
docker_pkg_list.extend([
"slirp4netns",
"fuse-overlayfs",
"libcgroup",
"container-selinux",
"fuse3-libs"])
if "Ubuntu_18.04_aarch64" in os_item:
docker_pkg_list.extend([
"dbus-user-session",
"dbus",
"libdbus-1-3",
"python3-lib2to3"])
if "Ubuntu_18.04_x86_64" in os_item:
docker_pkg_list.extend([
"libbinutils",
"binutils-common",
"dbus-user-session",
"dbus",
"libdbus-1-3",
"python3-lib2to3"])
if "veLinux" in os_item:
docker_pkg_list.extend(["dbus-user-session"])
if not os.path.exists(dst_dir):
os.makedirs(dst_dir, mode=0o750, exist_ok=True)
docker_dir = os.path.join(dst_dir, 'docker')
if not os.path.exists(docker_dir):
os.makedirs(docker_dir, mode=0o750, exist_ok=True)
LOG.info('item:{} save dir: {}'.format(os_item, os.path.basename(dst_dir)))
config_file = get_obs_downloader_path(
os.path.join(self.project_dir, 'downloader/config/{0}/pkg_info.json'.format(os_item)))
source_list_file = get_obs_downloader_path(
os.path.join(self.project_dir, 'downloader/config/{0}/source.list'.format(os_item)))
downloader = None
res = {'ok': [], 'failed': []}
content = SafeFileHandler.safe_read_json(config_file)
if isinstance(content, list):
package_count = len(content)
else:
package_count = 0
if not package_count:
return res
if os.path.exists(source_list_file):
if 'aarch64' in os_item:
downloader = Apt(source_list_file, 'aarch64')
else:
downloader = Apt(source_list_file, 'x86_64')
else:
source_repo_file = get_obs_downloader_path(
os.path.join(self.project_dir, 'downloader/config/{0}/source.repo'.format(os_item)))
if 'aarch64' in os_item:
downloader = Yum(source_repo_file, 'aarch64')
else:
downloader = Yum(source_repo_file, 'x86_64')
if downloader:
if not downloader.make_cache():
LOG.error('downloader make_cache failed')
res.get('failed', []).append(os_item)
raise RuntimeError('downloader make_cache failed')
print(f"Getting the list of software to be analyzed from {config_file}...")
data = SafeFileHandler.safe_read_json(config_file)
for item in data:
filename = item['name']
if filename in docker_pkg_list:
dst_dir = docker_dir
else:
dst_dir = os.path.join(dst, os_item)
if not downloader.download(item, dst_dir):
print(f'{filename} download failed')
LOG.error(f'{filename} download failed')
res.get('failed', []).append(filename)
raise RuntimeError(f'{filename} download failed')
res.get('ok', []).append(filename)
if "Ubuntu_22.04_aarch64" == os_item:
extra_source_list_file = get_obs_downloader_path(
os.path.join(self.project_dir, 'downloader/config/Ubuntu_20.04_aarch64/source.list'))
extra_downloader = Apt(extra_source_list_file, 'aarch64')
elif "Ubuntu_22.04_x86_64" == os_item:
extra_source_list_file = get_obs_downloader_path(
os.path.join(self.project_dir, 'downloader/config/Ubuntu_20.04_x86_64/source.list'))
extra_downloader = Apt(extra_source_list_file, 'x86_64')
if extra_downloader:
if not extra_downloader.make_cache():
LOG.error('extra_downloader make_cache failed')
res.get('failed', []).append(os_item)
raise RuntimeError('extra_downloader make_cache failed')
extra_config_file = get_obs_downloader_path(os.path.join(self.project_dir,
'downloader/config/{0}/extra_pkg_info.json'.format(
os_item)))
print(f"Getting the list of software to be analyzed from {extra_config_file}...")
data = SafeFileHandler.safe_read_json(extra_config_file)
for item in data:
filename = item['name']
dst_dir = os.path.join(dst, os_item)
if not extra_downloader.download(item, dst_dir):
print(f'{filename} download failed')
LOG.error(f'{filename} download failed')
res.get('failed', []).append(filename)
raise RuntimeError(f'{filename} download failed')
res.get('ok', []).append(filename)
if downloader:
downloader.clean_cache()
if extra_downloader:
extra_downloader.clean_cache()
return res