"""
This script is used to build Debian sysroot images for building Chromium.
"""
import argparse
import collections
import hashlib
import lzma
import os
import re
import shutil
import subprocess
import tempfile
import time
import requests
import reversion_glibc
DISTRO = "debian"
RELEASES = {
"amd64": "bullseye",
"i386": "bullseye",
"armhf": "bullseye",
"arm64": "bullseye",
"mipsel": "bullseye",
"mips64el": "bullseye",
"ppc64el": "bullseye",
"riscv64": "trixie",
}
GCC_VERSIONS = {
"bullseye": 10,
"trixie": 12,
}
SYSROOT_RELEASE = 1
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
CHROME_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, "..", "..", ".."))
KEYRING_FILE = os.path.join(SCRIPT_DIR, "keyring.gpg")
ARCHIVE_TIMESTAMP = "20250129T203412Z"
ARCHIVE_URL = f"https://snapshot.debian.org/archive/debian/{ARCHIVE_TIMESTAMP}/"
APT_SOURCES_LIST = [
("bookworm", ["main"]),
("bookworm-updates", ["main"]),
("bullseye", ["main", "contrib", "non-free"]),
("bullseye-updates", ["main", "contrib", "non-free"]),
("bullseye-backports", ["main", "contrib", "non-free"]),
]
APT_SOURCES_LIST_RISCV = [("trixie", ["main", "contrib"])]
APT_SOURCES_LISTS = {
"amd64": APT_SOURCES_LIST,
"i386": APT_SOURCES_LIST,
"armhf": APT_SOURCES_LIST,
"arm64": APT_SOURCES_LIST,
"mipsel": APT_SOURCES_LIST,
"mips64el": APT_SOURCES_LIST,
"ppc64el": APT_SOURCES_LIST,
"riscv64": APT_SOURCES_LIST_RISCV,
}
TRIPLES = {
"amd64": "x86_64-linux-gnu",
"i386": "i386-linux-gnu",
"armhf": "arm-linux-gnueabihf",
"arm64": "aarch64-linux-gnu",
"mipsel": "mipsel-linux-gnu",
"mips64el": "mips64el-linux-gnuabi64",
"ppc64el": "powerpc64le-linux-gnu",
"riscv64": "riscv64-linux-gnu",
}
LIB_DIRS = {
"bullseye": "lib",
"trixie": "usr/lib",
}
REQUIRED_TOOLS = [
"dpkg-deb",
"file",
"gpgv",
"readelf",
"tar",
"xz",
]
PACKAGES_EXT = "xz"
RELEASE_FILE = "Release"
RELEASE_FILE_GPG = "Release.gpg"
DEBIAN_PACKAGES = [
"libasound2-dev",
"libavformat-dev",
"libbluetooth-dev",
"libc6-dev",
"libcap-dev",
"libcolord-dev",
"libcups2-dev",
"libcupsimage2-dev",
"libcurl4-gnutls-dev",
"libdbusmenu-glib-dev",
"libdeflate-dev",
"libelf-dev",
"libflac-dev",
"libgbm-dev",
"libgcrypt20-dev",
"libgnutls28-dev",
"libgtk-3-dev",
"libgtk-4-dev",
"libinput-dev",
"libjbig-dev",
"libjpeg-dev",
"libjsoncpp-dev",
"libkrb5-dev",
"liblcms2-dev",
"liblzma-dev",
"libminizip-dev",
"libmtdev-dev",
"libncurses-dev",
"libnss3-dev",
"libopus-dev",
"libpam0g-dev",
"libpci-dev",
"libpipewire-0.3-dev",
"libpulse-dev",
"libre2-dev",
"libsnappy-dev",
"libspeechd-dev",
"libssl-dev",
"libsystemd-dev",
"libtiff-dev",
"libutempter-dev",
"libva-dev",
"libvpx-dev",
"libwayland-egl-backend-dev",
"libwebp-dev",
"libx11-xcb-dev",
"libxcb-dri2-0-dev",
"libxcb-dri3-dev",
"libxcb-glx0-dev",
"libxcb-image0-dev",
"libxcb-present-dev",
"libxcb-render-util0-dev",
"libxcb-util-dev",
"libxshmfence-dev",
"libxslt1-dev",
"libxss-dev",
"libxt-dev",
"libxxf86vm-dev",
"mesa-common-dev",
"qt6-base-dev",
"qtbase5-dev",
"valgrind-if-available",
]
def banner(message: str) -> None:
print("#" * 70)
print(message)
print("#" * 70)
def sub_banner(message: str) -> None:
print("-" * 70)
print(message)
print("-" * 70)
def hash_file(hasher, file_name: str) -> str:
with open(file_name, "rb") as f:
while chunk := f.read(8192):
hasher.update(chunk)
return hasher.hexdigest()
def atomic_copyfile(source: str, destination: str) -> None:
dest_dir = os.path.dirname(destination)
with tempfile.NamedTemporaryFile(mode="wb", delete=False,
dir=dest_dir) as temp_file:
temp_filename = temp_file.name
shutil.copyfile(source, temp_filename)
os.rename(temp_filename, destination)
def download_or_copy_non_unique_filename(url: str, dest: str) -> None:
"""
Downloads a file from a given URL to a destination with a unique filename,
based on the SHA-256 hash of the URL.
"""
hash_digest = hashlib.sha256(url.encode()).hexdigest()
unique_dest = f"{dest}.{hash_digest}"
download_or_copy(url, unique_dest)
atomic_copyfile(unique_dest, dest)
def download_or_copy(source: str, destination: str) -> None:
"""
Downloads a file from the given URL or copies it from a local path to the
specified destination.
"""
if os.path.exists(destination):
print(f"{destination} already in place")
return
if source.startswith(("http://", "https://")):
download_file(source, destination)
else:
atomic_copyfile(source, destination)
def download_file(url: str, dest: str, retries=5) -> None:
"""
Downloads a file from a URL to a specified destination with retry logic,
directory creation, and atomic write.
"""
print(f"Downloading from {url} -> {dest}")
os.makedirs(os.path.dirname(dest), exist_ok=True)
for attempt in range(retries):
try:
with requests.get(url, stream=True) as response:
response.raise_for_status()
with tempfile.NamedTemporaryFile(
mode="wb", delete=False,
dir=os.path.dirname(dest)) as temp_file:
for chunk in response.iter_content(chunk_size=8192):
temp_file.write(chunk)
os.rename(temp_file.name, dest)
print(f"Downloaded {dest}")
break
except requests.RequestException as e:
print(f"Attempt {attempt} failed: {e}")
time.sleep(2**attempt)
else:
raise Exception(f"Failed to download file after {retries} attempts")
def get_build_dir(arch: str) -> str:
return os.path.join(CHROME_DIR, "out", "sysroot-build", RELEASES[arch])
def sanity_check(build_dir: str) -> None:
"""
Performs sanity checks to ensure the environment is correctly set up.
"""
banner("Sanity Checks")
os.makedirs(build_dir, exist_ok=True)
print(f"Using build directory: {build_dir}")
missing = [tool for tool in REQUIRED_TOOLS if not shutil.which(tool)]
if missing:
raise Exception(f"Required tools not found: {', '.join(missing)}")
def clear_install_dir(install_root: str) -> None:
if os.path.exists(install_root):
shutil.rmtree(install_root)
os.makedirs(install_root)
def create_tarball(install_root: str, arch: str, build_dir: str) -> None:
tarball_path = os.path.join(
build_dir, f"{DISTRO}_{RELEASES[arch]}_{arch}_sysroot.tar.xz")
banner("Creating tarball " + tarball_path)
command = [
"tar",
"--owner=0",
"--group=0",
"--numeric-owner",
"--sort=name",
"--no-xattrs",
"-I",
"xz -z9 -T0 --lzma2='dict=256MiB'",
"-cf",
tarball_path,
"-C",
install_root,
".",
]
subprocess.run(command, check=True)
def generate_package_list_dist_repo(arch: str, dist: str, repo_name: str,
build_dir: str) -> list[dict[str, str]]:
repo_basedir = f"{ARCHIVE_URL}/dists/{dist}"
package_list = f"{build_dir}/Packages.{dist}_{repo_name}_{arch}"
package_list = f"{package_list}.{PACKAGES_EXT}"
package_file_arch = f"{repo_name}/binary-{arch}/Packages.{PACKAGES_EXT}"
package_list_arch = f"{repo_basedir}/{package_file_arch}"
download_or_copy_non_unique_filename(package_list_arch, package_list)
verify_package_listing(package_file_arch, package_list, dist, build_dir)
with lzma.open(package_list, "rt") as src:
return [
dict(
line.split(": ", 1) for line in package_meta.splitlines()
if not line.startswith(" "))
for package_meta in src.read().split("\n\n") if package_meta
]
def generate_package_list(arch: str, build_dir: str) -> dict[str, str]:
BROKEN_DEPS = {
"libgcc1",
"qt6-base-abi",
"libc-dev",
}
package_meta = {}
sources = APT_SOURCES_LISTS[arch]
for dist, repos in sources:
for repo_name in repos:
for meta in generate_package_list_dist_repo(
arch, dist, repo_name, build_dir):
package_meta[meta["Package"]] = meta
if "Provides" not in meta:
continue
for provides in meta["Provides"].split(", "):
provides = provides.split()[0]
if provides in package_meta:
continue
package_meta[provides] = meta
def add_package_dependencies(package: str) -> None:
if package in BROKEN_DEPS:
return
meta = package_meta[package]
url = ARCHIVE_URL + meta["Filename"]
if url in package_dict:
return
package_dict[url] = meta["SHA256"]
if "Depends" in meta:
for dep in meta["Depends"].split(", "):
add_package_dependencies(dep.split()[0].split(":")[0])
missing = set(DEBIAN_PACKAGES)
missing.add(f"libstdc++-{GCC_VERSIONS[RELEASES[arch]]}-dev")
package_dict: dict[str, str] = {}
for package in package_meta:
if package in missing:
missing.remove(package)
add_package_dependencies(package)
if missing:
raise Exception(f"Missing packages: {', '.join(missing)}")
output_file = os.path.join(SCRIPT_DIR, "generated_package_lists",
f"{RELEASES[arch]}.{arch}")
with open(output_file, "w") as f:
f.write("\n".join(sorted(package_dict)) + "\n")
return package_dict
def hacks_and_patches(install_root: str, script_dir: str, arch: str) -> None:
banner("Misc Hacks & Patches")
debian_dir = os.path.join(install_root, "debian")
control_file = os.path.join(debian_dir, "control")
open(control_file, "a").close()
qtchooser_conf = os.path.join(install_root, "usr", "lib", TRIPLES[arch],
"qt-default/qtchooser/default.conf")
if os.path.exists(qtchooser_conf):
os.remove(qtchooser_conf)
atomic_copyfile(
os.path.join(script_dir, "libxcomposite1-symbols"),
os.path.join(install_root, "debian", "libxcomposite1", "DEBIAN",
"symbols"),
)
features_h = os.path.join(install_root, "usr", "include", "features.h")
replace_in_file(features_h, r"(#define\s+__GLIBC_MINOR__)", r"\1 26 //")
replace_in_file(features_h, r"(#\s?define\s+__GLIBC_USE_C23_STRTOL)",
r"\1 0 //")
if arch == "riscv64":
os.remove(
os.path.join(install_root, "usr", "include", "riscv64-linux-gnu",
"sys", "hwprobe.h"))
fcntl_h = os.path.join(install_root, "usr", "include", "fcntl.h")
replace_in_file(
fcntl_h,
r"#ifndef __USE_FILE_OFFSET64(\nextern int fcntl)",
r"#if 1\1",
)
cppconfig_h = os.path.join(
install_root,
"usr",
"include",
TRIPLES[arch],
"c++",
str(GCC_VERSIONS[RELEASES[arch]]),
"bits",
"c++config.h",
)
replace_in_file(cppconfig_h,
r"(#define\s+_GLIBCXX_USE_PTHREAD_COND_CLOCKWAIT)",
r"// \1")
stdlib_h = os.path.join(install_root, "usr", "include", "stdlib.h")
replace_in_file(stdlib_h, r"(#include <stddef.h>)",
r"\1\n#include <limits.h>")
pkgconfig_dir = os.path.join(install_root, "usr", "lib", "pkgconfig")
os.makedirs(pkgconfig_dir, exist_ok=True)
triple_pkgconfig_dir = os.path.join(install_root, "usr", "lib",
TRIPLES[arch], "pkgconfig")
if os.path.exists(triple_pkgconfig_dir):
for file in os.listdir(triple_pkgconfig_dir):
shutil.move(os.path.join(triple_pkgconfig_dir, file),
pkgconfig_dir)
for lib in ["libc.so.6", "libm.so.6", "libcrypt.so.1"]:
lib_path = os.path.join(install_root, "lib", TRIPLES[arch], lib)
reversion_glibc.reversion_glibc(lib_path, arch)
gtk4_pc = os.path.join(pkgconfig_dir, "gtk4.pc")
replace_in_file(gtk4_pc, r"pango [>=0-9. ]*", "pango")
replace_in_file(gtk4_pc, r"pangocairo [>=0-9. ]*", "pangocairo")
os.remove(os.path.join(install_root, "usr/bin/X11"))
def create_extra_symlinks(install_root: str, arch: str):
if RELEASES[arch] != "bullseye":
for lib in ["libdl.so.2", "librt.so.1", "libpthread.so.0"]:
os.symlink(
lib,
os.path.join(install_root, "lib", TRIPLES[arch],
lib.rpartition(".")[0]))
def replace_in_file(file_path: str, search_pattern: str,
replace_pattern: str) -> None:
with open(file_path, "r") as file:
content = file.read()
with open(file_path, "w") as file:
file.write(re.sub(search_pattern, replace_pattern, content))
def install_into_sysroot(build_dir: str, install_root: str,
packages: dict[str, str]) -> None:
"""
Installs libraries and headers into the sysroot environment.
"""
banner("Install Libs And Headers Into Jail")
debian_packages_dir = os.path.join(build_dir, "debian-packages")
os.makedirs(debian_packages_dir, exist_ok=True)
debian_dir = os.path.join(install_root, "debian")
os.makedirs(debian_dir, exist_ok=True)
for package, sha256sum in packages.items():
package_name = os.path.basename(package)
package_path = os.path.join(debian_packages_dir, package_name)
banner(f"Installing {package_name}")
download_or_copy(package, package_path)
if hash_file(hashlib.sha256(), package_path) != sha256sum:
raise ValueError(f"SHA256 mismatch for {package_path}")
sub_banner(f"Extracting to {install_root}")
subprocess.run(["dpkg-deb", "-x", package_path, install_root],
check=True)
base_package = get_base_package_name(package_path)
debian_package_dir = os.path.join(debian_dir, base_package, "DEBIAN")
os.makedirs(debian_package_dir, exist_ok=True)
with subprocess.Popen(
["dpkg-deb", "-e", package_path, debian_package_dir],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
) as proc:
_, err = proc.communicate()
if proc.returncode != 0:
message = "Failed to extract control from"
raise Exception(
f"{message} {package_path}: {err.decode('utf-8')}")
USR_SHARE_ALLOWLIST = {
"fontconfig",
"pkgconfig",
"wayland",
"wayland-protocols",
}
usr_share = os.path.join(install_root, "usr", "share")
for item in os.listdir(usr_share):
full_path = os.path.join(usr_share, item)
if os.path.isdir(full_path) and item not in USR_SHARE_ALLOWLIST:
shutil.rmtree(full_path)
def get_base_package_name(package_path: str) -> str:
"""
Retrieves the base package name from a Debian package.
"""
result = subprocess.run(["dpkg-deb", "--field", package_path, "Package"],
capture_output=True,
text=True)
if result.returncode != 0:
raise Exception(
f"Failed to get package name from {package_path}: {result.stderr}")
return result.stdout.strip()
def cleanup_jail_symlinks(install_root: str) -> None:
"""
Cleans up jail symbolic links by converting absolute symlinks
into relative ones.
"""
for root, dirs, files in os.walk(install_root):
for name in files + dirs:
full_path = os.path.join(root, name)
if os.path.islink(full_path):
target_path = os.readlink(full_path)
if target_path == "/dev/null":
os.remove(full_path)
continue
if os.path.isabs(target_path):
absolute_target = os.path.join(install_root,
target_path.strip("/"))
else:
absolute_target = os.path.join(os.path.dirname(full_path),
target_path)
if not os.path.exists(absolute_target):
os.remove(full_path)
continue
if os.path.isabs(target_path):
relative_path = os.path.relpath(
os.path.join(install_root, target_path.strip("/")),
os.path.dirname(full_path),
)
joined_path = os.path.join(os.path.dirname(full_path),
relative_path)
if not os.path.exists(joined_path):
raise Exception(
f"Link target doesn't exist: {joined_path}")
os.remove(full_path)
os.symlink(relative_path, full_path)
def removing_unnecessary_files(install_root, arch):
"""
Minimizes the sysroot by removing unnecessary files.
"""
gcc_triple = "i686-linux-gnu" if arch == "i386" else TRIPLES[arch]
gcc_version = GCC_VERSIONS[RELEASES[arch]]
ALLOWLIST = {
"usr/bin/cups-config",
f"usr/lib/gcc/{gcc_triple}/{gcc_version}/libgcc.a",
f"usr/lib/{TRIPLES[arch]}/libc_nonshared.a",
f"usr/lib/{TRIPLES[arch]}/libffi_pic.a",
}
for file in ALLOWLIST:
assert os.path.exists(os.path.join(install_root,
file)), f"{file} does not exist"
reverse_links = collections.defaultdict(list)
remove = []
for root, _, files in os.walk(install_root):
for filename in files:
filepath = os.path.join(root, filename)
if os.path.relpath(filepath, install_root) in ALLOWLIST:
continue
if os.path.islink(filepath):
target_path = os.readlink(filepath)
if not os.path.isabs(target_path):
target_path = os.path.join(root, target_path)
reverse_links[os.path.realpath(target_path)].append(filepath)
elif "so" in filepath.split(".")[-3:]:
continue
elif os.access(filepath, os.X_OK) or filepath.endswith(".a"):
remove.append(filepath)
for filepath in remove:
os.remove(filepath)
for link in reverse_links[filepath]:
os.remove(link)
def strip_sections(install_root: str, arch: str):
"""
Strips all sections from ELF files except for dynamic linking and
essential sections. Skips static libraries (.a) and object files (.o).
"""
PRESERVED_SECTIONS = {
".dynamic",
".dynstr",
".dynsym",
".gnu.version",
".gnu.version_d",
".gnu.version_r",
".hash",
".note.ABI-tag",
".note.gnu.build-id",
}
for root, _, files in os.walk(install_root):
for file in files:
file_path = os.path.join(root, file)
if (os.access(file, os.X_OK) or file.endswith((".a", ".o"))
or os.path.islink(file_path)):
continue
with open(file_path, "rb") as f:
magic = f.read(4)
if magic != b"\x7fELF":
continue
objdump_cmd = ["objdump", "-h", file_path]
result = subprocess.run(objdump_cmd,
check=True,
text=True,
capture_output=True)
section_lines = result.stdout.splitlines()
sections = set()
for line in section_lines:
parts = line.split()
if len(parts) > 1 and parts[0].isdigit():
sections.add(parts[1])
sections_to_remove = sections - PRESERVED_SECTIONS
if sections_to_remove:
objcopy_arch = "amd64" if arch == "i386" else arch
objcopy_bin = TRIPLES[objcopy_arch] + "-objcopy"
objcopy_cmd = ([objcopy_bin] + [
f"--remove-section={section}"
for section in sections_to_remove
] + [file_path])
subprocess.run(objcopy_cmd, check=True, stderr=subprocess.PIPE)
def record_metadata(install_root: str) -> dict[str, tuple[float, float]]:
"""
Recursively walk the install_root directory and record the metadata of all
files. Symlinks are not followed. Returns a dictionary mapping each path
(relative to install_root) to its original metadata.
"""
metadata = {}
for root, dirs, files in os.walk(install_root):
for name in dirs + files:
full_path = os.path.join(root, name)
rel_path = os.path.relpath(full_path, install_root)
st = os.lstat(full_path)
metadata[rel_path] = (st.st_atime, st.st_mtime)
return metadata
def restore_metadata(install_root: str,
old_meta: dict[str, tuple[float, float]]) -> None:
"""
1. Restore the metadata of any file that exists in old_meta.
2. For all other files, set their timestamp to ARCHIVE_TIMESTAMP.
3. For all directories (including install_root), set the timestamp
to ARCHIVE_TIMESTAMP.
"""
archive_time = time.mktime(
time.strptime(ARCHIVE_TIMESTAMP, "%Y%m%dT%H%M%SZ"))
for root, dirs, files in os.walk(install_root):
os.utime(root, (archive_time, archive_time))
for file_name in files:
file_path = os.path.join(root, file_name)
if os.path.lexists(file_path):
rel_path = os.path.relpath(file_path, install_root)
if rel_path in old_meta:
restore_time = old_meta[rel_path]
else:
restore_time = (archive_time, archive_time)
os.utime(file_path, restore_time, follow_symlinks=False)
def build_sysroot(arch: str) -> None:
build_dir = get_build_dir(arch)
install_root = os.path.join(build_dir, f"{RELEASES[arch]}_{arch}_staging")
clear_install_dir(install_root)
packages = generate_package_list(arch, build_dir)
install_into_sysroot(build_dir, install_root, packages)
old_metadata = record_metadata(install_root)
hacks_and_patches(install_root, SCRIPT_DIR, arch)
create_extra_symlinks(install_root, arch)
cleanup_jail_symlinks(install_root)
removing_unnecessary_files(install_root, arch)
strip_sections(install_root, arch)
restore_metadata(install_root, old_metadata)
create_tarball(install_root, arch, build_dir)
def upload_sysroot(arch: str) -> str:
build_dir = get_build_dir(arch)
tarball_path = os.path.join(
build_dir, f"{DISTRO}_{RELEASES[arch]}_{arch}_sysroot.tar.xz")
command = [
"upload_to_google_storage_first_class.py",
"--bucket",
"chrome-linux-sysroot",
tarball_path,
]
return subprocess.check_output(command).decode("utf-8")
def verify_package_listing(file_path: str, output_file: str, dist: str,
build_dir: str) -> None:
"""
Verifies the downloaded Packages.xz file against its checksum and GPG keys.
"""
repo_basedir = f"{ARCHIVE_URL}/dists/{dist}"
release_list = f"{repo_basedir}/{RELEASE_FILE}"
release_list_gpg = f"{repo_basedir}/{RELEASE_FILE_GPG}"
release_file = os.path.join(build_dir, f"{dist}-{RELEASE_FILE}")
release_file_gpg = os.path.join(build_dir, f"{dist}-{RELEASE_FILE_GPG}")
if not os.path.exists(KEYRING_FILE):
raise Exception(f"KEYRING_FILE not found: {KEYRING_FILE}")
download_or_copy_non_unique_filename(release_list, release_file)
download_or_copy_non_unique_filename(release_list_gpg, release_file_gpg)
subprocess.run(
["gpgv", "--keyring", KEYRING_FILE, release_file_gpg, release_file],
check=True)
sha256sum_pattern = re.compile(r"([a-f0-9]{64})\s+\d+\s+" +
re.escape(file_path) + r"$")
sha256sum_match = None
with open(release_file, "r") as f:
for line in f:
if match := sha256sum_pattern.search(line):
sha256sum_match = match.group(1)
break
if not sha256sum_match:
raise Exception(
f"Checksum for {file_path} not found in {release_file}")
if hash_file(hashlib.sha256(), output_file) != sha256sum_match:
raise Exception(f"Checksum mismatch for {output_file}")
def main():
parser = argparse.ArgumentParser(
description="Build and upload Debian sysroot images for Chromium.")
parser.add_argument("command", choices=["build", "upload"])
parser.add_argument("architecture", choices=list(TRIPLES))
args = parser.parse_args()
sanity_check(get_build_dir(args.architecture))
if args.command == "build":
build_sysroot(args.architecture)
elif args.command == "upload":
upload_sysroot(args.architecture)
if __name__ == "__main__":
main()