309 lines
16 KiB
Python
Executable File
309 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Prepare pinned native macOS libraries or standalone helpers; no runtime package manager."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import platform
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import urllib.error
|
|
import urllib.request
|
|
import zipfile
|
|
|
|
REPOSITORY = Path(__file__).resolve().parent.parent
|
|
LOCK = REPOSITORY / "Dependencies.lock.json"
|
|
|
|
|
|
def sha256(path):
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as source:
|
|
for chunk in iter(lambda: source.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def download(asset, directory):
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
target = directory / asset["filename"]
|
|
if target.is_file() and sha256(target) == asset["sha256"]:
|
|
return target
|
|
temporary = None
|
|
try:
|
|
with tempfile.NamedTemporaryFile(dir=directory, delete=False) as output:
|
|
temporary = Path(output.name)
|
|
request = urllib.request.Request(asset["url"], headers={"User-Agent": "Terebi-dependency-preparer"})
|
|
with urllib.request.urlopen(request, timeout=120) as response:
|
|
shutil.copyfileobj(response, output)
|
|
actual = sha256(temporary)
|
|
if actual != asset["sha256"]:
|
|
raise RuntimeError(f"SHA256 mismatch for {asset['url']}: expected {asset['sha256']}, got {actual}; no unverified download installed")
|
|
temporary.chmod(0o644)
|
|
os.replace(temporary, target)
|
|
return target
|
|
finally:
|
|
if temporary is not None:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def archive_path(name, destination):
|
|
# Reject links altogether: the pinned source archives need none. This also
|
|
# prevents a preceding archive member from redirecting subsequent writes.
|
|
relative = PurePosixPath(name)
|
|
if relative.is_absolute() or ".." in relative.parts or "\\" in name:
|
|
raise RuntimeError(f"Unsafe archive member: {name!r}")
|
|
target = destination.joinpath(*relative.parts)
|
|
if not target.resolve().is_relative_to(destination.resolve()):
|
|
raise RuntimeError(f"Archive member escapes extraction directory: {name!r}")
|
|
return target
|
|
|
|
|
|
def extract(archive, destination):
|
|
destination.mkdir(parents=True, exist_ok=True)
|
|
if zipfile.is_zipfile(archive):
|
|
with zipfile.ZipFile(archive) as source:
|
|
for member in source.infolist():
|
|
target = archive_path(member.filename, destination)
|
|
mode = member.external_attr >> 16
|
|
if mode & 0o170000 not in (0, 0o100000, 0o040000):
|
|
raise RuntimeError(f"Unsupported archive member: {member.filename}")
|
|
if member.is_dir():
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
else:
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
with source.open(member) as data, target.open("wb") as output:
|
|
shutil.copyfileobj(data, output)
|
|
target.chmod(0o755 if mode & 0o111 else 0o644)
|
|
else:
|
|
with tarfile.open(archive) as source:
|
|
for member in source:
|
|
target = archive_path(member.name, destination)
|
|
if member.isdir():
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
elif member.isfile():
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
with source.extractfile(member) as data, target.open("wb") as output:
|
|
shutil.copyfileobj(data, output)
|
|
target.chmod(0o755 if member.mode & 0o111 else 0o644)
|
|
else:
|
|
raise RuntimeError(f"Unsupported archive member: {member.name}")
|
|
children = list(destination.iterdir())
|
|
return children[0] if len(children) == 1 and children[0].is_dir() else destination
|
|
|
|
|
|
def run(command, cwd=None, env=None):
|
|
print("+ " + shlex.join(map(str, command)), flush=True)
|
|
subprocess.run(list(map(str, command)), cwd=cwd, env=env, check=True)
|
|
|
|
|
|
def native(root, arch, lock):
|
|
for tool in ("xcrun", "make", "meson", "ninja", "pkg-config") + (("nasm",) if arch == "x86_64" else ()):
|
|
if shutil.which(tool) is None:
|
|
raise RuntimeError(f"Missing build prerequisite {tool}; install it before preparing native dependencies")
|
|
sdk = subprocess.check_output(["xcrun", "--sdk", "macosx", "--show-sdk-path"], text=True).strip()
|
|
clang = subprocess.check_output(["xcrun", "--find", "clang"], text=True).strip()
|
|
clangxx = subprocess.check_output(["xcrun", "--find", "clang++"], text=True).strip()
|
|
flags = shlex.join(["-arch", arch, "-isysroot", sdk, "-mmacosx-version-min=15.0"])
|
|
environment = dict(os.environ)
|
|
# Host packages must never leak into dependency discovery or compiler flags.
|
|
for name in ("CPATH", "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "LIBRARY_PATH", "SDKROOT", "ARCHFLAGS", "CPPFLAGS", "PKG_CONFIG_PATH", "PKG_CONFIG_SYSROOT_DIR", "DYLD_LIBRARY_PATH"):
|
|
environment.pop(name, None)
|
|
environment.update(CC=clang, CXX=clangxx, CFLAGS=flags, CXXFLAGS=flags,
|
|
OBJCFLAGS=flags, LDFLAGS=flags, MACOSX_DEPLOYMENT_TARGET="15.0",
|
|
PKG_CONFIG_LIBDIR=os.pathsep.join(str(root / name / "lib/pkgconfig") for name in ("dav1d", "xz")), PKG_CONFIG_PATH="")
|
|
work = root / "work/native"
|
|
shutil.rmtree(work, ignore_errors=True)
|
|
work.mkdir(parents=True)
|
|
for component in ("dav1d", "xz", "ffmpeg"):
|
|
shutil.rmtree(root / component, ignore_errors=True)
|
|
sources = root / "sources/native"
|
|
sources.mkdir(parents=True, exist_ok=True)
|
|
licenses = root / "licenses/native"
|
|
licenses.mkdir(parents=True, exist_ok=True)
|
|
trees = {}
|
|
for component in ("dav1d", "xz", "ffmpeg"):
|
|
archive = download(lock[component]["source"], root / "downloads")
|
|
shutil.copy2(archive, sources / archive.name)
|
|
trees[component] = extract(archive, work / component)
|
|
dav1d_command = ["meson", "setup", work / "dav1d-build", trees["dav1d"],
|
|
"--prefix=" + str(root / "dav1d"), "--libdir=lib", "--buildtype=release",
|
|
"--default-library=shared", "--wrap-mode=nodownload", "-Denable_tools=false",
|
|
"-Denable_tests=false", "-Denable_examples=false", "-Denable_docs=false"]
|
|
xz_command = [trees["xz"] / "configure", "--prefix=" + str(root / "xz"),
|
|
"--enable-shared", "--disable-static", "--disable-xz", "--disable-xzdec",
|
|
"--disable-lzmadec", "--disable-lzmainfo", "--disable-scripts", "--disable-doc", "--disable-nls"]
|
|
ffmpeg_command = [trees["ffmpeg"] / "configure", "--prefix=" + str(root / "ffmpeg"),
|
|
"--arch=" + arch, "--target-os=darwin", "--cc=" + clang, "--cxx=" + clangxx,
|
|
"--host-cc=" + clang, "--host-cflags=" + flags, "--host-ldflags=" + flags,
|
|
"--extra-cflags=" + flags + " " + shlex.quote("-I" + str(root / "xz/include")),
|
|
"--extra-ldflags=" + flags + " " + shlex.quote("-L" + str(root / "xz/lib")),
|
|
"--enable-shared", "--disable-static", "--disable-programs", "--disable-doc",
|
|
"--disable-encoders", "--disable-muxers", "--disable-avdevice", "--disable-avfilter",
|
|
"--enable-encoder=aac,h264_videotoolbox", "--enable-muxer=mpegts",
|
|
"--disable-gpl", "--disable-nonfree", "--disable-version3", "--disable-autodetect",
|
|
"--enable-libdav1d", "--enable-securetransport", "--enable-videotoolbox",
|
|
"--enable-audiotoolbox", "--enable-zlib", "--enable-bzlib", "--enable-lzma"]
|
|
configuration = {"arch": arch, "deployment_target": "15.0", "sdk": sdk,
|
|
"environment": {key: environment[key] for key in ("CC", "CXX", "CFLAGS", "CXXFLAGS", "LDFLAGS", "MACOSX_DEPLOYMENT_TARGET", "PKG_CONFIG_LIBDIR", "PKG_CONFIG_PATH")},
|
|
"dav1d": list(map(str, dav1d_command)), "xz": list(map(str, xz_command)),
|
|
"ffmpeg": list(map(str, ffmpeg_command))}
|
|
(licenses / "build-configuration.json").write_text(json.dumps(configuration, indent=2) + "\n")
|
|
run(dav1d_command, env=environment)
|
|
run(["ninja", "-C", work / "dav1d-build"], env=environment)
|
|
run(["ninja", "-C", work / "dav1d-build", "install"], env=environment)
|
|
xz_build = work / "xz-build"
|
|
xz_build.mkdir()
|
|
run(xz_command, cwd=xz_build, env=environment)
|
|
run(["make", "-j", str(os.cpu_count() or 1)], cwd=xz_build, env=environment)
|
|
run(["make", "install"], cwd=xz_build, env=environment)
|
|
build = work / "ffmpeg-build"
|
|
build.mkdir()
|
|
run(ffmpeg_command, cwd=build, env=environment)
|
|
run(["make", "-j", str(os.cpu_count() or 1)], cwd=build, env=environment)
|
|
run(["make", "install"], cwd=build, env=environment)
|
|
for filename in ("COPYING.LGPLv2.1", "LICENSE.md"):
|
|
shutil.copy2(trees["ffmpeg"] / filename, licenses / ("FFmpeg-" + filename))
|
|
shutil.copy2(trees["dav1d"] / "COPYING", licenses / "dav1d-COPYING")
|
|
shutil.copy2(trees["xz"] / "COPYING", licenses / "liblzma-COPYING")
|
|
for filename in ("config.h", "ffbuild/config.mak"):
|
|
shutil.copy2(build / filename, licenses / ("FFmpeg-" + Path(filename).name))
|
|
(licenses / "NOTICE.txt").write_text(
|
|
"FFmpeg 9.0.1: LGPL-2.1-or-later; dav1d 1.5.4: BSD-2-Clause; liblzma 5.8.4: 0BSD/public domain.\n"
|
|
"Exact unmodified source archives accompany this distribution under sources/native.\n"
|
|
"Build commands and flags are in build-configuration.json; source changes: none.\n"
|
|
"This software is based in part on the work of the Independent JPEG Group.\n"
|
|
"FFmpeg JPEG source files were not modified. Apple system libraries are supplied by macOS.\n"
|
|
"You may replace these shared libraries with compatible modified versions; no static linking is used.\n")
|
|
|
|
|
|
def helpers(root, arch, lock):
|
|
binaries = root / "bin"
|
|
binaries.mkdir(parents=True, exist_ok=True)
|
|
sources = root / "sources/helpers"
|
|
sources.mkdir(parents=True, exist_ok=True)
|
|
licenses = root / "licenses/helpers"
|
|
licenses.mkdir(parents=True, exist_ok=True)
|
|
for name in ("yt-dlp", "deno"):
|
|
component = lock[name]
|
|
asset = component["binary"] if name == "yt-dlp" else component["binaries"][arch]
|
|
archive = download(asset, root / "downloads")
|
|
with tempfile.TemporaryDirectory(dir=root) as temporary:
|
|
stage = Path(temporary)
|
|
executable = archive if name == "yt-dlp" else extract(archive, stage) / "deno"
|
|
if not executable.is_file():
|
|
raise RuntimeError(f"Pinned {name} archive does not contain its expected executable")
|
|
target = stage / "installed-executable"
|
|
shutil.copyfile(executable, target)
|
|
target.chmod(0o755)
|
|
os.replace(target, binaries / name)
|
|
for material in component["materials"]:
|
|
path = download(material, root / "downloads")
|
|
shutil.copy2(path, (sources if material.get("kind") == "source" else licenses) / path.name)
|
|
(licenses / (name + "-provenance.json")).write_text(json.dumps(component, indent=2) + "\n")
|
|
(licenses / (name + "-NOTICE.txt")).write_text(component["notice"] + "\n")
|
|
|
|
|
|
def outputs_exist(root, part, lock):
|
|
if part == "helpers":
|
|
return (all((root / "bin" / name).is_file() and os.access(root / "bin" / name, os.X_OK)
|
|
and all(((root / "sources/helpers" if material.get("kind") == "source" else root / "licenses/helpers") / material["filename"]).is_file()
|
|
for material in lock[name]["materials"])
|
|
and (root / "licenses/helpers" / (name + "-NOTICE.txt")).is_file()
|
|
and (root / "licenses/helpers" / (name + "-provenance.json")).is_file()
|
|
for name in ("yt-dlp", "deno")))
|
|
return ((root / "ffmpeg/include/libavcodec/avcodec.h").is_file()
|
|
and all((root / "ffmpeg/lib/pkgconfig" / (name + ".pc")).is_file()
|
|
and any((root / "ffmpeg/lib").glob(name + ".*.dylib"))
|
|
for name in ("libavcodec", "libavformat", "libavutil", "libswscale", "libswresample"))
|
|
and any((root / "dav1d/lib").glob("libdav1d.*.dylib"))
|
|
and any((root / "xz/lib").glob("liblzma.*.dylib"))
|
|
and (root / "licenses/native/build-configuration.json").is_file()
|
|
and all((root / "sources/native" / lock[name]["source"]["filename"]).is_file()
|
|
for name in ("ffmpeg", "dav1d", "xz")))
|
|
|
|
|
|
def self_check():
|
|
"""Offline archive-boundary check: python3 -c 'from Scripts.dependencies import self_check; self_check()'."""
|
|
build = REPOSITORY / ".build"
|
|
build.mkdir(exist_ok=True)
|
|
with tempfile.TemporaryDirectory(dir=build) as temporary:
|
|
root = Path(temporary)
|
|
for name in ("../escape", "/absolute", "safe/../../escape", "safe\\escape"):
|
|
try:
|
|
archive_path(name, root)
|
|
except RuntimeError:
|
|
pass
|
|
else:
|
|
raise AssertionError(f"Unsafe path accepted: {name}")
|
|
assert archive_path("safe/file", root) == root / "safe/file"
|
|
archive = root / "links.tar"
|
|
with tarfile.open(archive, "w") as output:
|
|
link = tarfile.TarInfo("link")
|
|
link.type = tarfile.SYMTYPE
|
|
link.linkname = "../outside"
|
|
output.addfile(link)
|
|
try:
|
|
extract(archive, root / "extracted")
|
|
except RuntimeError:
|
|
pass
|
|
else:
|
|
raise AssertionError("Archive symlink accepted")
|
|
archive = root / "traversal.zip"
|
|
with zipfile.ZipFile(archive, "w") as output:
|
|
output.writestr("../escape", "unsafe")
|
|
try:
|
|
extract(archive, root / "zip-extracted")
|
|
except RuntimeError:
|
|
pass
|
|
else:
|
|
raise AssertionError("ZIP traversal accepted")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--root", required=True, type=Path)
|
|
parser.add_argument("--arch", required=True, choices=("arm64", "x86_64"))
|
|
parser.add_argument("--part", required=True, choices=("native", "helpers"))
|
|
args = parser.parse_args()
|
|
if platform.system() != "Darwin" or platform.machine() != args.arch:
|
|
parser.error(f"Native macOS {args.arch} host required; running {platform.system()} {platform.machine()}. Cross-architecture builds are unsupported (do not run under Rosetta).")
|
|
translated = subprocess.run(["/usr/sbin/sysctl", "-in", "sysctl.proc_translated"],
|
|
capture_output=True, text=True, check=False)
|
|
if translated.stdout.strip() == "1":
|
|
parser.error("Rosetta builds are unsupported; use native arm64 Python and build tools.")
|
|
if not args.root.is_absolute():
|
|
parser.error("--root must be an absolute path beneath the repository's ignored .build directory")
|
|
root = args.root.resolve()
|
|
if not root.is_relative_to((REPOSITORY / ".build").resolve()):
|
|
parser.error("--root must be beneath the repository's ignored .build directory")
|
|
lock = json.loads(LOCK.read_text())
|
|
fingerprint = hashlib.sha256(LOCK.read_bytes() + Path(__file__).read_bytes() + args.arch.encode() + args.part.encode()).hexdigest()
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
stamp = root / (args.part + ".stamp")
|
|
if stamp.is_file() and stamp.read_text().strip() == fingerprint and outputs_exist(root, args.part, lock):
|
|
return 0
|
|
stamp.unlink(missing_ok=True)
|
|
try:
|
|
(native if args.part == "native" else helpers)(root, args.arch, lock)
|
|
if not outputs_exist(root, args.part, lock):
|
|
raise RuntimeError(f"{args.part} preparation completed without all required outputs")
|
|
shutil.copy2(LOCK, root / "licenses/Dependencies.lock.json")
|
|
shutil.copy2(Path(__file__), root / "licenses/dependencies.py")
|
|
with tempfile.NamedTemporaryFile(mode="w", dir=root, delete=False) as output:
|
|
output.write(fingerprint + "\n")
|
|
temporary = Path(output.name)
|
|
os.replace(temporary, stamp)
|
|
except (OSError, ValueError, RuntimeError, subprocess.CalledProcessError, tarfile.TarError, zipfile.BadZipFile) as error:
|
|
print(f"Dependency preparation failed: {error}. Fix the prerequisite/network issue and rerun the same command; no readiness stamp was published.", file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|