hl-valdemar
31c4759c1d
Build pinned LGPL playback libraries and package standalone YouTube helpers inside the signed native app. Keep the CLI inside the app and relocate private library links for installation without runtime Homebrew dependencies. Record adaptive fixture completion in its EXIT trap: cancellation can exit with status 143 before the TERM trap writes its marker, while native pause and quality preservation remain correct.
308 lines
16 KiB
Python
Executable File
308 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Prepare pinned native macOS libraries or standalone helpers; no runtime package manager."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import platform
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import urllib.error
|
|
import urllib.request
|
|
import zipfile
|
|
|
|
REPOSITORY = Path(__file__).resolve().parent.parent
|
|
LOCK = REPOSITORY / "Dependencies.lock.json"
|
|
|
|
|
|
def sha256(path):
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as source:
|
|
for chunk in iter(lambda: source.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def download(asset, directory):
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
target = directory / asset["filename"]
|
|
if target.is_file() and sha256(target) == asset["sha256"]:
|
|
return target
|
|
temporary = None
|
|
try:
|
|
with tempfile.NamedTemporaryFile(dir=directory, delete=False) as output:
|
|
temporary = Path(output.name)
|
|
request = urllib.request.Request(asset["url"], headers={"User-Agent": "Terebi-dependency-preparer"})
|
|
with urllib.request.urlopen(request, timeout=120) as response:
|
|
shutil.copyfileobj(response, output)
|
|
actual = sha256(temporary)
|
|
if actual != asset["sha256"]:
|
|
raise RuntimeError(f"SHA256 mismatch for {asset['url']}: expected {asset['sha256']}, got {actual}; no unverified download installed")
|
|
temporary.chmod(0o644)
|
|
os.replace(temporary, target)
|
|
return target
|
|
finally:
|
|
if temporary is not None:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def archive_path(name, destination):
|
|
# Reject links altogether: the pinned source archives need none. This also
|
|
# prevents a preceding archive member from redirecting subsequent writes.
|
|
relative = PurePosixPath(name)
|
|
if relative.is_absolute() or ".." in relative.parts or "\\" in name:
|
|
raise RuntimeError(f"Unsafe archive member: {name!r}")
|
|
target = destination.joinpath(*relative.parts)
|
|
if not target.resolve().is_relative_to(destination.resolve()):
|
|
raise RuntimeError(f"Archive member escapes extraction directory: {name!r}")
|
|
return target
|
|
|
|
|
|
def extract(archive, destination):
|
|
destination.mkdir(parents=True, exist_ok=True)
|
|
if zipfile.is_zipfile(archive):
|
|
with zipfile.ZipFile(archive) as source:
|
|
for member in source.infolist():
|
|
target = archive_path(member.filename, destination)
|
|
mode = member.external_attr >> 16
|
|
if mode & 0o170000 not in (0, 0o100000, 0o040000):
|
|
raise RuntimeError(f"Unsupported archive member: {member.filename}")
|
|
if member.is_dir():
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
else:
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
with source.open(member) as data, target.open("wb") as output:
|
|
shutil.copyfileobj(data, output)
|
|
target.chmod(0o755 if mode & 0o111 else 0o644)
|
|
else:
|
|
with tarfile.open(archive) as source:
|
|
for member in source:
|
|
target = archive_path(member.name, destination)
|
|
if member.isdir():
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
elif member.isfile():
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
with source.extractfile(member) as data, target.open("wb") as output:
|
|
shutil.copyfileobj(data, output)
|
|
target.chmod(0o755 if member.mode & 0o111 else 0o644)
|
|
else:
|
|
raise RuntimeError(f"Unsupported archive member: {member.name}")
|
|
children = list(destination.iterdir())
|
|
return children[0] if len(children) == 1 and children[0].is_dir() else destination
|
|
|
|
|
|
def run(command, cwd=None, env=None):
|
|
print("+ " + shlex.join(map(str, command)), flush=True)
|
|
subprocess.run(list(map(str, command)), cwd=cwd, env=env, check=True)
|
|
|
|
|
|
def native(root, arch, lock):
|
|
for tool in ("xcrun", "make", "meson", "ninja", "pkg-config") + (("nasm",) if arch == "x86_64" else ()):
|
|
if shutil.which(tool) is None:
|
|
raise RuntimeError(f"Missing build prerequisite {tool}; install it before preparing native dependencies")
|
|
sdk = subprocess.check_output(["xcrun", "--sdk", "macosx", "--show-sdk-path"], text=True).strip()
|
|
clang = subprocess.check_output(["xcrun", "--find", "clang"], text=True).strip()
|
|
clangxx = subprocess.check_output(["xcrun", "--find", "clang++"], text=True).strip()
|
|
flags = shlex.join(["-arch", arch, "-isysroot", sdk, "-mmacosx-version-min=15.0"])
|
|
environment = dict(os.environ)
|
|
# Host packages must never leak into dependency discovery or compiler flags.
|
|
for name in ("CPATH", "C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "LIBRARY_PATH", "SDKROOT", "ARCHFLAGS", "CPPFLAGS", "PKG_CONFIG_PATH", "PKG_CONFIG_SYSROOT_DIR", "DYLD_LIBRARY_PATH"):
|
|
environment.pop(name, None)
|
|
environment.update(CC=clang, CXX=clangxx, CFLAGS=flags, CXXFLAGS=flags,
|
|
OBJCFLAGS=flags, LDFLAGS=flags, MACOSX_DEPLOYMENT_TARGET="15.0",
|
|
PKG_CONFIG_LIBDIR=os.pathsep.join(str(root / name / "lib/pkgconfig") for name in ("dav1d", "xz")), PKG_CONFIG_PATH="")
|
|
work = root / "work/native"
|
|
shutil.rmtree(work, ignore_errors=True)
|
|
work.mkdir(parents=True)
|
|
for component in ("dav1d", "xz", "ffmpeg"):
|
|
shutil.rmtree(root / component, ignore_errors=True)
|
|
sources = root / "sources/native"
|
|
sources.mkdir(parents=True, exist_ok=True)
|
|
licenses = root / "licenses/native"
|
|
licenses.mkdir(parents=True, exist_ok=True)
|
|
trees = {}
|
|
for component in ("dav1d", "xz", "ffmpeg"):
|
|
archive = download(lock[component]["source"], root / "downloads")
|
|
shutil.copy2(archive, sources / archive.name)
|
|
trees[component] = extract(archive, work / component)
|
|
dav1d_command = ["meson", "setup", work / "dav1d-build", trees["dav1d"],
|
|
"--prefix=" + str(root / "dav1d"), "--libdir=lib", "--buildtype=release",
|
|
"--default-library=shared", "--wrap-mode=nodownload", "-Denable_tools=false",
|
|
"-Denable_tests=false", "-Denable_examples=false", "-Denable_docs=false"]
|
|
xz_command = [trees["xz"] / "configure", "--prefix=" + str(root / "xz"),
|
|
"--enable-shared", "--disable-static", "--disable-xz", "--disable-xzdec",
|
|
"--disable-lzmadec", "--disable-lzmainfo", "--disable-scripts", "--disable-doc", "--disable-nls"]
|
|
ffmpeg_command = [trees["ffmpeg"] / "configure", "--prefix=" + str(root / "ffmpeg"),
|
|
"--arch=" + arch, "--target-os=darwin", "--cc=" + clang, "--cxx=" + clangxx,
|
|
"--host-cc=" + clang, "--host-cflags=" + flags, "--host-ldflags=" + flags,
|
|
"--extra-cflags=" + flags + " " + shlex.quote("-I" + str(root / "xz/include")),
|
|
"--extra-ldflags=" + flags + " " + shlex.quote("-L" + str(root / "xz/lib")),
|
|
"--enable-shared", "--disable-static", "--disable-programs", "--disable-doc",
|
|
"--disable-encoders", "--disable-muxers", "--disable-avdevice", "--disable-avfilter",
|
|
"--disable-gpl", "--disable-nonfree", "--disable-version3", "--disable-autodetect",
|
|
"--enable-libdav1d", "--enable-securetransport", "--enable-videotoolbox",
|
|
"--enable-audiotoolbox", "--enable-zlib", "--enable-bzlib", "--enable-lzma"]
|
|
configuration = {"arch": arch, "deployment_target": "15.0", "sdk": sdk,
|
|
"environment": {key: environment[key] for key in ("CC", "CXX", "CFLAGS", "CXXFLAGS", "LDFLAGS", "MACOSX_DEPLOYMENT_TARGET", "PKG_CONFIG_LIBDIR", "PKG_CONFIG_PATH")},
|
|
"dav1d": list(map(str, dav1d_command)), "xz": list(map(str, xz_command)),
|
|
"ffmpeg": list(map(str, ffmpeg_command))}
|
|
(licenses / "build-configuration.json").write_text(json.dumps(configuration, indent=2) + "\n")
|
|
run(dav1d_command, env=environment)
|
|
run(["ninja", "-C", work / "dav1d-build"], env=environment)
|
|
run(["ninja", "-C", work / "dav1d-build", "install"], env=environment)
|
|
xz_build = work / "xz-build"
|
|
xz_build.mkdir()
|
|
run(xz_command, cwd=xz_build, env=environment)
|
|
run(["make", "-j", str(os.cpu_count() or 1)], cwd=xz_build, env=environment)
|
|
run(["make", "install"], cwd=xz_build, env=environment)
|
|
build = work / "ffmpeg-build"
|
|
build.mkdir()
|
|
run(ffmpeg_command, cwd=build, env=environment)
|
|
run(["make", "-j", str(os.cpu_count() or 1)], cwd=build, env=environment)
|
|
run(["make", "install"], cwd=build, env=environment)
|
|
for filename in ("COPYING.LGPLv2.1", "LICENSE.md"):
|
|
shutil.copy2(trees["ffmpeg"] / filename, licenses / ("FFmpeg-" + filename))
|
|
shutil.copy2(trees["dav1d"] / "COPYING", licenses / "dav1d-COPYING")
|
|
shutil.copy2(trees["xz"] / "COPYING", licenses / "liblzma-COPYING")
|
|
for filename in ("config.h", "ffbuild/config.mak"):
|
|
shutil.copy2(build / filename, licenses / ("FFmpeg-" + Path(filename).name))
|
|
(licenses / "NOTICE.txt").write_text(
|
|
"FFmpeg 9.0.1: LGPL-2.1-or-later; dav1d 1.5.4: BSD-2-Clause; liblzma 5.8.4: 0BSD/public domain.\n"
|
|
"Exact unmodified source archives accompany this distribution under sources/native.\n"
|
|
"Build commands and flags are in build-configuration.json; source changes: none.\n"
|
|
"This software is based in part on the work of the Independent JPEG Group.\n"
|
|
"FFmpeg JPEG source files were not modified. Apple system libraries are supplied by macOS.\n"
|
|
"You may replace these shared libraries with compatible modified versions; no static linking is used.\n")
|
|
|
|
|
|
def helpers(root, arch, lock):
|
|
binaries = root / "bin"
|
|
binaries.mkdir(parents=True, exist_ok=True)
|
|
sources = root / "sources/helpers"
|
|
sources.mkdir(parents=True, exist_ok=True)
|
|
licenses = root / "licenses/helpers"
|
|
licenses.mkdir(parents=True, exist_ok=True)
|
|
for name in ("yt-dlp", "deno"):
|
|
component = lock[name]
|
|
asset = component["binary"] if name == "yt-dlp" else component["binaries"][arch]
|
|
archive = download(asset, root / "downloads")
|
|
with tempfile.TemporaryDirectory(dir=root) as temporary:
|
|
stage = Path(temporary)
|
|
executable = archive if name == "yt-dlp" else extract(archive, stage) / "deno"
|
|
if not executable.is_file():
|
|
raise RuntimeError(f"Pinned {name} archive does not contain its expected executable")
|
|
target = stage / "installed-executable"
|
|
shutil.copyfile(executable, target)
|
|
target.chmod(0o755)
|
|
os.replace(target, binaries / name)
|
|
for material in component["materials"]:
|
|
path = download(material, root / "downloads")
|
|
shutil.copy2(path, (sources if material.get("kind") == "source" else licenses) / path.name)
|
|
(licenses / (name + "-provenance.json")).write_text(json.dumps(component, indent=2) + "\n")
|
|
(licenses / (name + "-NOTICE.txt")).write_text(component["notice"] + "\n")
|
|
|
|
|
|
def outputs_exist(root, part, lock):
|
|
if part == "helpers":
|
|
return (all((root / "bin" / name).is_file() and os.access(root / "bin" / name, os.X_OK)
|
|
and all(((root / "sources/helpers" if material.get("kind") == "source" else root / "licenses/helpers") / material["filename"]).is_file()
|
|
for material in lock[name]["materials"])
|
|
and (root / "licenses/helpers" / (name + "-NOTICE.txt")).is_file()
|
|
and (root / "licenses/helpers" / (name + "-provenance.json")).is_file()
|
|
for name in ("yt-dlp", "deno")))
|
|
return ((root / "ffmpeg/include/libavcodec/avcodec.h").is_file()
|
|
and all((root / "ffmpeg/lib/pkgconfig" / (name + ".pc")).is_file()
|
|
and any((root / "ffmpeg/lib").glob(name + ".*.dylib"))
|
|
for name in ("libavcodec", "libavformat", "libavutil", "libswscale", "libswresample"))
|
|
and any((root / "dav1d/lib").glob("libdav1d.*.dylib"))
|
|
and any((root / "xz/lib").glob("liblzma.*.dylib"))
|
|
and (root / "licenses/native/build-configuration.json").is_file()
|
|
and all((root / "sources/native" / lock[name]["source"]["filename"]).is_file()
|
|
for name in ("ffmpeg", "dav1d", "xz")))
|
|
|
|
|
|
def self_check():
|
|
"""Offline archive-boundary check: python3 -c 'from Scripts.dependencies import self_check; self_check()'."""
|
|
build = REPOSITORY / ".build"
|
|
build.mkdir(exist_ok=True)
|
|
with tempfile.TemporaryDirectory(dir=build) as temporary:
|
|
root = Path(temporary)
|
|
for name in ("../escape", "/absolute", "safe/../../escape", "safe\\escape"):
|
|
try:
|
|
archive_path(name, root)
|
|
except RuntimeError:
|
|
pass
|
|
else:
|
|
raise AssertionError(f"Unsafe path accepted: {name}")
|
|
assert archive_path("safe/file", root) == root / "safe/file"
|
|
archive = root / "links.tar"
|
|
with tarfile.open(archive, "w") as output:
|
|
link = tarfile.TarInfo("link")
|
|
link.type = tarfile.SYMTYPE
|
|
link.linkname = "../outside"
|
|
output.addfile(link)
|
|
try:
|
|
extract(archive, root / "extracted")
|
|
except RuntimeError:
|
|
pass
|
|
else:
|
|
raise AssertionError("Archive symlink accepted")
|
|
archive = root / "traversal.zip"
|
|
with zipfile.ZipFile(archive, "w") as output:
|
|
output.writestr("../escape", "unsafe")
|
|
try:
|
|
extract(archive, root / "zip-extracted")
|
|
except RuntimeError:
|
|
pass
|
|
else:
|
|
raise AssertionError("ZIP traversal accepted")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--root", required=True, type=Path)
|
|
parser.add_argument("--arch", required=True, choices=("arm64", "x86_64"))
|
|
parser.add_argument("--part", required=True, choices=("native", "helpers"))
|
|
args = parser.parse_args()
|
|
if platform.system() != "Darwin" or platform.machine() != args.arch:
|
|
parser.error(f"Native macOS {args.arch} host required; running {platform.system()} {platform.machine()}. Cross-architecture builds are unsupported (do not run under Rosetta).")
|
|
translated = subprocess.run(["/usr/sbin/sysctl", "-in", "sysctl.proc_translated"],
|
|
capture_output=True, text=True, check=False)
|
|
if translated.stdout.strip() == "1":
|
|
parser.error("Rosetta builds are unsupported; use native arm64 Python and build tools.")
|
|
if not args.root.is_absolute():
|
|
parser.error("--root must be an absolute path beneath the repository's ignored .build directory")
|
|
root = args.root.resolve()
|
|
if not root.is_relative_to((REPOSITORY / ".build").resolve()):
|
|
parser.error("--root must be beneath the repository's ignored .build directory")
|
|
lock = json.loads(LOCK.read_text())
|
|
fingerprint = hashlib.sha256(LOCK.read_bytes() + Path(__file__).read_bytes() + args.arch.encode() + args.part.encode()).hexdigest()
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
stamp = root / (args.part + ".stamp")
|
|
if stamp.is_file() and stamp.read_text().strip() == fingerprint and outputs_exist(root, args.part, lock):
|
|
return 0
|
|
stamp.unlink(missing_ok=True)
|
|
try:
|
|
(native if args.part == "native" else helpers)(root, args.arch, lock)
|
|
if not outputs_exist(root, args.part, lock):
|
|
raise RuntimeError(f"{args.part} preparation completed without all required outputs")
|
|
shutil.copy2(LOCK, root / "licenses/Dependencies.lock.json")
|
|
shutil.copy2(Path(__file__), root / "licenses/dependencies.py")
|
|
with tempfile.NamedTemporaryFile(mode="w", dir=root, delete=False) as output:
|
|
output.write(fingerprint + "\n")
|
|
temporary = Path(output.name)
|
|
os.replace(temporary, stamp)
|
|
except (OSError, ValueError, RuntimeError, subprocess.CalledProcessError, tarfile.TarError, zipfile.BadZipFile) as error:
|
|
print(f"Dependency preparation failed: {error}. Fix the prerequisite/network issue and rerun the same command; no readiness stamp was published.", file=sys.stderr)
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|