package.py (5248B)
1 import argparse 2 import hashlib 3 import io 4 import json 5 import os 6 from pathlib import Path 7 import platform 8 import subprocess 9 import tarfile 10 import tempfile 11 12 13 ROOT = Path(__file__).resolve().parent.parent 14 15 16 def source_licenses(manifest, source_root, root): 17 licenses = {} 18 for line in manifest.decode().splitlines(): 19 if not line.strip() or line.lstrip().startswith("#"): 20 continue 21 component, archive, *_ = line.split() 22 for value in (component, archive): 23 if value in (".", "..") or any(character not in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-" for character in value): 24 raise ValueError("invalid source name: " + value) 25 directory = source_root / archive.rsplit(".tar.", 1)[0] 26 if not directory.is_dir(): 27 raise ValueError("missing extracted source: " + str(directory)) 28 for path in sorted(directory.rglob("*")): 29 relative = path.relative_to(directory) 30 if (path.name.upper().startswith(("LICENSE", "COPYING", "COPYRIGHT", "NOTICE")) 31 or relative.parts[0].upper() == "LICENSES") and path.is_file(): 32 licenses[root + "/share/musllvm/licenses/" + component + "/" + relative.as_posix()] = path.read_bytes() 33 return licenses 34 35 36 def package(prefix, target, sources, output, source_root=None): 37 if not target or any(character not in "abcdefghijklmnopqrstuvwxyz0123456789_-" for character in target): 38 raise ValueError("invalid target: " + target) 39 if platform.system() != "Linux": 40 raise ValueError("packaging requires a Linux host") 41 prefix = prefix.resolve() 42 for required in ("host/bin/clang", "host/bin/ld.lld", "bin/" + target + "-cc", 43 "sysroot/usr/lib/libc.so", "share/musllvm/resource-version"): 44 if not (prefix / required).is_file(): 45 raise ValueError("incomplete toolchain: " + required) 46 for path in prefix.rglob("*"): 47 if path.is_symlink(): 48 if os.path.isabs(os.readlink(path)): 49 raise ValueError("absolute symlink: " + str(path)) 50 try: 51 path.resolve(strict=True).relative_to(prefix) 52 except (ValueError, FileNotFoundError, RuntimeError) as error: 53 raise ValueError("symlink outside the toolchain or missing its target: " + str(path)) from error 54 55 commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() 56 epoch = int(subprocess.check_output(["git", "show", "-s", "--format=%ct", commit], cwd=ROOT, text=True)) 57 host = platform.machine() + "-linux" 58 libc_name, libc_version = platform.libc_ver() 59 manifest = sources.read_bytes() 60 metadata = { 61 "commit": commit, 62 "host": host, 63 "host_libc": {"name": libc_name, "version": libc_version}, 64 "target": target, 65 "llvm_mode": "source", 66 "source_manifest_sha256": hashlib.sha256(manifest).hexdigest(), 67 } 68 root = "musllvm-" + target 69 generated = { 70 root + "/share/musllvm/sources.list": manifest, 71 root + "/share/musllvm/build-info.json": (json.dumps(metadata, indent=2, sort_keys=True) + "\n").encode(), 72 } 73 generated.update(source_licenses(manifest, source_root or ROOT / "build/src", root)) 74 75 def normalize(info): 76 info.uid = info.gid = 0 77 info.uname = info.gname = "" 78 info.mtime = epoch 79 return info 80 81 def installed(info): 82 if info.name == root + "/.musllvm-config" or info.name in generated: 83 return None 84 return normalize(info) 85 86 output.mkdir(parents=True, exist_ok=True) 87 archive = output / (root + "-host-" + host + ".tar.xz") 88 descriptor, temporary = tempfile.mkstemp(prefix=".package-", dir=output) 89 os.close(descriptor) 90 try: 91 with tarfile.open(temporary, "w:xz", dereference=False) as stream: 92 stream.add(prefix, arcname=root, filter=installed) 93 for name, content in generated.items(): 94 info = tarfile.TarInfo(name) 95 info.size = len(content) 96 info.mode = 0o644 97 stream.addfile(normalize(info), io.BytesIO(content)) 98 os.replace(temporary, archive) 99 finally: 100 if os.path.exists(temporary): 101 os.unlink(temporary) 102 digest = hashlib.sha256() 103 with archive.open("rb") as stream: 104 for block in iter(lambda: stream.read(1024 * 1024), b""): 105 digest.update(block) 106 archive.with_name(archive.name + ".sha256").write_text(digest.hexdigest() + " " + archive.name + "\n") 107 return archive 108 109 110 def main(): 111 parser = argparse.ArgumentParser(description="Package a toolchain with bundled host tools") 112 parser.add_argument("--prefix", type=Path, required=True) 113 parser.add_argument("--target", required=True) 114 parser.add_argument("--sources", type=Path, required=True) 115 parser.add_argument("--output", type=Path, required=True) 116 args = parser.parse_args() 117 try: 118 print(package(args.prefix, args.target, args.sources, args.output)) 119 except (OSError, ValueError, subprocess.CalledProcessError) as error: 120 parser.exit(1, "package: " + str(error) + "\n") 121 122 123 if __name__ == "__main__": 124 main()