#!/usr/bin/env python3 # 20251030 djb # SPDX-License-Identifier: LicenseRef-PD-hp OR CC0-1.0 OR 0BSD OR MIT-0 OR MIT # XXX: start supporting filc patches # XXX: take account of what's installed already import os import sys import shutil import subprocess import multiprocessing import traceback import apt aptcache = apt.Cache() cwd = os.getcwd() top = cwd+'/packages' assert top.startswith('/') # to avoid confusing dpkg srcprefix = '@' assert len(srcprefix) == 1 def issrc(package): return package.startswith(srcprefix) package_want = [] package_visited = set() package_why = {} package2src = {} src2version = {} deps = {} essential = set(['libgcc-s1','libc6','libstdc++6']) done = set() def package_add(package,why): if package not in package_visited: package_visited.add(package) if issrc(package): P = package[1:] if P == 'ncurses': package_add('libgpm-dev',package) else: if package in essential: return version = aptcache[package].candidate.version P = aptcache[package].candidate.source_name package2src[package] = P src2version[P] = version package_add(srcprefix+P,package) if P != 'glibc': for dgroup in list(aptcache[package].candidate.dependencies): package_add(dgroup[0].name,package) package_want.append(package) if package not in package_why: package_why[package] = [] if why not in package_why[package]: package_why[package].append(why) if why != '[REQUESTED]': if why not in deps: deps[why] = set() deps[why].add(package) dryrun = False for package in sys.argv[1:]: if package == '--dry-run': dryrun = True else: package_add(package,'_REQUESTED_') predone = set() for package in package_want: predone.add(package) why = ' '.join(package_why[package]) if package not in deps: deps[package] = set() depslist = ' '.join(sorted(deps[package])) print(f'planning {package} because [{why}] after [{depslist}]') sys.stdout.flush() predone.add(package) if any(q in predone for q in package_why[package]): raise Exception(f'circular: {[q for q in package_why[package] if q in predone]}') try: threads = len(os.sched_getaffinity(0)) except: threads = multiprocessing.cpu_count() threads = os.getenv('THREADS',default=threads) threads = int(threads) if threads < 1: threads = 1 todo = set(package_want) q = multiprocessing.Queue() package2process = {} package2threads = {} threadsused = set() os.makedirs(top,exist_ok=True) for package in package_want: try: shutil.rmtree(f'{top}/{package}',ignore_errors=False) except: pass try: os.remove(f'{top}/{package}') except: pass os.makedirs(f'{top}/{package}',exist_ok=True) def threadfmt(threads): result = [] lastrange = None for t in sorted(threads): if lastrange is None: lastrange = t,t continue if t == lastrange[1]+1: lastrange = lastrange[0],t continue if lastrange[0] == lastrange[1]: result += [f'{lastrange[0]}'] else: result += [f'{lastrange[0]}-{lastrange[1]}'] lastrange = t,t if lastrange[0] == lastrange[1]: result += [f'{lastrange[0]}'] else: result += [f'{lastrange[0]}-{lastrange[1]}'] return ','.join(result) dpkgenv = dict(os.environ,DPKG_GENSYMBOLS_CHECK_LEVEL='0',DEB_BUILD_OPTIONS='crossbuildcanrunhostbinaries nostrip',DEBIAN_FRONTEND='noninteractive',DEBEMAIL='djb@cr.yp.to',DEBFULLNAME='djb') def doit(package,packagethreads): if dryrun: status = 0 else: os.sched_setaffinity(0,packagethreads) status = -1 with open(f'{top}/{package}/log','w') as f: try: f.write(f'===== threads: {threadfmt(packagethreads)}\n') f.write('===== nproc: ') f.flush() subprocess.run(['nproc'],stdout=f,stderr=f) f.write('\n') f.flush() if issrc(package): P = package[1:] version = src2version[P] vers = version.split('-')[0] if P == 'glibc': os.makedirs(f'{top}/{package}/{P}-{vers}/debian',exist_ok=True) with open(f'{top}/{package}/{P}-{vers}/debian/changelog','w') as g: g.write(f'{P} ({version}) unstable; urgency=medium\n') g.write('\n') g.write(' * Initial Release.\n') g.write('\n') g.write(' -- djb Sun, 26 Oct 2025 16:05:17 +0000') with open(f'{top}/{package}/{P}-{vers}/debian/control','w') as g: g.write(f'Source: {P}\n') g.write('Maintainer: djb \n') g.write('Build-Depends: debhelper-compat (= 13)\n') g.write('\n') g.write(f'Package: libc6-dev\n') g.write('Architecture: any\n') g.write('Multi-Arch: same\n') g.write(f'Description: fake libc6-dev\n') with open(f'{top}/{package}/{P}-{vers}/debian/rules','w') as g: g.write('#!/usr/bin/make -f\n') g.write('\n') g.write('build-arch build-indep build \\\n') g.write('install-arch install-indep install \\\n') g.write('binary-arch binary-indep binary \\\n') g.write(':\n') g.write('\tdh $@\n') g.write('\n') g.write('override_dh_installchangelogs:\n') g.write('\techo skipping changelog\n') g.write('\n') g.write('clean:\n') g.write('\tdh_clean $@\n') else: f.write(f'===== apt source {P}\n') f.flush() child = subprocess.run(['apt','source',P],stdout=f,stderr=f,cwd=f'{top}/{package}') f.write(f'===== return code {child.returncode}\n') f.flush() bumps = 0 while True: child = subprocess.run(['dpkg-parsechangelog','--show-field','Version'],cwd=f'{top}/{package}/{P}-{vers}',capture_output=True,universal_newlines=True,check=True) curversion = child.stdout.strip() if curversion == version: break assert bumps < 100 # XXX bumps += 1 f.write(f'===== cd {P}-{vers}; dch --bin-nmu --noquery bump\n') f.flush() child = subprocess.run(['dch','--bin-nmu','--noquery','bump'],stdout=f,stderr=f,cwd=f'{top}/{package}/{P}-{vers}',env=dpkgenv) f.write(f'===== return code {child.returncode}\n') f.flush() f.write(f'===== cd {P}-{vers}; dpkg-buildpackage -d -us -uc -b -a amd64fil0\n') f.flush() child = subprocess.run(['dpkg-buildpackage','-d','-us','-uc','-b','-a','amd64fil0'],stdout=f,stderr=f,cwd=f'{top}/{package}/{P}-{vers}',env=dpkgenv) else: P = package2src[package] version = src2version[P] deb = f'{top}/{srcprefix}{P}/{package}_{version}_amd64fil0.deb' f.write(f'===== sudo dpkg -i {deb}\n') f.flush() child = subprocess.run(['sudo','dpkg','-i',deb],stdout=f,stderr=f) status = child.returncode f.write(f'===== return code {status}\n') f.flush() except: f.write(traceback.format_exc()) f.flush() status = -2 q.put((package,status)) def readytodo(package): return all(dep in done or dep in essential for dep in deps.get(package,[])) def printstatus(): words = sorted(done)+['<==done']+sorted(package2process)+['waiting==>']+sorted(todo) print(' '.join(words)) sys.stdout.flush() def printevent(description,package,packagethreads): print(f' {description} {package} using threads {threadfmt(packagethreads)}') sys.stdout.flush() installing = False printstatus() while len(todo) > 0 or len(threadsused) > 0: if len(todo) > 0 and len(threadsused) < threads: # XXX: use depth and package size to guide sorting srctodonow = [package for package in sorted(todo) if readytodo(package) and issrc(package)] bintodonow = [package for package in sorted(todo) if readytodo(package) and not issrc(package)] bintodonow = bintodonow[:1] if installing: bintodonow = [] todonow = bintodonow+srctodonow if len(todonow) > 0: threadsnow = [set() for j in range(len(todonow))] pos = 0 for j in range(threads): # XXX: want better heuristics to decide number of jobs per process if j not in threadsused: threadsnow[pos].add(j) pos = (pos+1)%len(todonow) for package,packagethreads in zip(todonow,threadsnow): printevent('starting',package,packagethreads) process = multiprocessing.Process(target=doit,args=(package,packagethreads)) process.start() package2process[package] = process package2threads[package] = packagethreads assert threadsused.isdisjoint(packagethreads) threadsused = threadsused.union(packagethreads) todo.remove(package) if not issrc(package): assert not installing installing = True printstatus() continue assert len(threadsused) > 0 package,status = q.get() packagethreads = package2threads[package] description = 'stopped' if status else 'finished' printevent(description,package,packagethreads) package2process[package].join() del package2process[package] threadsused -= packagethreads done.add(package) if not issrc(package): assert installing installing = False printstatus()