mirror of
https://git.savannah.gnu.org/git/emacs.git
synced 2025-01-17 17:58:46 +00:00
* admin/nt/dist-build/build-dep-zips.py: (deps src) handle zst
This patch fixes various problems preventing this script from working. The job of this script is to collect Emacs' dependanicies and the (MSYS2/MinGW64) source archives for dependancies (including 2nd etc. order dependancies) into zip archives. GPL requires distributing such archives (along with Emacs' own sources) when distributing GPL binaries. Most principle of the changes is support for MSYS2 "w64" source archives in zst format; previously this script assumed all sources were available as tar.gz files. Additionaly several DLLs are added to list of required, including sqllite3 and tree-sitter. In general this is a maintaince fix to ensure others have access to the code used to create pretest (and, presumably release) version of Emacs 30. Additional changes are required to progmatically identify changes in Emacs' dependancies (e.g. after building a new Emacs binary for distribution). There is commented out code to this effect in the changes I've installed which did work, for a brief period in time; I suspect some changes in MSYS2 made it more difficult (or impossible?) to invoke the MSYS2 shell (under which this python script must run) from the MINGW64 (under which Emacs is built). I may resort to a powershell script as a way to orchistrate/invoke both MSYS2 and MING64 but I'm currently still messing with a Makefile based approach (which would run under MINGW64), in any event I believe we're all agreed this script must interigate Emacs to create the "starting list" of dependancies to include as a way to resolve this issue (missing/incorrect deps are frequent source of broken binary releases). (bug#65206)
This commit is contained in:
Corwin Brust
parent
8064b2a679
commit
8b95549e90
@ -1,6 +1,6 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
## Copyright (C) 2017-2024 Free Software Foundation, Inc.
|
||||
## Copyright (C) 2017-2023 Free Software Foundation, Inc.
|
||||
|
||||
## This file is part of GNU Emacs.
|
||||
|
||||
@ -20,41 +20,56 @@
|
||||
import os
|
||||
import shutil
|
||||
import re
|
||||
import functools
|
||||
import operator
|
||||
import subprocess
|
||||
|
||||
from subprocess import check_output
|
||||
|
||||
## Constants
|
||||
EMACS_MAJOR_VERSION="28"
|
||||
EMACS_MAJOR_VERSION= os.getenv('EMACS_MAJOR_VERSION') or "30"
|
||||
|
||||
# This list derives from the features we want Emacs to compile with.
|
||||
# Base URI for the package sources mapped in PKG_REQ
|
||||
SRC_REPO="https://repo.msys2.org/mingw/sources"
|
||||
|
||||
# Map items in `dynamic-library-alist' to source pakages
|
||||
PKG_REQ='''mingw-w64-x86_64-giflib
|
||||
mingw-w64-x86_64-gnutls
|
||||
mingw-w64-x86_64-harfbuzz
|
||||
mingw-w64-x86_64-jansson
|
||||
mingw-w64-x86_64-lcms2
|
||||
mingw-w64-x86_64-libjpeg-turbo
|
||||
mingw-w64-x86_64-libpng
|
||||
mingw-w64-x86_64-librsvg
|
||||
mingw-w64-x86_64-libwebp
|
||||
mingw-w64-x86_64-libtiff
|
||||
mingw-w64-x86_64-libxml2
|
||||
mingw-w64-x86_64-xpm-nox'''.split()
|
||||
mingw-w64-x86_64-gmp
|
||||
mingw-w64-x86_64-xpm-nox
|
||||
mingw-w64-x86_64-tree-sitter
|
||||
mingw-w64-x86_64-sqlite3'''.split()
|
||||
|
||||
DLL_REQ='''libgif
|
||||
libgnutls
|
||||
libharfbuzz
|
||||
liblcms2
|
||||
libturbojpeg
|
||||
libpng
|
||||
librsvg
|
||||
libtiff
|
||||
libxml
|
||||
libXpm'''.split()
|
||||
# Emacs style path to dependancy DLLs on build system
|
||||
DLL_SRC="c:/msys64/mingw64/bin"
|
||||
|
||||
# libraries we never include
|
||||
DLL_SKIP=["libgccjit-0.dll"]
|
||||
|
||||
# Report first existing file for entries in dynamic-library-alist
|
||||
# ELISP_PROG="""
|
||||
# (message "%s" (mapconcat 'identity (remove nil
|
||||
# (mapcar (lambda(lib)
|
||||
# (seq-find
|
||||
# (lambda(file)
|
||||
# (file-exists-p
|
||||
# (file-name-concat "{}"
|
||||
# file)))
|
||||
# (cdr lib)))
|
||||
# dynamic-library-alist)
|
||||
# ) "\\n"))
|
||||
# """.format(DLL_SRC)
|
||||
|
||||
## Options
|
||||
DRY_RUN=False
|
||||
|
||||
# NEW_EMACS="bin/emacs.exe"
|
||||
|
||||
def check_output_maybe(*args,**kwargs):
|
||||
if(DRY_RUN):
|
||||
@ -62,31 +77,76 @@ def check_output_maybe(*args,**kwargs):
|
||||
else:
|
||||
return check_output(*args,**kwargs)
|
||||
|
||||
####################
|
||||
## DLL Capture
|
||||
|
||||
# entry point
|
||||
def gather_deps():
|
||||
|
||||
os.mkdir("x86_64")
|
||||
os.chdir("x86_64")
|
||||
|
||||
for dep in full_dll_dependency():
|
||||
check_output_maybe(["cp /mingw64/bin/{}*.dll .".format(dep)],
|
||||
shell=True)
|
||||
#full=full_dll_dependency(init_deps())
|
||||
#filtered=filter(lambda x: x not in DLL_SKIP, full)
|
||||
#print("full:",full.len(), " filtered:",filtered.len())
|
||||
#exit
|
||||
|
||||
print("Zipping")
|
||||
check_output_maybe("zip -9r ../emacs-{}-{}deps.zip *"
|
||||
.format(EMACS_MAJOR_VERSION, DATE),
|
||||
shell=True)
|
||||
for dep in full_dll_dependency(init_deps()):
|
||||
if dep not in DLL_SKIP:
|
||||
if args.l != True:
|
||||
print("Adding dep", dep)
|
||||
check_output_maybe(["cp /mingw64/bin/{} .".format(dep)], shell=True)
|
||||
else:
|
||||
if args.l != True:
|
||||
print("Skipping dep", dep)
|
||||
|
||||
zipfile="../emacs-{}-{}deps.zip".format(EMACS_MAJOR_VERSION, DATE)
|
||||
tmpfile="{}.tmp".format(zipfile)
|
||||
print("Zipping deps in", os.getcwd(), "as", tmpfile)
|
||||
check_output_maybe("zip -9vr {} *.dll".format(tmpfile), shell=True)
|
||||
if os.path.isfile(zipfile):
|
||||
os.remove(zipfile)
|
||||
os.rename(tmpfile, zipfile)
|
||||
print("Deps updated in", os.getcwd(), "as", zipfile)
|
||||
os.chdir("../")
|
||||
|
||||
## Return all Emacs dependencies
|
||||
def full_dll_dependency():
|
||||
deps = [dll_dependency(dep) for dep in DLL_REQ]
|
||||
return set(sum(deps, []) + DLL_REQ)
|
||||
# Return dependancies listed in Emacs
|
||||
def init_deps():
|
||||
return '''libXpm-nox4.dll
|
||||
libpng16-16.dll
|
||||
libjpeg-8.dll
|
||||
libgif-7.dll
|
||||
librsvg-2-2.dll
|
||||
libwebp-7.dll
|
||||
libwebpdemux-2.dll
|
||||
libsqlite3-0.dll
|
||||
libgdk_pixbuf-2.0-0.dll
|
||||
libglib-2.0-0.dll
|
||||
libgio-2.0-0.dll
|
||||
libgobject-2.0-0.dll
|
||||
libgnutls-30.dll
|
||||
libxml2-2.dll
|
||||
zlib1.dll
|
||||
liblcms2-2.dll
|
||||
libgccjit-0.dll
|
||||
libtree-sitter.dll'''.split()
|
||||
# job_args=[NEW_EMACS, "--batch", "--eval", ELISP_PROG]
|
||||
# #print("args: ", job_args)
|
||||
# return subprocess.check_output(job_args, stderr=subprocess.STDOUT
|
||||
# ).decode('utf-8').splitlines()
|
||||
|
||||
## Dependencies for a given DLL
|
||||
# Return all second order dependencies
|
||||
def full_dll_dependency(dlls):
|
||||
deps = [dll_dependency(dep) for dep in dlls]
|
||||
return set(sum(deps, []) + dlls)
|
||||
|
||||
#xs = filter(lambda x: x.attribute == value, xs)
|
||||
|
||||
# Dependencies for a given DLL
|
||||
def dll_dependency(dll):
|
||||
output = check_output(["/mingw64/bin/ntldd", "--recursive",
|
||||
"/mingw64/bin/{}*.dll".format(dll)]).decode("utf-8")
|
||||
"/mingw64/bin/{}".format(dll)]
|
||||
).decode("utf-8")
|
||||
## munge output
|
||||
return ntldd_munge(output)
|
||||
|
||||
@ -101,9 +161,8 @@ def ntldd_munge(out):
|
||||
|
||||
## if it's the former, we want it, if its the later we don't
|
||||
splt = dep.split()
|
||||
if len(splt) > 2 and "msys64" in splt[2]:
|
||||
print("Adding dep", splt[0])
|
||||
rtn.append(splt[0].split(".")[0])
|
||||
if len(splt) > 2 and "mingw64" in splt[2]:
|
||||
rtn.append(splt[0])
|
||||
|
||||
return rtn
|
||||
|
||||
@ -112,26 +171,92 @@ def ntldd_munge(out):
|
||||
## Packages to fiddle with
|
||||
## Source for gcc-libs is part of gcc
|
||||
SKIP_SRC_PKGS=["mingw-w64-gcc-libs"]
|
||||
SKIP_DEP_PKGS=frozenset(["mingw-w64-x86_64-glib2"])
|
||||
MUNGE_SRC_PKGS={"mingw-w64-libwinpthread-git":"mingw-w64-winpthreads-git"}
|
||||
SKIP_DEP_PKGS=["mingw-w64-glib2", "mingw-w64-ca-certificates-20211016-3"]
|
||||
MUNGE_SRC_PKGS={
|
||||
"mingw-w64-libwinpthread-git":"mingw-w64-winpthreads-git",
|
||||
"mingw-w64-gettext-runtime":"mingw-w64-gettext"
|
||||
}
|
||||
MUNGE_DEP_PKGS={
|
||||
"mingw-w64-x86_64-libwinpthread":"mingw-w64-x86_64-libwinpthread-git",
|
||||
"mingw-w64-x86_64-libtre": "mingw-w64-x86_64-libtre-git",
|
||||
}
|
||||
SRC_EXT={
|
||||
"mingw-w64-freetype": ".src.tar.zst",
|
||||
"mingw-w64-fribidi": ".src.tar.zst",
|
||||
"mingw-w64-glib2": ".src.tar.zst",
|
||||
"mingw-w64-harfbuzz": ".src.tar.zst",
|
||||
"mingw-w64-libunistring": ".src.tar.zst",
|
||||
"mingw-w64-winpthreads-git": ".src.tar.zst",
|
||||
"mingw-w64-ca-certificates": ".src.tar.zst",
|
||||
"mingw-w64-libxml2": ".src.tar.zst",
|
||||
"mingw-w64-ncurses": ".src.tar.zst",
|
||||
"mingw-w64-openssl": ".src.tar.zst",
|
||||
"mingw-w64-pango": ".src.tar.zst",
|
||||
"mingw-w64-python": ".src.tar.zst",
|
||||
"mingw-w64-sqlite3": ".src.tar.zst",
|
||||
"mingw-w64-xpm-nox": ".src.tar.zst",
|
||||
"mingw-w64-xz": ".src.tar.zst",
|
||||
"mingw-w64-bzip2": ".src.tar.zst",
|
||||
"mingw-w64-cairo": ".src.tar.zst",
|
||||
"mingw-w64-expat": ".src.tar.zst",
|
||||
"mingw-w64-fontconfig": ".src.tar.zst",
|
||||
"mingw-w64-gdk-pixbuf2": ".src.tar.zst",
|
||||
"mingw-w64-giflib": ".src.tar.zst",
|
||||
"mingw-w64-gmp": ".src.tar.zst",
|
||||
"mingw-w64-gnutls": ".src.tar.zst",
|
||||
"mingw-w64-graphite2": ".src.tar.zst",
|
||||
"mingw-w64-jbigkit": ".src.tar.zst",
|
||||
"mingw-w64-lcms2": ".src.tar.zst",
|
||||
"mingw-w64-lerc": ".src.tar.zst",
|
||||
"mingw-w64-libdatrie": ".src.tar.zst",
|
||||
"mingw-w64-libffi": ".src.tar.zst",
|
||||
"mingw-w64-libiconv": ".src.tar.zst",
|
||||
"mingw-w64-libiconv": ".src.tar.zst",
|
||||
"mingw-w64-libpng": ".src.tar.zst",
|
||||
"mingw-w64-librsvg": ".src.tar.zst",
|
||||
"mingw-w64-libsystre": ".src.tar.zst",
|
||||
"mingw-w64-libtasn": ".src.tar.zst",
|
||||
"mingw-w64-libthai": ".src.tar.zst",
|
||||
"mingw-w64-libtiff": ".src.tar.zst",
|
||||
"mingw-w64-libtre-git": ".src.tar.zst",
|
||||
"mingw-w64-libwebp": ".src.tar.zst",
|
||||
"mingw-w64-mpdecimal": ".src.tar.zst",
|
||||
"mingw-w64-nettle": ".src.tar.zst",
|
||||
"mingw-w64-p11-kit": ".src.tar.zst",
|
||||
"mingw-w64-pcre": ".src.tar.zst",
|
||||
"mingw-w64-pixman": ".src.tar.zst",
|
||||
"mingw-w64-python-packaging": ".src.tar.zst",
|
||||
"mingw-w64-readline": ".src.tar.zst",
|
||||
"mingw-w64-tcl": ".src.tar.zst",
|
||||
"mingw-w64-termcap": ".src.tar.zst",
|
||||
"mingw-w64-tk": ".src.tar.zst",
|
||||
"mingw-w64-tree-sitter": ".src.tar.zst",
|
||||
"mingw-w64-tzdata": ".src.tar.zst",
|
||||
"mingw-w64-wineditline": ".src.tar.zst",
|
||||
"mingw-w64-zlib": ".src.tar.zst",
|
||||
"mingw-w64-zstd": ".src.tar.zst",
|
||||
"mingw-w64-brotli": ".src.tar.zst",
|
||||
"mingw-w64-gettext": ".src.tar.zst",
|
||||
"mingw-w64-libdeflate": ".src.tar.zst",
|
||||
"mingw-w64-libidn2": ".src.tar.zst",
|
||||
"mingw-w64-libjpeg-turbo": ".src.tar.zst",
|
||||
"mingw-w64-libtasn1": ".src.tar.zst",
|
||||
"mingw-w64-pcre2": ".src.tar.zst",
|
||||
}
|
||||
|
||||
## Currently no packages seem to require this!
|
||||
ARCH_PKGS=[]
|
||||
SRC_REPO="https://repo.msys2.org/mingw/sources"
|
||||
|
||||
def immediate_deps(pkg):
|
||||
package_info = check_output(["pacman", "-Si", pkg]).decode("utf-8").split("\n")
|
||||
|
||||
def immediate_deps(pkgs):
|
||||
package_info = check_output(["pacman", "-Si"] + pkgs).decode("utf-8").splitlines()
|
||||
## Extract the "Depends On" line
|
||||
depends_on = [x for x in package_info if x.startswith("Depends On")][0]
|
||||
## Remove "Depends On" prefix
|
||||
dependencies = depends_on.split(":")[1]
|
||||
|
||||
## Extract the packages listed for "Depends On:" lines.
|
||||
dependencies = [line.split(":")[1].split() for line in package_info
|
||||
if line.startswith("Depends On")]
|
||||
## Flatten dependency lists from multiple packages into one list.
|
||||
dependencies = functools.reduce(operator.iconcat, dependencies, [])
|
||||
## Split into dependencies
|
||||
dependencies = dependencies.strip().split(" ")
|
||||
|
||||
## Remove > signs TODO can we get any other punctuation here?
|
||||
dependencies = [d.split(">")[0] for d in dependencies if d]
|
||||
@ -147,18 +272,16 @@ def extract_deps():
|
||||
print( "Extracting deps" )
|
||||
|
||||
# Get a list of all dependencies needed for packages mentioned above.
|
||||
pkgs = set(PKG_REQ)
|
||||
newdeps = pkgs
|
||||
print("adding...")
|
||||
while True:
|
||||
subdeps = frozenset(immediate_deps(list(newdeps)))
|
||||
newdeps = subdeps - SKIP_DEP_PKGS - pkgs
|
||||
if not newdeps:
|
||||
break
|
||||
print('\n'.join(newdeps))
|
||||
pkgs |= newdeps
|
||||
pkgs = PKG_REQ[:]
|
||||
n = 0
|
||||
while n < len(pkgs):
|
||||
subdeps = immediate_deps(pkgs[n])
|
||||
for p in subdeps:
|
||||
if not (p in pkgs or p in SKIP_DEP_PKGS):
|
||||
pkgs.append(p)
|
||||
n = n + 1
|
||||
|
||||
return list(pkgs)
|
||||
return sorted(pkgs)
|
||||
|
||||
|
||||
def download_source(tarball):
|
||||
@ -206,14 +329,24 @@ def gather_source(deps):
|
||||
## Switch names if necessary
|
||||
pkg_name = MUNGE_SRC_PKGS.get(pkg_name,pkg_name)
|
||||
|
||||
tarball = "{}-{}.src.tar.gz".format(pkg_name,pkg_version)
|
||||
## src archive is usually a .tar.gz
|
||||
if pkg_name in SRC_EXT.keys():
|
||||
src_ext = SRC_EXT[pkg_name]
|
||||
else:
|
||||
src_ext = ".src.tar.gz"
|
||||
|
||||
tarball = "{}-{}{}".format(pkg_name,pkg_version,src_ext)
|
||||
|
||||
download_source(tarball)
|
||||
|
||||
print("Zipping")
|
||||
check_output_maybe("zip -9 ../emacs-{}-{}deps-mingw-w64-src.zip *"
|
||||
.format(EMACS_MAJOR_VERSION,DATE),
|
||||
shell=True)
|
||||
srczip="../emacs-{}-{}deps-mingw-w64-src.zip".format(EMACS_MAJOR_VERSION,DATE)
|
||||
tmpzip="{}.tmp".format(srczip)
|
||||
print("Zipping Dsrc in", os.getcwd(), "as", tmpzip)
|
||||
check_output_maybe("zip -9 {} *".format(tmpzip), shell=True)
|
||||
if os.path.isfile(srczip):
|
||||
os.remove(srczip)
|
||||
os.rename(tmpzip, srczip)
|
||||
print("Dsrc updated in", os.getcwd(), "as", srczip)
|
||||
|
||||
os.chdir("..")
|
||||
|
||||
@ -231,6 +364,9 @@ def clean():
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
#parser.add_argument("emacs", help="emacs executable")
|
||||
|
||||
parser.add_argument("-s", help="snapshot build",
|
||||
action="store_true")
|
||||
|
||||
@ -243,19 +379,29 @@ def clean():
|
||||
parser.add_argument("-d", help="dry run",
|
||||
action="store_true")
|
||||
|
||||
parser.add_argument("-l", help="list dependencies only",
|
||||
parser.add_argument("-l", help="list dependencies",
|
||||
action="store_true")
|
||||
|
||||
parser.add_argument("-e", help="extract direct dependancies",
|
||||
action="store_true")
|
||||
|
||||
args = parser.parse_args()
|
||||
do_all=not (args.c or args.r)
|
||||
|
||||
|
||||
#NEW_EMACS=args.emacs
|
||||
|
||||
DRY_RUN=args.d
|
||||
|
||||
if( args.e ):
|
||||
print("\n".join(init_deps()))
|
||||
|
||||
if( args.l ):
|
||||
print("List of dependencies")
|
||||
print( deps )
|
||||
print("List of dependencies:")
|
||||
print(full_dll_dependency(init_deps()))
|
||||
print("List of source packages:")
|
||||
print( extract_deps() )
|
||||
|
||||
if( args.e or args.l ):
|
||||
exit(0)
|
||||
|
||||
if args.s:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user