From 3fc05cfaecf03a249bb9df1a9566d16ec005c7c3 Mon Sep 17 00:00:00 2001 From: Phillip Lord Date: Tue, 24 Oct 2017 18:34:35 +0100 Subject: [PATCH] Scripts to automate windows binary distribution * admin/nt/dist-build/README-windows-binaries, admin/nt/dist-build/build-dep-zips.py, admin/nt/dist-build/build-zips.sh: New Files --- admin/nt/dist-build/README-windows-binaries | 45 ++++ admin/nt/dist-build/build-dep-zips.py | 223 ++++++++++++++++++++ admin/nt/dist-build/build-zips.sh | 143 +++++++++++++ 3 files changed, 411 insertions(+) create mode 100644 admin/nt/dist-build/README-windows-binaries create mode 100755 admin/nt/dist-build/build-dep-zips.py create mode 100755 admin/nt/dist-build/build-zips.sh diff --git a/admin/nt/dist-build/README-windows-binaries b/admin/nt/dist-build/README-windows-binaries new file mode 100644 index 00000000000..27a5483c02b --- /dev/null +++ b/admin/nt/dist-build/README-windows-binaries @@ -0,0 +1,45 @@ +Windows Binaries +================ + +Currently, we provide four different binary packages for Emacs, which +are: + +emacs-$VERSION-x86_64.zip + +Contains a 64-bit build of Emacs with dependencies. Mostly, this is +the best one to install. + +emacs-$VERSION-x86_64-no-deps.zip + +Contains a 64-bit build of Emacs without any dependencies. This may be +useful if you wish to install where the dependencies are already +available, or if you want the small possible Emacs. + +emacs-$VERSION-i686.zip + +Contains a 32-bit build of Emacs with dependencies. This is useful for +running on a 32-bit machine. + +emacs-$VERSION-i686-no-deps.zip + +Contains a 32-bit build of Emacs without dependencies + +In addition, we provide the following files which will not be useful +for most end-users. + +emacs-26-x86_64-deps.zip + +The dependencies. Unzipping this file on top of +emacs-$VERSION-x86_64-no-deps.zip should result in the same install as +emacs-$VERSION-x86_64.zip. + +emacs-26-i686-deps.zip + +The 32-bit version of the dependencies. + +emacs-26-deps-mingw-w64-src.zip + +The source for the dependencies. Source for Emacs itself is available +in the main distribution tarball. These dependencies were produced +from an updated msys2 at the point of the first pre-test. It is not +intended that these will be updated after that point. \ No newline at end of file diff --git a/admin/nt/dist-build/build-dep-zips.py b/admin/nt/dist-build/build-dep-zips.py new file mode 100755 index 00000000000..33bc4b586c9 --- /dev/null +++ b/admin/nt/dist-build/build-dep-zips.py @@ -0,0 +1,223 @@ +#!/usr/bin/python3 + +## Copyright (C) 2017 Free Software Foundation, Inc. + +## This file is part of GNU Emacs. + +## GNU Emacs is free software: you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation, either version 3 of the License, or +## (at your option) any later version. + +## GNU Emacs is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. + +## You should have received a copy of the GNU General Public License +## along with GNU Emacs. If not, see . +import argparse +import multiprocessing as mp +import glob +import os +import shutil +import re + +from subprocess import check_output + +## Constants +EMACS_MAJOR_VERSION="26" + + +## Options +DRY_RUN=False + +## Packages to fiddle with +SKIP_PKGS=["mingw-w64-gcc-libs"] +MUNGE_PKGS ={"mingw-w64-libwinpthread-git":"mingw-w64-winpthreads-git"} +ARCH_PKGS=["mingw-w64-mpc", + "mingw-w64-termcap", + "mingw-w64-xpm-nox"] +SRC_REPO="https://sourceforge.net/projects/msys2/files/REPOS/MINGW/Sources" + + +def check_output_maybe(*args,**kwargs): + if(DRY_RUN): + print("Calling: {}{}".format(args,kwargs)) + else: + return check_output(*args,**kwargs) + +def extract_deps(): + + # This list derives from the features we want Emacs to compile with. + PKG_REQ='''mingw-w64-x86_64-giflib +mingw-w64-x86_64-gnutls +mingw-w64-x86_64-libjpeg-turbo +mingw-w64-x86_64-libpng +mingw-w64-x86_64-librsvg +mingw-w64-x86_64-libtiff +mingw-w64-x86_64-libxml2 +mingw-w64-x86_64-xpm-nox +mingw-w64-x86_64-lcms2'''.split() + + # Get a list of all dependencies needed for packages mentioned above. + # Run `pactree -lu' for each elment of $PKG_REQ + pkgs = set() + for x in PKG_REQ: + pkgs.update( + check_output(["pactree", "-lu", x]).decode("utf-8").split() + ) + + return sorted(pkgs) + +def gather_deps(deps, arch, directory): + + os.mkdir(arch) + os.chdir(arch) + + ## Replace the architecture with the correct one + deps = [re.sub(r"x86_64",arch,x) for x in deps] + + ## find all files the transitive dependencies + deps_files = check_output( + ["pacman", "-Ql"] + deps + ).decode("utf-8").split("\n") + + ## Produces output like + ## mingw-w64-x86_64-zlib /mingw64/lib/libminizip.a + + ## drop the package name + tmp = deps_files.copy() + deps_files=[] + for d in tmp: + slt = d.split() + if(not slt==[]): + deps_files.append(slt[1]) + + ## sort uniq + deps_files = sorted(list(set(deps_files))) + ## copy all files into local + print("Copying dependencies: {}".format(arch)) + check_output_maybe(["rsync", "-R"] + deps_files + ["."]) + + ## And package them up + os.chdir(directory) + print("Zipping: {}".format(arch)) + check_output_maybe("zip -9r ../../emacs-26-{}-deps.zip *".format(arch), + shell=True) + os.chdir("../../") + + +def download_source(tarball): + print("Downloading {}...".format(tarball)) + check_output_maybe( + "wget -a ../download.log -O {} {}/{}/download" + .format(tarball, SRC_REPO, tarball), + shell=True + ) + print("Downloading {}... done".format(tarball)) + +def gather_source(deps): + + + ## Source for gcc-libs is part of gcc + ## Source for libwinpthread is in libwinpthreads + ## mpc, termcap, xpm -- has x86_64, and i686 versions + + ## This needs to have been run first at the same time as the + ## system was udpated. + os.mkdir("emacs-src") + os.chdir("emacs-src") + + to_download = [] + for pkg in deps: + pkg_name_and_version= \ + check_output(["pacman","-Q", pkg]).decode("utf-8").strip() + + ## Produces output like: + ## mingw-w64-x86_64-zlib 2.43.2 + pkg_name_components = pkg_name_and_version.split() + pkg_name=pkg_name_components[0] + pkg_version=pkg_name_components[1] + + ## make a simple name to make lookup easier + simple_pkg_name = re.sub(r"x86_64-","",pkg_name) + + if(simple_pkg_name in SKIP_PKGS): + continue + + ## Some packages have different source files for different + ## architectures. For these we need two downloads. + if(simple_pkg_name in ARCH_PKGS): + downloads = [pkg_name, + re.sub(r"x86_64","i686",pkg_name)] + else: + downloads = [simple_pkg_name] + + for d in downloads: + ## Switch names if necessary + d = MUNGE_PKGS.get(d,d) + + tarball = "{}-{}.src.tar.gz".format(d,pkg_version) + + to_download.append(tarball) + + ## Download in parallel or it is just too slow + p = mp.Pool(16) + p.map(download_source,to_download) + + print("Zipping") + check_output_maybe("zip -9 ../emacs-{}-deps-mingw-w64-src.zip *" + .format(EMACS_MAJOR_VERSION), + shell=True) + + os.chdir("..") + + +def clean(): + print("Cleaning") + os.path.isdir("emacs-src") and shutil.rmtree("emacs-src") + os.path.isdir("i686") and shutil.rmtree("i686") + os.path.isdir("x86_64") and shutil.rmtree("x86_64") + os.path.isfile("download.log") and os.remove("download.log") + + +if(os.environ["MSYSTEM"] != "MSYS"): + print("Run this script in an MSYS-shell!") + exit(1) + + +parser = argparse.ArgumentParser() +parser.add_argument("-t", help="32 bit deps only", + action="store_true") + +parser.add_argument("-f", help="64 bit deps only", + action="store_true") + +parser.add_argument("-s", help="source code only", + action="store_true") + +parser.add_argument("-c", help="clean only", + action="store_true") + +parser.add_argument("-d", help="dry run", + action="store_true") + +args = parser.parse_args() +do_all=not (args.c or args.s or args.f or args.t) + +deps=extract_deps() + +DRY_RUN=args.d + +if( do_all or args.t ): + gather_deps(deps,"i686","mingw32") + +if( do_all or args.f ): + gather_deps(deps,"x86_64","mingw64") + +if( do_all or args.s ): + gather_source(deps) + +if( args.c ): + clean() diff --git a/admin/nt/dist-build/build-zips.sh b/admin/nt/dist-build/build-zips.sh new file mode 100755 index 00000000000..e78f72cf9c3 --- /dev/null +++ b/admin/nt/dist-build/build-zips.sh @@ -0,0 +1,143 @@ +#!/bin/bash + +## Copyright (C) 2017 Free Software Foundation, Inc. + +## This file is part of GNU Emacs. + +## GNU Emacs is free software: you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation, either version 3 of the License, or +## (at your option) any later version. + +## GNU Emacs is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. + +## You should have received a copy of the GNU General Public License +## along with GNU Emacs. If not, see . + + +function git_up { + echo Making git worktree for Emacs $VERSION + cd $HOME/emacs-build/git/emacs-$MAJOR_VERSION + git pull + git worktree add ../emacs-$BRANCH emacs-$BRANCH + + cd ../emacs-$BRANCH + ./autogen.sh + +} + +function build_zip { + + ARCH=$1 + PKG=$2 + HOST=$3 + + echo Building Emacs-$VERSION for $ARCH + if [ $ARCH == "i686" ] + then + PATH=/mingw32/bin:$PATH + MSYSTEM=MINGW32 + fi + + mkdir --parents $HOME/emacs-build/build/emacs-$VERSION/$ARCH + cd $HOME/emacs-build/build/emacs-$VERSION/$ARCH + + export PKG_CONFIG_PATH=$PKG + ../../../git/emacs-$BRANCH/configure \ + --without-dbus \ + --host=$HOST --without-compress-install \ + CFLAGS="-O2 -static -g3" + make -j 8 install \ + prefix=$HOME/emacs-build/install/emacs-$VERSION/$ARCH + cd $HOME/emacs-build/install/emacs-$VERSION/$ARCH + cp $HOME/emacs-build/deps/libXpm/$ARCH/libXpm-noX4.dll bin + zip -r -9 emacs-$VERSION-$ARCH-no-deps.zip * + mv emacs-$VERSION-$ARCH.zip $HOME/emacs-upload + rm bin/libXpm-noX4.dll + unzip $HOME/emacs-build/deps/emacs-26-$ARCH-deps.zip + zip -r -9 emacs-$VERSION-$ARCH.zip * + mv emacs-$VERSION-$ARCH.zip ~/emacs-upload +} + + +##set -o xtrace +set -o errexit + +SNAPSHOT= + +BUILD_32=1 +BUILD_64=1 +GIT_UP=0 + +while getopts "36ghsV:" opt; do + case $opt in + 3) + BUILD_32=1 + BUILD_64=0 + GIT_UP=0 + ;; + 6) + BUILD_32=0 + BUILD_64=1 + GIT_UP=0 + ;; + + g) + BUILD_32=0 + BUILD_64=0 + GIT_UP=1 + ;; + V) + VERSION=$OPTARG + ;; + s) + SNAPSHOT="-snapshot" + ;; + h) + echo "build-zips.sh" + echo " -3 32 bit build only" + echo " -6 64 bit build only" + echo " -g git update and worktree only" + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + ;; + esac +done + +if [ -z $VERSION ]; +then + echo "doing version thing" + VERSION=` + sed -n 's/^AC_INIT(GNU Emacs,[ ]*\([^ ,)]*\).*/\1/p' < ../../../configure.ac +` +fi + +if [ -z $VERSION ]; +then + echo Cannot determine Emacs version + exit 1 +fi + +MAJOR_VERSION="$(echo $VERSION | cut -d'.' -f1)" +BRANCH=$VERSION +VERSION=$VERSION$SNAPSHOT + +if (($GIT_UP)) +then + git_up +fi + +if (($BUILD_32)) +then + build_zip i686 /mingw32/lib/pkgconfig i686-w64-mingw32 +fi + +if (($BUILD_64)) +then + build_zip x86_64 /mingw64/lib/pkgconfig x86_64-w64-mingw32 +fi