From d895f1b957b69e21dfaa89c6e16d7d4036c6ca43 Mon Sep 17 00:00:00 2001 From: Arnout Engelen Date: Thu, 27 Mar 2025 14:17:41 +0100 Subject: [PATCH] jemalloc: fix reproducibility issues Working towards making the build deterministic regardless of the build host Page size: https://github.com/jemalloc/jemalloc/issues/467 https://sources.debian.org/src/jemalloc/5.3.0-3/debian/rules/ https://github.com/NixOS/nixpkgs/issues/380852 Darwin: Similar to https://github.com/NixOS/nixpkgs/pull/394620 LG_VADDR: https://github.com/jemalloc/jemalloc/issues/1997 --- .../libraries/jemalloc/default.nix | 43 +++++++++++++++---- .../libraries/jemalloc/o3-to-o2.patch | 15 +++++++ 2 files changed, 50 insertions(+), 8 deletions(-) create mode 100644 pkgs/development/libraries/jemalloc/o3-to-o2.patch diff --git a/pkgs/development/libraries/jemalloc/default.nix b/pkgs/development/libraries/jemalloc/default.nix index 9c2eeadd676e..5700ca81ceb2 100644 --- a/pkgs/development/libraries/jemalloc/default.nix +++ b/pkgs/development/libraries/jemalloc/default.nix @@ -1,8 +1,11 @@ { lib, stdenv, - fetchurl, + fetchFromGitHub, fetchpatch, + autogen, + autoconf, + automake, # By default, jemalloc puts a je_ prefix onto all its symbols on OSX, which # then stops downstream builds (mariadb in particular) from detecting it. This # option should remove the prefix and give us a working jemalloc. @@ -16,9 +19,11 @@ stdenv.mkDerivation rec { pname = "jemalloc"; version = "5.3.0"; - src = fetchurl { - url = "https://github.com/jemalloc/jemalloc/releases/download/${version}/${pname}-${version}.tar.bz2"; - sha256 = "sha256-LbgtHnEZ3z5xt2QCGbbf6EeJvAU3mDw7esT3GJrs/qo="; + src = fetchFromGitHub { + owner = "jemalloc"; + repo = "jemalloc"; + tag = version; + hash = "sha256-bb0OhZVXyvN+hf9BpPSykn5cGm87a0C+Y/iXKt9wTSs="; }; patches = [ @@ -33,11 +38,32 @@ stdenv.mkDerivation rec { url = "https://github.com/jemalloc/jemalloc/commit/4422f88d17404944a312825a1aec96cd9dc6c165.patch"; hash = "sha256-dunkE7XHzltn5bOb/rSHqzpRniAFuGubBStJeCxh0xo="; }) + # -O3 appears to introduce an unreproducibility where + # `rtree_read.constprop.0` shows up in some builds but + # not others, so we fall back to O2: + ./o3-to-o2.patch ]; + nativeBuildInputs = [ + autogen + autoconf + automake + ]; + + preConfigure = + '' + ./autogen.sh + '' + + lib.optionalString stdenv.hostPlatform.isDarwin '' + export LDFLAGS="$LDFLAGS -Wl,-no_uuid -Wl,-install_name,@rpath/libjemalloc.2.dylib"; + ''; + configureFlags = + [ + "--with-lg-vaddr=${with stdenv.hostPlatform; toString (if isILP32 then 32 else parsed.cpu.bits)}" + ] # see the comment on stripPrefix - lib.optional stripPrefix "--with-jemalloc-prefix=" + ++ lib.optional stripPrefix "--with-jemalloc-prefix=" ++ lib.optional disableInitExecTls "--disable-initial-exec-tls" # jemalloc is unable to correctly detect transparent hugepage support on # ARM (https://github.com/jemalloc/jemalloc/issues/526), and the default @@ -46,9 +72,10 @@ stdenv.mkDerivation rec { "--disable-thp" "je_cv_thp=no" ] - # AArch64 has configurable page size up to 64k. The default configuration - # for jemalloc only supports 4k page sizes. - ++ lib.optional stdenv.hostPlatform.isAarch64 "--with-lg-page=16" + # The upstream default is dependent on the builders' page size + # https://github.com/jemalloc/jemalloc/issues/467 + # https://sources.debian.org/src/jemalloc/5.3.0-3/debian/rules/ + ++ [ (if stdenv.hostPlatform.isAarch64 then "--with-lg-page=16" else "--with-lg-page=12") ] # See https://github.com/jemalloc/jemalloc/issues/1997 # Using a value of 48 should work on both emulated and native x86_64-darwin. ++ lib.optional (stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isx86_64) "--with-lg-vaddr=48"; diff --git a/pkgs/development/libraries/jemalloc/o3-to-o2.patch b/pkgs/development/libraries/jemalloc/o3-to-o2.patch new file mode 100644 index 000000000000..66a0310ad2de --- /dev/null +++ b/pkgs/development/libraries/jemalloc/o3-to-o2.patch @@ -0,0 +1,15 @@ +diff --git a/configure.ac b/configure.ac +index 1c9c5067..a0856b89 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -1372,8 +1372,8 @@ AC_SUBST([enable_debug]) + dnl Only optimize if not debugging. + if test "x$enable_debug" = "x0" ; then + if test "x$GCC" = "xyes" ; then +- JE_CFLAGS_ADD([-O3]) +- JE_CXXFLAGS_ADD([-O3]) ++ JE_CFLAGS_ADD([-O2]) ++ JE_CXXFLAGS_ADD([-O2]) + JE_CFLAGS_ADD([-funroll-loops]) + elif test "x$je_cv_msvc" = "xyes" ; then + JE_CFLAGS_ADD([-O2])