talexander/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Get bazel building on all major platforms, with tests

Browse Source
This commit is contained in:
Guillaume Maudoux 2023-10-23 21:05:21 +02:00
parent 30cbfd470d
commit 022befe8ae
10 changed files with 641 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
pkgs/development/tools/build-managers/bazel/bazel_7/bazel-repository-cache.nix
View File

@ -1,5 +1,5 @@
{ lib
, nix
, rnix-hashes
, runCommand
, fetchurl
# The path to the right MODULE.bazel.lock
@ -29,7 +29,7 @@ let
# like foldl', force evaluation of intermediate results
builtins.seq acc' children;
extract_source = acc: value:
extract_source = f: acc: value:
# We take any "attributes" object that has a "sha256" field. Every value
# under "attributes" is assumed to be an object, and all the "attributes"
# with a "sha256" field are assumed to have either a "urls" or "url" field.
@ -47,8 +47,6 @@ let
# };
# }
let
# remove the "--" prefix, abusing undocumented negative substring length
sanitize = builtins.substring 2 (-1);
attrs = value.attributes;
entry = hash: urls: {
${hash} = fetchurl {
@ -58,31 +56,26 @@ let
passthru.sha256 = hash;
};
};
insert = acc: hash: urls: acc // entry (sanitize hash) (map sanitize urls);
insert = acc: hash: urls:
acc // entry (sanitize hash) (map sanitize urls);
accWithRemotePatches = lib.foldlAttrs
(acc: url: hash: insert acc hash [ url ])
acc
(attrs.remote_patches or { });
accWithNewSource = insert
accWithRemotePatches
(attrs.integrity or attrs.sha256)
(attrs.urls or [ attrs.url ]);
in
if builtins.isAttrs value && value ? attributes
&& (attrs ? sha256 || attrs ? integrity)
then
#builtins.trace attrs
(
insert
acc
(attrs.integrity or attrs.sha256)
(attrs.urls or [ attrs.url ])
)
else if builtins.isAttrs value && value ? remote_patches
&& builtins.isAttrs value.remote_patches
then
#builtins.trace value.remote_patches
(
lib.foldlAttrs
(acc: url: hash: insert acc hash [ url ])
acc
value.remote_patches
)
else acc;
&& f attrs.name
then accWithNewSource else acc;
inputs = foldlJSON extract_source { } modules;
# remove the "--" prefix, abusing undocumented negative substring length
sanitize = builtins.substring 2 (-1);
inputs = foldlJSON (extract_source (n: builtins.trace (sanitize n) true)) { } modules;
requiredHashes = builtins.fromJSON (builtins.readFile requiredDeps);
requiredAttrs = lib.genAttrs requiredHashes throw;
@ -94,16 +87,16 @@ let
command = ''
mkdir -p $out/content_addressable/sha256
cd $out/content_addressable/sha256
cd $out
'' + lib.concatMapStrings
# TODO: Do not re-hash. Use nix-hash to convert hashes
(drv: ''
filename=$(basename "${lib.head drv.urls}")
echo Caching $filename
hash=$(${nix}/bin/nix-hash --type sha256 --to-base16 ${drv.sha256})
mkdir -p $hash
ln -sfn ${drv} $hash/file
ln -sfn ${drv} $filename
hash=$(${rnix-hashes}/bin/rnix-hashes --encoding BASE16 ${drv.sha256} | cut -f 2)
mkdir -p content_addressable/sha256/$hash
ln -sfn ${drv} content_addressable/sha256/$hash/file
# Expect file name conflicts
ln -sn ${drv} $filename || true
'')
(builtins.attrValues requiredInputs ++ extraInputs)
;

198
pkgs/development/tools/build-managers/bazel/bazel_7/default.nix
View File

@ -54,7 +54,7 @@
# Always assume all markers valid (this is needed because we remove markers; they are non-deterministic).
# Also, don't clean up environment variables (so that NIX_ environment variables are passed to compilers).
, enableNixHacks ? false
}:
}@args:
let
version = "7.0.0-pre.20230917.3";
@ -136,71 +136,75 @@ let
# on aarch64 Darwin, `uname -m` returns "arm64"
arch = with stdenv.hostPlatform; if isDarwin && isAarch64 then "arm64" else parsed.cpu.name;
#build --extra_toolchains=@local_jdk//:all
#build --tool_java_runtime_version=local_jdk
#build --java_runtime_version=local_jdk
#build --repo_env=JAVA_HOME=${buildJdk}${if isDarwin then "/zulu-11.jdk/Contents/Home" else "/lib/openjdk"}
bazelRC = writeTextFile {
name = "bazel-rc";
text = ''
startup --server_javabase=${buildJdk}
build --extra_toolchains=@local_jdk//:all
# Register nix-specific nonprebuilt java toolchains
build --extra_toolchains=@bazel_tools//tools/jdk:all
# and set bazel to use them by default
build --tool_java_runtime_version=local_jdk
build --java_runtime_version=local_jdk
build --repo_env=JAVA_HOME=${buildJdk}${if isDarwin then "/zulu-11.jdk/Contents/Home" else "/lib/openjdk"}
# load default location for the system wide configuration
try-import /etc/bazel.bazelrc
'';
};
bazelNixFlagsScript = writeScript "bazel-nix-flags" ''
cat << EOF
common --announce_rc
build --toolchain_resolution_debug=".*"
build --local_ram_resources=HOST_RAM*.5
build --local_cpu_resources=HOST_CPUS*.75
build --copt=$(echo $NIX_CFLAGS_COMPILE | sed -e 's/ / --copt=/g')
build --host_copt=$(echo $NIX_CFLAGS_COMPILE | sed -e 's/ / --host_copt=/g')
build --linkopt=$(echo $(< ${stdenv.cc}/nix-support/libcxx-ldflags) | sed -e 's/ / --linkopt=/g')
build --host_linkopt=$(echo $(< ${stdenv.cc}/nix-support/libcxx-ldflags) | sed -e 's/ / --host_linkopt=/g')
build --linkopt=-Wl,$(echo $NIX_LDFLAGS | sed -e 's/ / --linkopt=-Wl,/g')
build --host_linkopt=-Wl,$(echo $NIX_LDFLAGS | sed -e 's/ / --host_linkopt=-Wl,/g')
build --extra_toolchains=@bazel_tools//tools/jdk:nonprebuilt_toolchain_definition
build --verbose_failures
build --curses=no
build --features=-layering_check
build --experimental_strict_java_deps=off
build --strict_proto_deps=off
build --extra_toolchains=@bazel_tools//tools/jdk:nonprebuilt_toolchain_java11_definition
build --extra_toolchains=@local_jdk//:all
build --tool_java_runtime_version=local_jdk_11
build --java_runtime_version=local_jdk_11
build --repo_env=JAVA_HOME=${buildJdk}${if isDarwin then "/zulu-11.jdk/Contents/Home" else "/lib/openjdk"}
EOF
'';
in
stdenv.mkDerivation rec {
pname = "bazel";
inherit version;
meta = with lib; {
homepage = "https://github.com/bazelbuild/bazel/";
description = "Build tool that builds code quickly and reliably";
sourceProvenance = with sourceTypes; [
fromSource
binaryBytecode # source bundles dependencies as jars
];
license = licenses.asl20;
maintainers = lib.teams.bazel.members;
inherit platforms;
};
inherit src;
inherit version src;
inherit sourceRoot;
patches = [
# TODO: Make GSON work,
# In particular, our bazel build cannot emit MODULE.bazel.lock
# it only produces an empty json object `{ }`.
./serialize_nulls.patch
# --extra_toolchains defined later should come before the ones defined earlier.
# As-is, this patch also inverts the order of extra_toolchains lists, but it's just a hack
./extra_toolchains_precedence.patch
#./toolchain_better_debug.patch
./toolchain_group_debug.patch
# Remote java toolchains do not work on NixOS because they download binaries,
# so we need to use the @local_jdk//:jdk
# It could in theory be done by registering @local_jdk//:all toolchains,
# but these java toolchains still bundle binaries for ijar and stuff. So we
# need a nonprebult java toolchain (where ijar and stuff is built from
# sources).
# There is no such java toolchain, so we introduce one here.
# By providing no version information, the toolchain will set itself to the
# version of $JAVA_HOME/bin/java, just like the local_jdk does.
# To ensure this toolchain gets used, we can set
# --{,tool_}java_runtime_version=local_jdk and rely on the fact no java
# toolchain registered by default uses the local_jdk, making the selection
# unambiguous.
# This toolchain has the advantage that it can use any ambiant java jdk,
# not only a given, fixed version. It allows bazel to work correctly in any
# environment where JAVA_HOME is set to the right java version, like inside
# nix derivations.
# However, this patch breaks bazel hermeticity, by picking the ambiant java
# version instead of the more hermetic remote_jdk prebuilt binaries that
# rules_java provide by default. It also requires the user to have a
# JAVA_HOME set to the exact version required by the project.
# With more code, we could define java toolchains for all the java versions
# supported by the jdk as in rules_java's
# toolchains/local_java_repository.bzl, but this is not implemented here.
# To recover vanilla behavior, non NixOS users can set
# --{,tool_}java_runtime_version=remote_jdk, effectively reverting the
# effect of this patch and the fake system bazelrc.
./java_toolchain.patch
# Bazel integrates with apple IOKit to inhibit and track system sleep.
# Inside the darwin sandbox, these API calls are blocked, and bazel
# crashes. It seems possible to allow these APIs inside the sandbox, but it
@ -239,10 +243,7 @@ stdenv.mkDerivation rec {
# This patch removes using the -fobjc-arc compiler option and makes the code
# compile without automatic reference counting. Caveat: this leaks memory, but
# we accept this fact because xcode_locator is only a short-lived process used during the build.
(substituteAll {
src = ./no-arc.patch;
multiBinPatch = if stdenv.hostPlatform.system == "aarch64-darwin" then "arm64" else "x86_64";
})
./no-arc.patch
# --experimental_strict_action_env (which may one day become the default
# see bazelbuild/bazel#2574) hardcodes the default
@ -270,13 +271,10 @@ stdenv.mkDerivation rec {
# See enableNixHacks argument above.
++ lib.optional enableNixHacks ./nix-hacks.patch;
# Bazel starts a local server and needs to bind a local address.
__darwinAllowLocalNetworking = true;
postPatch =
let
darwinPatches = ''
bazelLinkFlags () {
eval set -- "$NIX_LDFLAGS"
local flag
@ -285,10 +283,6 @@ stdenv.mkDerivation rec {
done
}
# Disable Bazel's Xcode toolchain detection which would configure compilers
# and linkers from Xcode instead of from PATH
export BAZEL_USE_CPP_ONLY_TOOLCHAIN=1
# Explicitly configure gcov since we don't have it on Darwin, so autodetection fails
export GCOV=${coreutils}/bin/false
@ -300,6 +294,10 @@ stdenv.mkDerivation rec {
# https://github.com/NixOS/nixpkgs/pull/41589
export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -isystem ${lib.getDev libcxx}/include/c++/v1"
# This variable is used by bazel to propagate env vars for homebrew,
# which is exactly what we need too.
export HOMEBREW_RUBY_PATH="foo"
# don't use system installed Xcode to run clang, use Nix clang instead
sed -i -E \
-e "s;/usr/bin/xcrun (--sdk macosx )?clang;${stdenv.cc}/bin/clang $NIX_CFLAGS_COMPILE $(bazelLinkFlags) -framework CoreFoundation;g" \
@ -307,8 +305,6 @@ stdenv.mkDerivation rec {
scripts/bootstrap/compile.sh \
tools/osx/BUILD
substituteInPlace scripts/bootstrap/compile.sh --replace ' -mmacosx-version-min=10.9' ""
# nixpkgs's libSystem cannot use pthread headers directly, must import GCD headers instead
sed -i -e "/#include <pthread\/spawn.h>/i #include <dispatch/dispatch.h>" src/main/cpp/blaze_util_darwin.cc
@ -316,23 +312,26 @@ stdenv.mkDerivation rec {
# invocations of gcc to clang, but vanilla clang doesn't
sed -i -e 's;_find_generic(repository_ctx, "gcc", "CC", overriden_tools);_find_generic(repository_ctx, "clang", "CC", overriden_tools);g' tools/cpp/unix_cc_configure.bzl
# This is necessary to avoid:
# "error: no visible @interface for 'NSDictionary' declares the selector
# 'initWithContentsOfURL:error:'"
# This can be removed when the apple_sdk is upgraded beyond 10.13+
sedVerbose tools/osx/xcode_locator.m \
-e '/initWithContentsOfURL:versionPlistUrl/ {
N
s/error:nil\];/\];/
}'
sed -i -e 's;"/usr/bin/libtool";_find_generic(repository_ctx, "libtool", "LIBTOOL", overriden_tools);g' tools/cpp/unix_cc_configure.bzl
wrappers=( tools/cpp/osx_cc_wrapper.sh.tpl )
for wrapper in "''${wrappers[@]}"; do
sed -i -e "s,/usr/bin/gcc,${stdenv.cc}/bin/clang,g" $wrapper
sed -i -e "s,/usr/bin/install_name_tool,${cctools}/bin/install_name_tool,g" $wrapper
sed -i -e "s,/usr/bin/xcrun install_name_tool,${cctools}/bin/install_name_tool,g" $wrapper
sedVerbose $wrapper \
-e "s,/usr/bin/xcrun install_name_tool,${cctools}/bin/install_name_tool,g"
done
'';
# -e "s,%{cc},${stdenv.cc}/bin/clang,g" \
genericPatches = ''
function sedVerbose() {
local path=$1; shift;
sed -i".bak-nix" "$path" "$@"
diff -U0 "$path.bak-nix" "$path" | sed "s/^/ /" || true
rm -f "$path.bak-nix"
}
# unzip builtins_bzl.zip so the contents get patched
builtins_bzl=src/main/java/com/google/devtools/build/lib/bazel/rules/builtins_bzl
unzip ''${builtins_bzl}.zip -d ''${builtins_bzl}_zip >/dev/null
@ -348,10 +347,7 @@ stdenv.mkDerivation rec {
grep -rlZ /bin/ \
src/main/java/com/google/devtools \
src/main/starlark/builtins_bzl/common/python \
tools/python \
tools/cpp \
tools/build_rules \
tools/osx/BUILD \
tools \
| while IFS="" read -r -d "" path; do
# If you add more replacements here, you must change the grep above!
# Only files containing /bin are taken into account.
@ -377,23 +373,24 @@ stdenv.mkDerivation rec {
# Passing EXTRA_BAZEL_ARGS is tricky due to quoting.
sedVerbose compile.sh \
-e "/bazel_build /a\ --copt=\"$(echo $NIX_CFLAGS_COMPILE | sed -e 's/ /" --copt=\"/g')\" \\\\" \
-e "/bazel_build /a\ --host_copt=\"$(echo $NIX_CFLAGS_COMPILE | sed -e 's/ /" --host_copt=\"/g')\" \\\\" \
-e "/bazel_build /a\ --linkopt=\"$(echo $(< ${stdenv.cc}/nix-support/libcxx-ldflags) | sed -e 's/ /" --linkopt=\"/g')\" \\\\" \
-e "/bazel_build /a\ --host_linkopt=\"$(echo $(< ${stdenv.cc}/nix-support/libcxx-ldflags) | sed -e 's/ /" --host_linkopt=\"/g')\" \\\\" \
-e "/bazel_build /a\ --linkopt=\"-Wl,$(echo $NIX_LDFLAGS | sed -e 's/ /" --linkopt=\"-Wl,/g')\" \\\\" \
-e "/bazel_build /a\ --host_linkopt=\"-Wl,$(echo $NIX_LDFLAGS | sed -e 's/ /" --host_linkopt=\"-Wl,/g')\" \\\\" \
-e "/bazel_build /a\ --verbose_failures \\\\" \
-e "/bazel_build /a\ --curses=no \\\\" \
-e "/bazel_build /a\ --features=-layering_check \\\\" \
-e "/bazel_build /a\ --experimental_strict_java_deps=off \\\\" \
-e "/bazel_build /a\ --strict_proto_deps=off \\\\" \
-e "/bazel_build /a\ --action_env=NIX_CFLAGS_COMPILE=\"$NIX_CFLAGS_COMPILE\" \\\\" \
-e "/bazel_build /a\ --action_env=NIX_LDFLAGS=\"$NIX_LDFLAGS\" \\\\" \
-e "/bazel_build /a\ --host_action_env=NIX_CFLAGS_COMPILE=\"$NIX_CFLAGS_COMPILE\" \\\\" \
-e "/bazel_build /a\ --host_action_env=NIX_LDFLAGS=\"$NIX_LDFLAGS\" \\\\" \
-e "/bazel_build /a\ --toolchain_resolution_debug='@bazel_tools//tools/jdk:(runtime_)?toolchain_type' \\\\" \
-e "/bazel_build /a\ --tool_java_runtime_version=local_jdk \\\\" \
-e "/bazel_build /a\ --java_runtime_version=local_jdk \\\\" \
-e "/bazel_build /a\ --repo_env=JAVA_HOME=${buildJdk}/${if isDarwin then "/zulu-11.jdk/Contents/Home" else "/lib/openjdk"} \\\\" \
-e "/bazel_build /a\ --extra_toolchains=@local_jdk//:all \\\\" \
-e "/bazel_build /a\ --toolchain_resolution_debug=@bazel_tools//tools/jdk:runtime_toolchain_type \\\\" \
-e "/bazel_build /a\ --sandbox_debug --verbose_failures \\\\" \
-e "/bazel_build /a\ --extra_toolchains=@bazel_tools//tools/jdk:all \\\\" \
-e "/bazel_build /a\ --distdir=${distDir} \\\\" \
#-e "/bazel_build /a\ --action_env=NIX_BINTOOLS=\"$NIX_BINTOOLS\" \\\\" \
#-e "/bazel_build /a\ --action_env=NIX_CC=\"$NIX_CC\" \\\\" \
#-e "/bazel_build /a\ --action_env=nativeBuildInputs=\"$nativeBuildInputs\" \\\\" \
# Also build parser_deploy.jar with bootstrap bazel
# TODO: Turn into a proper patch
@ -401,17 +398,6 @@ stdenv.mkDerivation rec {
-e 's!bazel_build !bazel_build src/tools/execlog:parser_deploy.jar !' \
-e 's!clear_log!cp $(get_bazel_bin_path)/src/tools/execlog/parser_deploy.jar output\nclear_log!'
# This is necessary to avoid:
# "error: no visible @interface for 'NSDictionary' declares the selector
# 'initWithContentsOfURL:error:'"
# This can be removed when the apple_sdk is upgraded beyond 10.13+
sedVerbose tools/osx/xcode_locator.m \
-e '/initWithContentsOfURL:versionPlistUrl/ {
N
s/error:nil\];/\];/
}'
# append the PATH with defaultShellPath in tools/bash/runfiles/runfiles.bash
echo "PATH=\$PATH:${defaultShellPath}" >> runfiles.bash.tmp
cat tools/bash/runfiles/runfiles.bash >> runfiles.bash.tmp
@ -427,7 +413,31 @@ stdenv.mkDerivation rec {
patchShebangs . >/dev/null
'';
in
lib.optionalString isDarwin darwinPatches + genericPatches;
''
function sedVerbose() {
local path=$1; shift;
sed -i".bak-nix" "$path" "$@"
diff -U0 "$path.bak-nix" "$path" | sed "s/^/ /" || true
rm -f "$path.bak-nix"
}
''
+ lib.optionalString stdenv.hostPlatform.isDarwin darwinPatches
+ genericPatches;
meta = with lib; {
homepage = "https://github.com/bazelbuild/bazel/";
description = "Build tool that builds code quickly and reliably";
sourceProvenance = with sourceTypes; [
fromSource
binaryBytecode # source bundles dependencies as jars
];
license = licenses.asl20;
maintainers = lib.teams.bazel.members;
inherit platforms;
};
# Bazel starts a local server and needs to bind a local address.
__darwinAllowLocalNetworking = true;
buildInputs = [ buildJdk ] ++ defaultShellUtils;
@ -444,9 +454,9 @@ stdenv.mkDerivation rec {
] ++ lib.optionals (stdenv.isDarwin) [
cctools
libcxx
Foundation
CoreFoundation
CoreServices
Foundation
];
# Bazel makes extensive use of symlinks in the WORKSPACE.

13
pkgs/development/tools/build-managers/bazel/bazel_7/extra_toolchains_precedence.patch Normal file
View File

@ -0,0 +1,13 @@
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/RegisteredToolchainsFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/RegisteredToolchainsFunction.java
index 2b407202a3..7591e78f85 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/RegisteredToolchainsFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/RegisteredToolchainsFunction.java
@@ -88,7 +88,7 @@ public class RegisteredToolchainsFunction implements SkyFunction {
try {
targetPatternBuilder.addAll(
TargetPatternUtil.parseAllSigned(
- platformConfiguration.getExtraToolchains(), mainRepoParser));
+ platformConfiguration.getExtraToolchains().reverse(), mainRepoParser));
} catch (InvalidTargetPatternException e) {
throw new RegisteredToolchainsFunctionException(
new InvalidToolchainLabelException(e), Transience.PERSISTENT);

27
pkgs/development/tools/build-managers/bazel/bazel_7/java_toolchain.patch
View File

@ -1,20 +1,29 @@
diff --git a/tools/jdk/BUILD.tools b/tools/jdk/BUILD.tools
index a8af76e90c..af2540f838 100644
index a8af76e90c..7f8b030f63 100644
--- a/tools/jdk/BUILD.tools
+++ b/tools/jdk/BUILD.tools
@@ -146,6 +146,16 @@ py_test(
@@ -146,6 +146,25 @@ py_test(
],
)
+##### Nonprebuilt toolchains definitions for NixOS and nix build sandboxes ####
+
+load("@rules_java//toolchains:default_java_toolchain.bzl", "default_java_toolchain", "NONPREBUILT_TOOLCHAIN_CONFIGURATION")
+
+default_java_toolchain(
+ name = "nonprebuilt_toolchain_java11",
+ configuration = NONPREBUILT_TOOLCHAIN_CONFIGURATION,
+ java_runtime = "@local_jdk//:jdk",
+ source_version = "11",
+ target_version = "11",
+)
+[
+ default_java_toolchain(
+ name = "nonprebuilt_toolchain_java" + str(version),
+ configuration = NONPREBUILT_TOOLCHAIN_CONFIGURATION,
+ java_runtime = "@local_jdk//:jdk",
+ source_version = str(version),
+ target_version = str(version),
+ )
+ # Ideally we would only define toolchains for the java versions that the
+ # local jdk supports. But we cannot access this information in a BUILD
+ # file, and this is a hack anyway, so just pick a large enough upper bound.
+ # At the current pace, java <= 30 should cover all realeases until 2028.
+ for version in range(8, 31)
+]
+
#### Aliases to rules_java to keep backward-compatibility (begin) ####

5
pkgs/development/tools/build-managers/bazel/bazel_7/no-arc.patch
View File

@ -2,15 +2,14 @@ diff --git a/tools/osx/BUILD b/tools/osx/BUILD
index 0358fb0ffe..baae1bf65b 100644
--- a/tools/osx/BUILD
+++ b/tools/osx/BUILD
@@ -27,9 +27,9 @@ exports_files([
@@ -27,9 +27,8 @@ exports_files([
])
DARWIN_XCODE_LOCATOR_COMPILE_COMMAND = """
- /usr/bin/xcrun --sdk macosx clang -mmacosx-version-min=10.13 -fobjc-arc -framework CoreServices \
- -framework Foundation -arch arm64 -arch x86_64 -Wl,-no_adhoc_codesign -Wl,-no_uuid -o $@ $< && \
- env -i codesign --identifier $@ --force --sign - $@
+ /usr/bin/xcrun --sdk macosx clang -mmacosx-version-min=10.13 -framework CoreServices \
+ -framework Foundation -Wl,-no_uuid -o $@ $< && \
+ /usr/bin/xcrun --sdk macosx clang -framework CoreServices -framework Foundation -Wl,-no_uuid -o $@ $< && \
+ /usr/bin/env -i /usr/bin/codesign --identifier $@ --force --sign - $@
"""

40
pkgs/development/tools/build-managers/bazel/bazel_7/patched_sources.nix Normal file
View File

@ -0,0 +1,40 @@
{ lib
, stdenv
, version
, src
, sourceRoot
, substituteAll
, defaultShellPath
, bazelRC
, enableNixHacks
, coreutils
, CoreFoundation
, CoreServices
, Foundation
, IOKit
, libcxx
, cctools
, sigtool
, bash
, python3
, buildJdk
, unzip
, zip
, ...
}:
stdenv.mkDerivation {
name = "bazel-patched-sources";
inherit version;
inherit src;
dontBuild = true;
nativeBuildInputs = [ zip unzip ];
installPhase = ''
mkdir $out
cp -r . $out
'';
}

3
pkgs/development/tools/build-managers/bazel/bazel_7/tests.nix
View File

@ -2,6 +2,7 @@
# tooling
, fetchFromGitHub
, newScope
, recurseIntoAttrs
, runCommandCC
, stdenv
# inputs
@ -12,6 +13,7 @@
, repoCache
, runJdk
, xe
, bazel-watcher
}:
let
inherit (stdenv.hostPlatform) isDarwin;
@ -112,7 +114,6 @@ let
inherit Foundation;
extraBazelArgs = ''
--repository_cache=${repoCache} \
--repo_env=JAVA_HOME=${runJdk}${if isDarwin then "/zulu-17.jdk/Contents/Home" else "/lib/openjdk"} \
'';
bazel = bazel_self;
};

157
pkgs/development/tools/build-managers/bazel/bazel_7/toolchain_better_debug.patch Normal file
View File

@ -0,0 +1,157 @@
diff --git a/src/main/java/com/google/devtools/build/lib/analysis/platform/DeclaredToolchainInfo.java b/src/main/java/com/google/devtools/build/lib/analysis/platform/DeclaredToolchainInfo.java
index fa322cf6f0..4a94087a3c 100644
--- a/src/main/java/com/google/devtools/build/lib/analysis/platform/DeclaredToolchainInfo.java
+++ b/src/main/java/com/google/devtools/build/lib/analysis/platform/DeclaredToolchainInfo.java
@@ -47,6 +47,9 @@ public abstract class DeclaredToolchainInfo implements TransitiveInfoProvider {
/** The label of the toolchain to resolve for use in toolchain-aware rules. */
public abstract Label toolchainLabel();
+ /** The label of the toolchain native rule invocation, for debugging. */
+ public abstract Label label();
+
/** Builder class to assist in creating {@link DeclaredToolchainInfo} instances. */
public static class Builder {
private ToolchainTypeInfo toolchainType;
@@ -55,6 +58,7 @@ public abstract class DeclaredToolchainInfo implements TransitiveInfoProvider {
private ImmutableList.Builder<ConfigMatchingProvider> targetSettings =
new ImmutableList.Builder<>();
private Label toolchainLabel;
+ private Label label;
/** Sets the type of the toolchain being declared. */
@CanIgnoreReturnValue
@@ -100,6 +104,13 @@ public abstract class DeclaredToolchainInfo implements TransitiveInfoProvider {
return this;
}
+ /** Sets the label of the toolchain to resolve for use in toolchain-aware rules. */
+ @CanIgnoreReturnValue
+ public Builder label(Label label) {
+ this.label = label;
+ return this;
+ }
+
/** Returns the newly created {@link DeclaredToolchainInfo} instance. */
public DeclaredToolchainInfo build() throws DuplicateConstraintException {
// Handle constraint duplication in attributes separately, so they can be reported correctly.
@@ -128,7 +139,8 @@ public abstract class DeclaredToolchainInfo implements TransitiveInfoProvider {
execConstraints,
targetConstraints,
targetSettings.build(),
- toolchainLabel);
+ toolchainLabel,
+ label);
}
}
diff --git a/src/main/java/com/google/devtools/build/lib/rules/platform/Toolchain.java b/src/main/java/com/google/devtools/build/lib/rules/platform/Toolchain.java
index 16d14a5edd..93ad4e47ec 100644
--- a/src/main/java/com/google/devtools/build/lib/rules/platform/Toolchain.java
+++ b/src/main/java/com/google/devtools/build/lib/rules/platform/Toolchain.java
@@ -57,6 +57,7 @@ public class Toolchain implements RuleConfiguredTargetFactory {
.collect(toImmutableList());
Label toolchainLabel =
ruleContext.attributes().get(ToolchainRule.TOOLCHAIN_ATTR, BuildType.NODEP_LABEL);
+ Label label = ruleContext.getLabel();
DeclaredToolchainInfo registeredToolchain;
try {
@@ -67,6 +68,7 @@ public class Toolchain implements RuleConfiguredTargetFactory {
.addTargetConstraints(targetConstraints)
.addTargetSettings(targetSettings)
.toolchainLabel(toolchainLabel)
+ .label(label)
.build();
} catch (DeclaredToolchainInfo.DuplicateConstraintException e) {
if (e.execConstraintsException() != null) {
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/SingleToolchainResolutionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/SingleToolchainResolutionFunction.java
index 933c200471..5c61ebcb12 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/SingleToolchainResolutionFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/SingleToolchainResolutionFunction.java
@@ -182,11 +182,12 @@ public class SingleToolchainResolutionFunction implements SkyFunction {
if (!nonmatchingSettings.isEmpty()) {
debugMessage(
eventHandler,
- " Type %s: %s platform %s: Rejected toolchain %s; mismatching config settings: %s",
+ " Type %s: %s platform %s: Rejected toolchain %s (from %s); mismatching config settings: %s",
toolchainType.toolchainType(),
"target",
targetPlatform.label(),
toolchain.toolchainLabel(),
+ toolchain.label(),
String.join(", ", nonmatchingSettings));
continue;
}
@@ -198,7 +199,8 @@ public class SingleToolchainResolutionFunction implements SkyFunction {
"target",
targetPlatform,
toolchainType,
- toolchain.toolchainLabel())) {
+ toolchain.toolchainLabel(),
+ toolchain.label())) {
continue;
}
@@ -208,12 +210,13 @@ public class SingleToolchainResolutionFunction implements SkyFunction {
if (platformKeysSeen.contains(executionPlatformKey)) {
debugMessage(
eventHandler,
- " Type %s: target platform %s: execution platform %s: Skipping toolchain %s;"
+ " Type %s: target platform %s: execution platform %s: Skipping toolchain %s (from %s);"
+ " execution platform already has selected toolchain",
toolchainType.toolchainType(),
targetPlatform.label(),
executionPlatformKey.getLabel(),
- toolchain.toolchainLabel());
+ toolchain.toolchainLabel(),
+ toolchain.label());
continue;
}
@@ -224,17 +227,19 @@ public class SingleToolchainResolutionFunction implements SkyFunction {
"execution",
executionPlatform,
toolchainType,
- toolchain.toolchainLabel())) {
+ toolchain.toolchainLabel(),
+ toolchain.label())) {
continue;
}
debugMessage(
eventHandler,
- " Type %s: target platform %s: execution %s: Selected toolchain %s",
+ " Type %s: target platform %s: execution %s: Selected toolchain %s (from %s)",
toolchainType.toolchainType(),
targetPlatform.label(),
executionPlatformKey.getLabel(),
- toolchain.toolchainLabel());
+ toolchain.toolchainLabel(),
+ toolchain.label());
builder.put(executionPlatformKey, toolchain.toolchainLabel());
platformKeysSeen.add(executionPlatformKey);
}
@@ -276,7 +281,8 @@ public class SingleToolchainResolutionFunction implements SkyFunction {
String platformType,
PlatformInfo platform,
ToolchainTypeRequirement toolchainType,
- Label toolchainLabel) {
+ Label toolchainLabel,
+ Label label) {
// Check every constraint_setting in either the toolchain or the platform.
ImmutableSet<ConstraintSettingInfo> mismatchSettings =
@@ -315,11 +321,12 @@ public class SingleToolchainResolutionFunction implements SkyFunction {
}
debugMessage(
eventHandler,
- " Type %s: %s platform %s: Rejected toolchain %s%s%s",
+ " Type %s: %s platform %s: Rejected toolchain %s (from %s)%s%s",
toolchainType.toolchainType(),
platformType,
platform.label(),
toolchainLabel,
+ label,
mismatchValues,
missingSettings);
}

271
pkgs/development/tools/build-managers/bazel/bazel_7/toolchain_group_debug.patch Normal file
View File

@ -0,0 +1,271 @@
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/SingleToolchainResolutionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/SingleToolchainResolutionFunction.java
index 933c200471..1e5bbaa44b 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/SingleToolchainResolutionFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/SingleToolchainResolutionFunction.java
@@ -32,7 +32,6 @@ import com.google.devtools.build.lib.analysis.platform.PlatformInfo;
import com.google.devtools.build.lib.analysis.platform.ToolchainTypeInfo;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.events.Event;
-import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.server.FailureDetails.Toolchain.Code;
import com.google.devtools.build.lib.skyframe.ConfiguredTargetKey;
import com.google.devtools.build.lib.skyframe.toolchains.PlatformLookupUtil.InvalidPlatformException;
@@ -91,14 +90,21 @@ public class SingleToolchainResolutionFunction implements SkyFunction {
.debugToolchainResolution(key.toolchainType().toolchainType());
// Find the right one.
- return resolveConstraints(
+ ArrayList<String> trace = debug ? new ArrayList<>() : null;
+ SingleToolchainResolutionValue toolchainResolution = resolveConstraints(
key.toolchainType(),
key.toolchainTypeInfo(),
key.availableExecutionPlatformKeys(),
key.targetPlatformKey(),
toolchains.registeredToolchains(),
env,
- debug ? env.getListener() : null);
+ trace);
+
+ if (debug) {
+ env.getListener().handle(Event.info(String.join("\n", trace)));
+ }
+
+ return toolchainResolution;
}
/**
@@ -114,7 +120,7 @@ public class SingleToolchainResolutionFunction implements SkyFunction {
ConfiguredTargetKey targetPlatformKey,
ImmutableList<DeclaredToolchainInfo> toolchains,
Environment env,
- @Nullable EventHandler eventHandler)
+ @Nullable List<String> trace)
throws ToolchainResolutionFunctionException, InterruptedException {
// Load the PlatformInfo needed to check constraints.
@@ -151,10 +157,20 @@ public class SingleToolchainResolutionFunction implements SkyFunction {
toolchain.toolchainType().typeLabel().equals(toolchainType.toolchainType()))
.collect(toImmutableList());
+ debugMessage(
+ trace,
+ "Performing resolution of %s for target platform %s",
+ toolchainType.toolchainType(),
+ targetPlatform.label());
+
for (DeclaredToolchainInfo toolchain : filteredToolchains) {
// Make sure the target setting matches but watch out for resolution errors.
ArrayList<String> nonmatchingSettings = new ArrayList<>();
ArrayList<String> errors = new ArrayList<>();
+ debugMessage(
+ trace,
+ " toolchain %s",
+ toolchain.toolchainLabel());
// TODO(blaze-configurability-team): If this pattern comes up more often, add a central
// facility for merging multiple MatchResult
@@ -181,11 +197,8 @@ public class SingleToolchainResolutionFunction implements SkyFunction {
}
if (!nonmatchingSettings.isEmpty()) {
debugMessage(
- eventHandler,
- " Type %s: %s platform %s: Rejected toolchain %s; mismatching config settings: %s",
- toolchainType.toolchainType(),
- "target",
- targetPlatform.label(),
+ trace,
+ " Rejected toolchain %s; mismatching config settings: %s",
toolchain.toolchainLabel(),
String.join(", ", nonmatchingSettings));
continue;
@@ -193,35 +206,36 @@ public class SingleToolchainResolutionFunction implements SkyFunction {
// Make sure the target platform matches.
if (!checkConstraints(
- eventHandler,
+ trace,
toolchain.targetConstraints(),
- "target",
+ " Rejected toolchain %1s%3s",
targetPlatform,
toolchainType,
toolchain.toolchainLabel())) {
continue;
}
+
+ debugMessage(
+ trace,
+ " Toolchain %s is compatible with target plaform, searching for execution platforms:",
+ toolchain.toolchainLabel());
// Find the matching execution platforms.
for (ConfiguredTargetKey executionPlatformKey : availableExecutionPlatformKeys) {
// Only check the toolchains if this is a new platform.
if (platformKeysSeen.contains(executionPlatformKey)) {
debugMessage(
- eventHandler,
- " Type %s: target platform %s: execution platform %s: Skipping toolchain %s;"
- + " execution platform already has selected toolchain",
- toolchainType.toolchainType(),
- targetPlatform.label(),
- executionPlatformKey.getLabel(),
- toolchain.toolchainLabel());
+ trace,
+ " Skipping execution platform %s; it has already selected a toolchain",
+ executionPlatformKey.getLabel());
continue;
}
PlatformInfo executionPlatform = platforms.get(executionPlatformKey);
if (!checkConstraints(
- eventHandler,
+ trace,
toolchain.execConstraints(),
- "execution",
+ " Incompatible execution platform %2s%3s",
executionPlatform,
toolchainType,
toolchain.toolchainLabel())) {
@@ -229,51 +243,64 @@ public class SingleToolchainResolutionFunction implements SkyFunction {
}
debugMessage(
- eventHandler,
- " Type %s: target platform %s: execution %s: Selected toolchain %s",
- toolchainType.toolchainType(),
- targetPlatform.label(),
- executionPlatformKey.getLabel(),
- toolchain.toolchainLabel());
+ trace,
+ " Compatible execution platform %s",
+ executionPlatformKey.getLabel());
builder.put(executionPlatformKey, toolchain.toolchainLabel());
platformKeysSeen.add(executionPlatformKey);
}
}
ImmutableMap<ConfiguredTargetKey, Label> resolvedToolchainLabels = builder.buildOrThrow();
- if (resolvedToolchainLabels.isEmpty()) {
- debugMessage(
- eventHandler,
- " Type %s: target platform %s: No toolchains found.",
- toolchainType.toolchainType(),
- targetPlatform.label());
+ if (trace != null) {
+ if (resolvedToolchainLabels.isEmpty()) {
+ debugMessage(
+ trace,
+ "No %s toolchain found for target platform %s.",
+ toolchainType.toolchainType(),
+ targetPlatform.label());
+ } else {
+ debugMessage(
+ trace,
+ "Recap of selected %s toolchains for target platform %s:",
+ toolchainType.toolchainType(),
+ targetPlatform.label());
+ resolvedToolchainLabels.forEach((executionPlatformKey, toolchainLabel) ->
+ debugMessage(
+ trace,
+ " Selected toolchain %s to run on exec platform %s",
+ toolchainLabel,
+ executionPlatformKey.getLabel())
+ );
+ }
}
return SingleToolchainResolutionValue.create(toolchainTypeInfo, resolvedToolchainLabels);
}
/**
- * Helper method to print a debugging message, if the given {@link EventHandler} is not {@code
+ * Helper method to print a debugging message, if the given {@link trace} is not {@code
* null}.
*/
@FormatMethod
private static void debugMessage(
- @Nullable EventHandler eventHandler, @FormatString String template, Object... args) {
- if (eventHandler == null) {
+ @Nullable List<String> trace, @FormatString String template, Object... args) {
+ if (trace == null) {
return;
}
- eventHandler.handle(Event.info("ToolchainResolution: " + String.format(template, args)));
+ trace.add(String.format(template, args));
}
/**
* Returns {@code true} iff all constraints set by the toolchain and in the {@link PlatformInfo}
* match.
*/
+ @SuppressWarnings("FormatStringAnnotation")
private static boolean checkConstraints(
- @Nullable EventHandler eventHandler,
+ @Nullable List<String> trace,
ConstraintCollection toolchainConstraints,
- String platformType,
+ String debugMessage,
PlatformInfo platform,
ToolchainTypeRequirement toolchainType,
Label toolchainLabel) {
@@ -295,7 +322,7 @@ public class SingleToolchainResolutionFunction implements SkyFunction {
.filter(toolchainConstraints::hasWithoutDefault)
.collect(ImmutableSet.toImmutableSet());
- if (!mismatchSettingsWithDefault.isEmpty()) {
+ if (trace != null && !mismatchSettingsWithDefault.isEmpty()) {
String mismatchValues =
mismatchSettingsWithDefault.stream()
.filter(toolchainConstraints::has)
@@ -314,14 +341,11 @@ public class SingleToolchainResolutionFunction implements SkyFunction {
missingSettings = "; missing: " + missingSettings;
}
debugMessage(
- eventHandler,
- " Type %s: %s platform %s: Rejected toolchain %s%s%s",
- toolchainType.toolchainType(),
- platformType,
- platform.label(),
+ trace,
+ debugMessage,
toolchainLabel,
- mismatchValues,
- missingSettings);
+ platform.label(),
+ mismatchValues + missingSettings);
}
return mismatchSettingsWithDefault.isEmpty();
diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/ToolchainResolutionFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/ToolchainResolutionFunction.java
index 3bdf2d0b23..fea848eca0 100644
--- a/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/ToolchainResolutionFunction.java
+++ b/src/main/java/com/google/devtools/build/lib/skyframe/toolchains/ToolchainResolutionFunction.java
@@ -80,14 +80,7 @@ public class ToolchainResolutionFunction implements SkyFunction {
Preconditions.checkNotNull(configuration.getFragment(PlatformConfiguration.class));
// Check if debug output should be generated.
- boolean debug =
- key.debugTarget()
- || configuration
- .getFragment(PlatformConfiguration.class)
- .debugToolchainResolution(
- key.toolchainTypes().stream()
- .map(ToolchainTypeRequirement::toolchainType)
- .collect(toImmutableSet()));
+ boolean debug = key.debugTarget();
// Load the configured target for the toolchain types to ensure that they are valid and
// resolve aliases.
@@ -138,9 +131,10 @@ public class ToolchainResolutionFunction implements SkyFunction {
.handle(
Event.info(
String.format(
- "ToolchainResolution: Target platform %s: Selected execution platform %s,"
+ "ToolchainResolution: Target platform %s (%s): Selected execution platform %s, "
+ " %s",
unloadedToolchainContext.targetPlatform().label(),
+ configuration,
unloadedToolchainContext.executionPlatform().label(),
selectedToolchains)));
}

16
pkgs/development/tools/build-managers/bazel/java-test.nix
View File

@ -7,7 +7,6 @@
, lib
, openjdk8
, jdk11_headless
, jdk17_headless
, runLocal
, runtimeShell
, writeScript
@ -46,19 +45,16 @@ let
inherit workspaceDir;
bazelPkg = bazel;
buildInputs = [
#(if lib.strings.versionOlder bazel.version "5.0.0" then openjdk8
#else if lib.strings.versionOlder bazel.version "7.0.0" then jdk11_headless
#else jdk17_headless)
jdk17_headless
(if lib.strings.versionOlder bazel.version "5.0.0" then openjdk8 else jdk11_headless)
];
bazelScript = ''
${bazel}/bin/bazel \
run \
--announce_rc \
--toolchain_resolution_debug='@bazel_tools//tools/jdk:(runtime_)?toolchain_type' \
--distdir=${distDir} \
--verbose_failures \
--curses=no \
--sandbox_debug \
--strict_java_deps=off \
//:ProjectRunner \
'' + lib.optionalString (lib.strings.versionOlder bazel.version "5.0.0") ''
@ -68,6 +64,14 @@ let
'' + extraBazelArgs;
};
# --repo_env=JAVA_HOME=${jdk11_headless}/${if stdenv.hostPlatform.isDarwin then "/zulu-17.jdk/Contents/Home" else "/lib/openjdk"} \
#--java_language_version=17 \
#--java_language_version=17 \
#--java_runtime_version=local_jdk \
# --java_language_version=11 \
# --tool_java_runtime_version=local_jdk_17 \
# --tool_java_language_version=17 \
#--java_runtime_version=local_jdk_11 \
in
testBazel