Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: provisional parallelization support #1202

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

WIP: provisional parallelization support #1202

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions foreign_cc/boost_build.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ def _create_configure_script(configureParameters):
"##copy_dir_contents_to_dir## $$EXT_BUILD_ROOT$$/{}/. .".format(root),
"chmod -R +w .",
"##enable_tracing##",
"##enable_parallel_build##",
"./bootstrap.sh {}".format(" ".join(ctx.attr.bootstrap_options)),
"./b2 install {} --prefix=.".format(" ".join(user_options)),
"##disable_tracing##",
Expand Down
1 change: 1 addition & 0 deletions foreign_cc/built_tools/private/built_tools_framework.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ def built_tool_rule_impl(ctx, script_lines, out_dir, mnemonic, additional_tools
]

script.append("##enable_tracing##")
script.append("##enable_parallel_build##")
script.extend(script_lines)
script.append("##disable_tracing##")

Expand Down
1 change: 1 addition & 0 deletions foreign_cc/private/cmake_script.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ def create_cmake_script(
directory = "$$EXT_BUILD_ROOT$$/" + root

script.append("##enable_tracing##")
script.append("##enable_parallel_build##")

# Configure the CMake generate command
cmake_prefixes = [cmake_prefix] if cmake_prefix else []
Expand Down
1 change: 1 addition & 0 deletions foreign_cc/private/configure_script.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ def create_configure_script(

script.append("##export_var## MAKE {}".format(make_path))
script.append("##enable_tracing##")
script.append("##enable_parallel_build##")

if autogen:
# NOCONFIGURE is pseudo standard and tells the script to not invoke configure.
Expand Down
4 changes: 4 additions & 0 deletions foreign_cc/private/framework/toolchains/commands.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,10 @@ PLATFORM_COMMANDS = {
arguments = [_argument_info(name = "text", data_type = type(""), doc = "Text to output")],
doc = "Outputs 'text' to stdout",
),
"enable_parallel_build": _command_info(
arguments = [],
doc = "Enable parallization (auto-detect cpu count and set MAKEFLAGS and CMAKE_BUILD_PARALLEL_LEVEL appropriately)",
),
"enable_tracing": _command_info(
arguments = [],
doc = "Enable script tracing. eg: `set -x`",
Expand Down
5 changes: 5 additions & 0 deletions foreign_cc/private/framework/toolchains/freebsd_commands.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ def enable_tracing():
def disable_tracing():
return "set +x"

def enable_parallel_build():
# XXX not implemented
return ":"

def mkdirs(path):
return "mkdir -p " + path

Expand Down Expand Up @@ -282,6 +286,7 @@ commands = struct(
define_sandbox_paths = define_sandbox_paths,
disable_tracing = disable_tracing,
echo = echo,
enable_parallel_build = enable_parallel_build,
enable_tracing = enable_tracing,
env = env,
export_var = export_var,
Expand Down
8 changes: 8 additions & 0 deletions foreign_cc/private/framework/toolchains/linux_commands.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,13 @@ def enable_tracing():
def disable_tracing():
return "set +x"

def enable_parallel_build():
return """
_cpu_count=$(nproc)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a slightly naiive way of getting a cpu account as nproc is not cgroup aware AFAIK.
When running under RBE its likely that you will be constrained in a cgroup and so this added parallelism there will likely actually result in slower builds in those environments. I've yet to see a good way of interrogating cgroups to work out how many processes you actually want to spawn in a reliable way.

export MAKEFLAGS="-j$_cpu_count${MAKEFLAGS:+ }${MAKEFLAGS:-}"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd also suggest setting -l$_cpu_count here just to be a good citizen to the machine as bazel has only assigned a single core to the action in its scheduling and so limiting the parallelism under load is a useful thing to do.
I'm not familiar enough with CMake's parallelism config to know if there's something equivalent there?

export CMAKE_BUILD_PARALLEL_LEVEL="$_cpu_count"
"""

def mkdirs(path):
return "mkdir -p " + path

Expand Down Expand Up @@ -264,6 +271,7 @@ commands = struct(
define_sandbox_paths = define_sandbox_paths,
disable_tracing = disable_tracing,
echo = echo,
enable_parallel_build = enable_parallel_build,
enable_tracing = enable_tracing,
env = env,
export_var = export_var,
Expand Down
8 changes: 8 additions & 0 deletions foreign_cc/private/framework/toolchains/macos_commands.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,13 @@ def enable_tracing():
def disable_tracing():
return "set +x"

def enable_parallel_build():
return """
_cpu_count=$(sysctl -n hw.ncpu)
export MAKEFLAGS="-j$_cpu_count${MAKEFLAGS:+ }${MAKEFLAGS:-}"
export CMAKE_BUILD_PARALLEL_LEVEL="$_cpu_count"
"""

def mkdirs(path):
return "mkdir -p " + path

Expand Down Expand Up @@ -277,6 +284,7 @@ commands = struct(
define_sandbox_paths = define_sandbox_paths,
disable_tracing = disable_tracing,
echo = echo,
enable_parallel_build = enable_parallel_build,
enable_tracing = enable_tracing,
env = env,
export_var = export_var,
Expand Down
5 changes: 5 additions & 0 deletions foreign_cc/private/framework/toolchains/windows_commands.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ def enable_tracing():
def disable_tracing():
return "set +x"

def enable_parallel_build():
# XXX not implemented
return ":"

def mkdirs(path):
return "mkdir -p " + path

Expand Down Expand Up @@ -281,6 +285,7 @@ commands = struct(
define_sandbox_paths = define_sandbox_paths,
disable_tracing = disable_tracing,
echo = echo,
enable_parallel_build = enable_parallel_build,
enable_tracing = enable_tracing,
env = env,
export_var = export_var,
Expand Down
1 change: 1 addition & 0 deletions foreign_cc/private/make_script.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ def create_make_script(
script.append("##symlink_contents_to_dir## $$EXT_BUILD_ROOT$$/{} $$BUILD_TMPDIR$$ False".format(root))

script.append("##enable_tracing##")
script.append("##enable_parallel_build##")
configure_vars = get_make_env_vars(workspace_name, tools, flags, env_vars, deps, inputs, make_commands)
script.extend(["{env_vars} {command}".format(
env_vars = configure_vars,
Expand Down
8 changes: 8 additions & 0 deletions test/cmake_text_tests.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,7 @@ def _merge_flag_values_no_toolchain_file_test(ctx):
export CXX="/usr/bin/gcc"
export CXXFLAGS="foo=\\\"bar\\\" -Fbat"
##enable_tracing##
##enable_parallel_build##
emcmake cmake -DCMAKE_AR="/usr/bin/ar" -DCMAKE_BUILD_TYPE="RelWithDebInfo" -DCMAKE_INSTALL_PREFIX="test_rule" -DCMAKE_PREFIX_PATH="$$EXT_BUILD_DEPS$$" -DCMAKE_RANLIB="" -G 'Unix Makefiles' $$EXT_BUILD_ROOT$$/external/test_rule
##disable_tracing##
"""
Expand Down Expand Up @@ -315,6 +316,7 @@ export CFLAGS="-U_FORTIFY_SOURCE -fstack-protector -Wall"
export CXXFLAGS="-U_FORTIFY_SOURCE -fstack-protector -Wall"
export ASMFLAGS="-U_FORTIFY_SOURCE -fstack-protector -Wall"
##enable_tracing##
##enable_parallel_build##
cmake -DCMAKE_AR="/usr/bin/ar" -DCMAKE_SHARED_LINKER_FLAGS="-shared -fuse-ld=gold" -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=gold -Wl -no-as-needed" -DNOFORTRAN="on" -DCMAKE_BUILD_TYPE="Debug" -DCMAKE_INSTALL_PREFIX="test_rule" -DCMAKE_PREFIX_PATH="$$EXT_BUILD_DEPS$$;/abc/def" -DCMAKE_RANLIB="" --debug-output -Wdev -G 'Ninja' $$EXT_BUILD_ROOT$$/external/test_rule
##disable_tracing##
"""
Expand Down Expand Up @@ -371,6 +373,7 @@ export CFLAGS="-U_FORTIFY_SOURCE -fstack-protector -Wall"
export CXXFLAGS="-U_FORTIFY_SOURCE -fstack-protector -Wall"
export ASMFLAGS="-U_FORTIFY_SOURCE -fstack-protector -Wall"
##enable_tracing##
##enable_parallel_build##
cmake -DCMAKE_AR="/usr/bin/ar" -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=gold -Wl -no-as-needed" -DCMAKE_BUILD_TYPE="Debug" -DCMAKE_INSTALL_PREFIX="test_rule" -DCMAKE_PREFIX_PATH="$$EXT_BUILD_DEPS$$;/abc/def" -DCMAKE_RANLIB="" --debug-output -Wdev -G 'Ninja' $$EXT_BUILD_ROOT$$/external/test_rule
##disable_tracing##
"""
Expand Down Expand Up @@ -437,6 +440,7 @@ set(CMAKE_SHARED_LINKER_FLAGS_INIT "$$__var_CMAKE_SHARED_LINKER_FLAGS_INIT$$")
EOF

##enable_tracing##
##enable_parallel_build##
cmake -DNOFORTRAN="on" -DCMAKE_TOOLCHAIN_FILE="$$BUILD_TMPDIR$$/crosstool_bazel.cmake" -DCMAKE_BUILD_TYPE="Debug" -DCMAKE_INSTALL_PREFIX="test_rule" -DCMAKE_PREFIX_PATH="$$EXT_BUILD_DEPS$$" -DCMAKE_RANLIB="" --debug-output -Wdev -G 'Ninja' $$EXT_BUILD_ROOT$$/external/test_rule
##disable_tracing##
"""
Expand Down Expand Up @@ -504,6 +508,7 @@ export CXXFLAGS="--quoted=\\\"abc def\\\" --sysroot=/abc/sysroot --gcc_toolchain
export ASMFLAGS="assemble assemble-user"
export CUSTOM_ENV="YES"
##enable_tracing##
##enable_parallel_build##
cmake -DCMAKE_AR="/cxx_linker_static" -DCMAKE_CXX_LINK_EXECUTABLE="became" -DCMAKE_SHARED_LINKER_FLAGS="shared1 shared2" -DCMAKE_EXE_LINKER_FLAGS="executable" -DCMAKE_BUILD_TYPE="user_type" -DCUSTOM_CACHE="YES" -DCMAKE_INSTALL_PREFIX="test_rule" -DCMAKE_PREFIX_PATH="$$EXT_BUILD_DEPS$$" -DCMAKE_RANLIB="" --debug-output -Wdev -G 'Ninja' $$EXT_BUILD_ROOT$$/external/test_rule
##disable_tracing##
"""
Expand Down Expand Up @@ -571,6 +576,7 @@ export CXXFLAGS="--quoted=\\\"abc def\\\" --sysroot=/abc/sysroot --gcc_toolchain
export ASMFLAGS="assemble assemble-user"
export CUSTOM_ENV="YES"
##enable_tracing##
##enable_parallel_build##
cmake -DCMAKE_AR="/cxx_linker_static" -DCMAKE_CXX_LINK_EXECUTABLE="became" -DCMAKE_SHARED_LINKER_FLAGS="shared1 shared2" -DCMAKE_EXE_LINKER_FLAGS="executable" -DCMAKE_BUILD_TYPE="user_type" -DCUSTOM_CACHE="YES" -DCMAKE_INSTALL_PREFIX="test_rule" -DCMAKE_PREFIX_PATH="$$EXT_BUILD_DEPS$$" -DCMAKE_RANLIB="" -DANDROID="YES" -DCMAKE_SYSTEM_NAME="Linux" -DCMAKE_SYSTEM_PROCESSOR="x86_64" --debug-output -Wdev -G 'Ninja' $$EXT_BUILD_ROOT$$/external/test_rule
##disable_tracing##
"""
Expand Down Expand Up @@ -638,6 +644,7 @@ export CXXFLAGS="--quoted=\\\"abc def\\\" --sysroot=/abc/sysroot --gcc_toolchain
export ASMFLAGS="assemble assemble-user"
export CUSTOM_ENV="YES"
##enable_tracing##
##enable_parallel_build##
cmake -DCMAKE_AR="/cxx_linker_static" -DCMAKE_CXX_LINK_EXECUTABLE="became" -DCMAKE_SHARED_LINKER_FLAGS="shared1 shared2" -DCMAKE_EXE_LINKER_FLAGS="executable" -DCMAKE_BUILD_TYPE="user_type" -DCUSTOM_CACHE="YES" -DCMAKE_INSTALL_PREFIX="test_rule" -DCMAKE_PREFIX_PATH="$$EXT_BUILD_DEPS$$" -DCMAKE_RANLIB="" -DCMAKE_SYSTEM_NAME="Linux" -DCMAKE_SYSTEM_PROCESSOR="aarch64" --debug-output -Wdev -G 'Ninja' $$EXT_BUILD_ROOT$$/external/test_rule
##disable_tracing##
"""
Expand Down Expand Up @@ -726,6 +733,7 @@ EOF

export CUSTOM_ENV="YES"
##enable_tracing##
##enable_parallel_build##
cmake -DCUSTOM_CACHE="YES" -DCMAKE_TOOLCHAIN_FILE="$$BUILD_TMPDIR$$/crosstool_bazel.cmake" -DCMAKE_BUILD_TYPE="Debug" -DCMAKE_INSTALL_PREFIX="test_rule" -DCMAKE_PREFIX_PATH="$$EXT_BUILD_DEPS$$" -DCMAKE_RANLIB="" --debug-output -Wdev -G 'Ninja' $$EXT_BUILD_ROOT$$/external/test_rule
##disable_tracing##
"""
Expand Down