Implement command database generation (i.e. compile_commands.json). (#…

…399)

This implements obtaining and generating compile commands from toolsets that compile C or C++ sources. I.e. implements both the --command-database=json amd --command-database-out=<filename> CLI options. Although it implements the toolset changes for most compilers, only a few are tested.

Fixes #395

.vscode/launch.json

@@ -93,7 +93,7 @@
 			"type": "cppdbg",
 			"request": "launch",
 			"program": "${workspaceFolder}/.build/gcc-13/debug/cxxstd-11-iso/threading-multi/b2",
-			"args": [],
+			"args": ["-d1", "--command-database=json", "b2"],
 			"stopAtEntry": false,
 			"cwd": "${workspaceFolder}",
 			"environment": [],

Jamroot.jam

@@ -22,7 +22,7 @@ import virtual-target ;
 
 path-constant SELF : . ;
 
-project b2
+project /bfgroup/b2
     : build-dir .build
     : requirements
         <cxxstd>11
@@ -34,6 +34,9 @@ project b2
         <toolset>clang-win:<define>_CRT_NONSTDC_NO_DEPRECATE=1
         <toolset>clang:<cxxflags>-Wno-deprecated-declarations
         <toolset>gcc,<variant>debug,<target-os>linux:<linkflags>-rdynamic
+        # Don't warn on ignored/unknown attributes.
+        <toolset>gcc:<cxxflags>-Wno-attributes
+        <toolset>clang:<cxxflags>-Wno-attributes
     ;
 
 #|

doc/src/history.adoc

@@ -3,6 +3,9 @@
 
 == Version 5.2.0
 
+* *New*: Add support for generating `compile_commands.json` command database
+  for some IDE integration.
+  -- _René Ferdinand Rivera Morell_
 * *New*: Addition of a module (`db`) for structured data management. Has a
   `property-db` class that can write out as JSON data.
   -- _René Ferdinand Rivera Morell_

doc/src/overview.adoc

@@ -626,8 +626,7 @@ B2 recognizes the following command line options.
 `-d0`::
   Suppress all informational messages.
 `-d N`::
-  Enable cumulative debugging levels from 1 to n. Values are:
-  +
+  Enable cumulative debugging levels from 1 to n. Values are: +
   1.  Show the actions taken for building targets, as they are executed
   (the default).
   2.  Show "quiet" actions and display all action text, as they are
@@ -653,6 +652,11 @@ B2 recognizes the following command line options.
 `-s var=value`::
   Set the variable `var` to `value` in the global scope of the jam language
   interpreter, overriding variables imported from the environment.
+`--command-database=_format_`::
+  Output a compile commands database as _format_. Currently _format_ can be:
+  `json`. (See xref:tasks#b2.tasks.commanddb[Command Database] for details.)
+`--command-database-out=_file_`::
+  Specify the _file_ path to output the commands database to.
 
 [[bbv2.overview.invocation.properties]]
 === Properties

doc/src/tasks.adoc

@@ -821,3 +821,50 @@ When loading a `*.jam` file as the _path_ it is equivalent to calling:
 In this case it means that the file will be loaded as part of the referenced
 project and hence any bare targets or information it declares will be part of
 the project.
+
+[[b2.tasks.commanddb]]
+== Command Database, and IDE Integration
+
+Many IDE programs accept the use of a
+https://clang.llvm.org/docs/JSONCompilationDatabase.html[`compile_commands.json`]
+file to learn what and how your project builds. B2 supports generating such
+files for any build you make. B2 supports this through a generic facility to
+extract commands from the actions it executes. There are two options that
+control this. The `--command-database=_format_` option indicates to generate the
+file for the given _format_. It has a couple of effects when specified:
+
+* It tells B2 to start observing and extracting commands from actions (as
+  specified by the toolset).
+* It disables execution of actions. I.e. equivalent to adding the `-n` option.
+* It enables building all default and specified targets. I.e. the equivalent to
+  adding the `-a` option.
+* It disables all action execution output. I.e. as if specifying `-d0` option.
+* At the end of the main build it writes out the results of what it observed
+  to the database file.
+
+Currently on `json` is supported as a format that follows the
+https://clang.llvm.org/docs/JSONCompilationDatabase.html[Clang JSON Compilation
+Database Format Specification].
+
+The `--command-database-out=_file_` option controls the name, and optionally
+location, of the generated file. By default the _file_ is
+`compile_commands.json` to follow the ecosystem convention. And it is generated,
+by default, in one of the following locations:
+
+* Relative to the `build-dir` of the root project, if it's specified by the
+  project. With the default _file_ name or as given.
+* At the absolute _file_ path if it is rooted.
+* At the _current working directory_.
+
+The following fields are populated in the generated database:
+
+* `directory` - This will always be the current directory as B2 makes all paths
+  relative to that (or absolute).
+* `file` - The first source of each action recorded.
+* `command` - The quoted, full, command as extracted by the toolset.
+* `output` - The first target file of each action recorded. As B2 can build
+  multiple variants at once this is required to differentiate between multiple
+  compilations of the same source file.
+
+NOTE: Only one command database file is generated per `b2` invocation. And each
+time it is generated it overwrites any previous such file.

src/build/project.jam

@@ -42,6 +42,7 @@ import print ;
 import property-set ;
 import regex ;
 import sequence ;
+import command-db ;
 
 
 .debug-loading = [ MATCH ^(--debug-loading)$ : [ modules.peek : ARGV ] ] ;
@@ -712,11 +713,12 @@ rule initialize (
 
     # load-parent can end up loading this module again. Make sure this is not
     # duplicated.
+    local attributes ;
     if ! $($(module-name).attributes)
     {
         $(module-name).attributes = [ new project-attributes $(location)
             $(module-name) ] ;
-        local attributes = $($(module-name).attributes) ;
+        attributes = $($(module-name).attributes) ;
 
         if $(location)
         {
@@ -791,6 +793,12 @@ rule initialize (
     }
 
     .current-project = [ target $(module-name) ] ;
+
+    if $(jamroot) && [ $(attributes).get build-dir ]
+    {
+        command-db.set-output-dir
+            [ path.native [ $(attributes).get build-dir ] ] ;
+    }
 }
 
 
@@ -1312,6 +1320,7 @@ module project-rules
     {
         import path ;
         import project ;
+        import command-db ;
 
         local caller = [ CALLER_MODULE ] ;
         local attributes = [ project.attributes $(caller) ] ;
@@ -1382,6 +1391,12 @@ module project-rules
                 }
             }
         }
+
+        if $(explicit-build-dir)
+        {
+            command-db.set-output-dir
+                [ path.native [ $(attributes).get build-dir ] ] ;
+        }
     }
 
     # Declare and set a project global constant. Project global constants are

src/engine/bindjam.cpp

@@ -21,6 +21,7 @@ Distributed under the Boost Software License, Version 1.0.
 #include "value.h"
 #include "variable.h"
 
+#include "mod_command_db.h"
 #include "mod_db.h"
 #include "mod_jam_builtin.h"
 #include "mod_jam_class.h"
@@ -823,7 +824,8 @@ void bind_jam(FRAME * f)
 		.bind(string_module())
 		.bind(sysinfo_module())
 		.bind(version_module())
-		.bind(db_module());
+		.bind(db_module())
+		.bind(command_db_module());
 }
 
 }} // namespace b2::jam

src/engine/build.bat

@@ -170,13 +170,15 @@ set B2_SOURCES=
 set B2_SOURCES=%B2_SOURCES% bindjam.cpp builtins.cpp class.cpp
 set B2_SOURCES=%B2_SOURCES% command.cpp compile.cpp constants.cpp cwd.cpp
 set B2_SOURCES=%B2_SOURCES% debug.cpp debugger.cpp
+set B2_SOURCES=%B2_SOURCES% events.cpp
 set B2_SOURCES=%B2_SOURCES% execcmd.cpp execnt.cpp execunix.cpp filent.cpp filesys.cpp fileunix.cpp frames.cpp function.cpp
 set B2_SOURCES=%B2_SOURCES% glob.cpp hash.cpp hcache.cpp hdrmacro.cpp headers.cpp jam.cpp
 set B2_SOURCES=%B2_SOURCES% jamgram.cpp lists.cpp make.cpp make1.cpp md5.cpp mem.cpp modules.cpp
 set B2_SOURCES=%B2_SOURCES% native.cpp option.cpp output.cpp parse.cpp pathnt.cpp
 set B2_SOURCES=%B2_SOURCES% pathsys.cpp pathunix.cpp regexp.cpp rules.cpp scan.cpp search.cpp jam_strings.cpp
 set B2_SOURCES=%B2_SOURCES% startup.cpp tasks.cpp
 set B2_SOURCES=%B2_SOURCES% timestamp.cpp value.cpp variable.cpp w32_getreg.cpp
+set B2_SOURCES=%B2_SOURCES% mod_command_db.cpp
 set B2_SOURCES=%B2_SOURCES% mod_db.cpp
 set B2_SOURCES=%B2_SOURCES% mod_jam_builtin.cpp
 set B2_SOURCES=%B2_SOURCES% mod_jam_class.cpp

src/engine/build.sh

@@ -448,6 +448,7 @@ constants.cpp \
 cwd.cpp \
 debug.cpp \
 debugger.cpp \
+events.cpp \
 execcmd.cpp \
 execnt.cpp \
 execunix.cpp \
@@ -487,6 +488,7 @@ timestamp.cpp \
 value.cpp \
 variable.cpp \
 w32_getreg.cpp \
+mod_command_db.cpp \
 mod_db.cpp \
 mod_jam_builtin.cpp \
 mod_jam_class.cpp \

src/engine/events.cpp

@@ -0,0 +1,123 @@
+/*
+Copyright 2024 René Ferdinand Rivera Morell
+Distributed under the Boost Software License, Version 1.0.
+(See accompanying file LICENSE.txt or https://www.bfgroup.xyz/b2/LICENSE.txt)
+*/
+
+#include "events.h"
+
+#include <algorithm>
+#include <memory>
+#include <set>
+#include <vector>
+
+namespace b2 {
+
+struct event_base
+{
+	event_tag tag;
+	int32_t priority;
+	uint64_t id;
+
+	inline bool operator<(const event_base & o) const
+	{
+		if (tag < o.tag) return true;
+		if ((0 - priority) < (0 - o.priority)) return true;
+		return id < o.id;
+	}
+};
+
+template <typename F>
+struct event : public event_base
+{
+	F call;
+};
+
+struct events
+{
+	struct ecmp
+	{
+		inline bool operator()(const event_base * a, const event_base * b) const
+		{
+			return (*a) < (*b);
+		}
+	};
+	std::set<const event_base *, ecmp> sorted_items;
+	std::vector<std::unique_ptr<event_base>> items;
+
+	static events & get()
+	{
+		static events e;
+		return e;
+	}
+
+	template <typename F>
+	uint64_t add(event_tag tag, F && call, int32_t priority)
+	{
+		uint64_t id = items.empty() ? 1 : items.back()->id + 1;
+		std::unique_ptr<event<F>> e(new event<F>);
+		e->tag = tag;
+		e->priority = priority;
+		e->id = id;
+		e->call = call;
+		sorted_items.insert(e.get());
+		items.push_back(std::move(e));
+		return id;
+	}
+
+	void remove(uint64_t e)
+	{
+		auto i = std::lower_bound(items.begin(), items.end(), e,
+			[](const std::unique_ptr<event_base> & a, uint64_t id) -> bool {
+				return a->id < id;
+			});
+		if (i != items.end() && (*i)->id == e)
+		{
+			items.erase(i);
+		}
+	}
+
+	template <typename... Args>
+	void trigger(event_tag tag, Args... args)
+	{
+		using E = event<std::function<void(Args...)>>;
+		static event_base x = { tag, std::numeric_limits<int32_t>::max(), 0 };
+		auto i = sorted_items.lower_bound(&x);
+		static event_base y = { tag, std::numeric_limits<int32_t>::min(), 0 };
+		auto j = sorted_items.lower_bound(&y);
+		if (j != sorted_items.end()) ++j;
+		for (; i != j; ++i)
+		{
+			if ((*i)->tag != tag) break;
+			static_cast<const E *>(*i)->call(args...);
+		}
+	}
+};
+
+void remove_event_callback(uint64_t e) { events::get().remove(e); }
+
+template <>
+uint64_t add_event_callback(
+	event_tag tag, std::function<void(TARGET *)> && call, int32_t priority)
+{
+	return events::get().add(tag, std::move(call), priority);
+}
+
+void trigger_event_pre_exec_cmd(TARGET * t)
+{
+	events::get().trigger<TARGET *>(event_tag::pre_exec_cmd, t);
+}
+
+template <>
+uint64_t add_event_callback(
+	event_tag tag, std::function<void(int)> && call, int32_t priority)
+{
+	return events::get().add(tag, std::move(call), priority);
+}
+
+void trigger_event_exit_main(int status)
+{
+	events::get().trigger<int>(event_tag::exit_main, status);
+}
+
+} // namespace b2

src/engine/events.h

@@ -0,0 +1,36 @@
+/*
+Copyright 2024 René Ferdinand Rivera Morell
+Distributed under the Boost Software License, Version 1.0.
+(See accompanying file LICENSE.txt or https://www.bfgroup.xyz/b2/LICENSE.txt)
+*/
+
+#ifndef B2_EVENTS_H
+#define B2_EVENTS_H
+
+#include "config.h"
+
+#include "rules.h"
+
+#include <cstdint>
+#include <functional>
+
+namespace b2 {
+
+enum class event_tag : uint16_t
+{
+	unknown = 0,
+	pre_exec_cmd,
+	exit_main
+};
+
+template <typename F>
+uint64_t add_event_callback(
+	event_tag tag, std::function<F> && call, int32_t priority = 0);
+void remove_event_callback(uint64_t e);
+
+void trigger_event_pre_exec_cmd(TARGET * t);
+void trigger_event_exit_main(int status);
+
+} // namespace b2
+
+#endif