Allow loggers to be configured and to write to files

Replicate what Ocean is doing, but reading a YAML file.

doc/config.example.yaml

@@ -153,12 +153,57 @@ dns:
 
 ################################################################################
 ##                               Logging options                              ##
+##                                                                            ##
+## Our logging system is hierarchical: Logger names are, e.g. `a.b.c.d`.      ##
+## Inside agora, every module that logs have a module-level logger matching   ##
+## it's module name, e.g. `agora.node.main`.                                  ##
+##                                                                            ##
+## In addition, some modules can have more nested loggers: one such use case  ##
+## is for a module which deals with client connections, which we'll refer to  ##
+## as `agora.network.Client` here. Such a module would produce a confusing    ##
+## output if it was logging all clients interactions at module level, because ##
+## such interactions are intertwinned and requests/responses would be hard to ##
+## follow. Hence, using a predictable identifier to extend the hierarchy,     ##
+## such as the public key (when dealing with validators), would lead to the   ##
+## following loggers: `agora.network.Client.GABC`,                            ##
+## `agora.network.Client.GEFG`, `agora.network.Client.G1234`, etc...          ##
+##                                                                            ##
+## When configuring loggers, the configuration applies to the referenced      ##
+## hierarchy and any child. Using the previous example, configuring           ##
+## `agora.network` will lead to all clients having the same configuration,    ##
+## as well as the module `agora.network.Foobar`.                              ##
+##                                                                            ##
+## The 'root' name allows to configure the parent of all other loggers.       ##
 ################################################################################
 logging:
-  # Values: Trace, Info, Warn, Error, Fatal, None (default)
-  level: None
+  root:
+    # Set the log level for the root logger.
+    # This is the default log level, and is overriden by more specialized configs
+    #
+    # Values: Trace, Info, Warn, Error, Fatal, None (default)
+    level: Info
+    # Whether or not to log output to the console
+    console: true
+    # Output file to write the logging output to
+    # Note that output of a more specialized logger that uses another file won't be
+    # written to this file.
+    # The path is relative to `data_dir` unless an absolute path is supplied.
+    # Intermediate directories will be created as needed.
+    # This setting is optional, as no file would be written to if empty / not supplied.
+    file: log/root.log
 
-################################################################################
+  # Nested logger configuration
+  # Order does not matter as long as there is no duplication
+  - name: agora.network
+    level: Trace
+    console: false
+    file: log/network.log
+  - name: agora.node
+    level: Trace
+    console: false
+    file: log/node.log
+
+ ################################################################################
 ##                               Event Handlers                               ##
 ################################################################################
 event_handlers:

source/agora/common/Config.d

@@ -113,7 +113,7 @@ public struct Config
     public immutable string[] dns_seeds;
 
     /// Logging config
-    public LoggingConfig logging;
+    public immutable(LoggerConfig)[] logging;
 
     /// Event handler config
     public EventHandlerConfig event_handlers;
@@ -244,16 +244,6 @@ public struct AdminConfig
     public ushort port = 0xB0B;
 }
 
-/// Configuration for logging
-public struct LoggingConfig
-{
-    static assert(!hasUnsharedAliasing!(typeof(this)),
-        "Type must be shareable accross threads");
-
-    /// The logging level
-    LogLevel level = LogLevel.Error;
-}
-
 /// Configuration for URLs to push a data when an event occurs
 public struct EventHandlerConfig
 {
@@ -740,11 +730,42 @@ private BanManager.Config parseBanManagerConfig (Node* node, in CommandLine cmdl
 
 *******************************************************************************/
 
-private LoggingConfig parseLoggingSection (Node* ptr, in CommandLine c)
+private immutable(LoggerConfig)[] parseLoggingSection (Node* ptr, in CommandLine)
 {
-    LoggingConfig ret;
-    ret.level = get!(LogLevel, "logging", "level")(c, ptr);
-    return ret;
+    // By default, configure the root logger to be verbose enough for users
+    // to see what's happening
+    static immutable LoggerConfig[] DefaultConfig = [ {
+        name: null,
+        level: LogLevel.Info,
+        propagate: true,
+        console: true,
+        additive: true,
+    } ];
+
+    if (ptr is null)
+        return DefaultConfig;
+
+    immutable(LoggerConfig)[] result;
+    foreach (string name_, Node value; *ptr)
+    {
+        LoggerConfig c = { name: name_ != "root" ? name_ : null };
+        if (auto p = "level" in value)
+            c.level = (*p).as!string.to!LogLevel;
+        if (auto p = "propagate" in value)
+            c.propagate = (*p).as!bool;
+        if (auto p = "console" in value)
+            c.console = (*p).as!bool;
+        if (auto p = "additive" in value)
+            c.additive = (*p).as!bool;
+        if (auto p = "file" in value)
+            c.file = (*p).as!string;
+        if (auto p = "buffer_size" in value)
+            c.buffer_size = (*p).as!size_t;
+
+        result ~= c;
+    }
+
+    return result.length ? result : DefaultConfig;
 }
 
 ///
@@ -758,30 +779,31 @@ unittest
 foo:
   bar: Useless
 logging:
-  level: Trace
+  root:
+    level: Trace
+  agora.network:
+    level: Error
 `;
         auto node = Loader.fromString(conf_example).load();
         auto config = parseLoggingSection("logging" in node, cmdln);
-        assert(config.level == LogLevel.Trace);
-
-        cmdln.overrides["logging.level"] = [ "None" ];
-        auto config2 = parseLoggingSection("logging" in node, cmdln);
-        assert(config2.level == LogLevel.None);
+        assert(config[0].name.length == 0);
+        assert(config[0].level == LogLevel.Trace);
+        assert(config[1].name == "agora.network");
+        assert(config[1].level == LogLevel.Error);
     }
 
     {
         CommandLine cmdln;
         immutable conf_example = `
-logging:
+noLoggingSectionHere:
   foo: bar
 `;
         auto node = Loader.fromString(conf_example).load();
         auto config = parseLoggingSection("logging" in node, cmdln);
-        assert(config.level == LogLevel.Error);
-
-        cmdln.overrides["logging.level"] = [ "Trace" ];
-        auto config2 = parseLoggingSection("logging" in node, cmdln);
-        assert(config2.level == LogLevel.Trace);
+        assert(config.length == 1);
+        assert(config[0].name.length == 0);
+        assert(config[0].level == LogLevel.Info);
+        assert(config[0].console == true);
     }
 }
 

source/agora/node/Runner.d

@@ -61,8 +61,13 @@ public alias Listeners = Tuple!(
 
 public Listeners runNode (Config config)
 {
-    setVibeLogLevel(config.logging.level);
-    Log.root.level(config.logging.level, true);
+    foreach (const ref settings; config.logging)
+    {
+        if (settings.name.length == 0 || settings.name == "vibe")
+            setVibeLogLevel(settings.level);
+        configureLogger(settings);
+    }
+
     auto log = Logger(__MODULE__);
     log.trace("Config is: {}", config);
 

source/agora/utils/Log.d

@@ -137,16 +137,70 @@ public alias LogLevel = Ocean.Level;
 /// Ditto
 public alias Log = Ocean.Log;
 
-/// Initialize the logger
-static this ()
+/// Define options to configure a Logger
+/// Loosely inspired from `ocean.utils.log.Config`
+public struct LoggerConfig
 {
-    version (unittest)
-        auto appender = CircularAppender!()();
-    else
+    /// Name of the logger to configure
+    public string name;
+
+    /// Level to set the logger to (messages at a lower level won't be printed)
+    public Ocean.Level level;
+
+    /// Whether to propagate that level to the children
+    public bool propagate;
+
+    /// Whether to use console output or not
+    public bool console;
+
+    /// Whether to use file output and if, which file path
+    public string file;
+
+    /// Whether this logger should be additive or not
+    public bool additive;
+
+    /// Buffer size of the buffer output
+    public size_t buffer_size = 16_384;
+}
+
+/***************************************************************************
+
+    Configure a Logger based on the provided `settings`
+
+    Params:
+        settings = Configuration to apply
+
+***************************************************************************/
+
+public void configureLogger (in LoggerConfig settings)
+{
+	auto log = settings.name ? Log.lookup(settings.name) : Log.root;
+
+    if (settings.buffer_size)
+        log.buffer(new char[](settings.buffer_size));
+
+    log.clear();
+
+	// if console/file/syslog is specifically set, don't inherit other
+    // appenders (unless we have been specifically asked to be additive)
+	log.additive = settings.additive ||
+        !(settings.console || settings.file.length);
+
+    if (settings.console)
+    {
         auto appender = new AppendConsole();
+        appender.layout(new AgoraLayout());
+        log.add(appender);
+    }
 
-    appender.layout(new AgoraLayout());
-    Log.root.add(appender);
+    if (settings.file.length)
+    {
+        auto appender = new PhobosFileAppender(settings.file);
+        appender.layout(new AgoraLayout());
+        log.add(appender);
+    }
+
+    log.level(settings.level, settings.propagate);
 }
 
 /// Circular appender which appends to an internal buffer
@@ -277,6 +331,49 @@ unittest
     assert(get_log_output("0123456789") == "3456789");
 }
 
+/// A file appender that uses Phobos
+public class PhobosFileAppender : Appender
+{
+    /// Mask
+    private Mask mask_;
+
+    ///
+    private File file;
+
+    ///
+    public this (string path)
+    {
+        import std.file, std.path;
+        path.dirName.mkdirRecurse();
+        this.file = File(path, "a");
+    }
+
+    /// Returns: the name of this class
+    public override istring name ()
+    {
+        return this.classinfo.name;
+    }
+
+    /// Return the fingerprint for this class
+    public final override Mask mask ()
+    {
+        return this.mask_;
+    }
+
+    /// Append an event to the buffer
+    public final override void append (LogEvent event)
+    {
+        scope writer = this.file.lockingTextWriter();
+        this.layout.format(event,
+            (cstring content)
+            {
+                formattedWrite(writer, content);
+            });
+        version (Windows) writer.put("\r\n");
+        else              writer.put("\n");
+    }
+}
+
 /// A layout with colored LogLevel
 public class AgoraLayout : Appender.Layout
 {

tests/system/node/0/config.yaml

@@ -60,5 +60,5 @@ network:
 ##                               Logging options                              ##
 ################################################################################
 logging:
-  # Values: Trace, Info, Warn, Error, Fatal, None (default)
-  level: Info
+  root:
+    level: Info

tests/system/node/2/config.yaml

@@ -66,5 +66,5 @@ network:
 ##                               Logging options                              ##
 ################################################################################
 logging:
-  # Values: Trace, Info, Warn, Error, Fatal, None (default)
-  level: Info
+  root:
+    level: Info

tests/system/node/3/config.yaml

@@ -66,5 +66,5 @@ network:
 ##                               Logging options                              ##
 ################################################################################
 logging:
-  # Values: Trace, Info, Warn, Error, Fatal, None (default)
-  level: Info
+  root:
+    level: Info

tests/system/node/4/config.yaml

@@ -66,5 +66,5 @@ network:
 ##                               Logging options                              ##
 ################################################################################
 logging:
-  # Values: Trace, Info, Warn, Error, Fatal, None (default)
-  level: Info
+  config:
+    level: Info

tests/system/node/5/config.yaml

@@ -66,5 +66,5 @@ network:
 ##                               Logging options                              ##
 ################################################################################
 logging:
-  # Values: Trace, Info, Warn, Error, Fatal, None (default)
-  level: Info
+  config:
+    level: Info

tests/system/node/6/config.yaml

@@ -66,5 +66,5 @@ network:
 ##                               Logging options                              ##
 ################################################################################
 logging:
-  # Values: Trace, Info, Warn, Error, Fatal, None (default)
-  level: Info
+  config:
+    level: Info

tests/system/node/7/config.yaml

@@ -66,5 +66,5 @@ network:
 ##                               Logging options                              ##
 ################################################################################
 logging:
-  # Values: Trace, Info, Warn, Error, Fatal, None (default)
-  level: Info
+  config:
+    level: Info