Stats: Merge stats interface with http[s]

Stats interface is also a http[s] interface, before it was required to
configure as a separate interface. This change ease configuration of
interfaces and still allows configuring stats interface on
separate address/port.

devel/config-single.yaml

@@ -20,9 +20,10 @@ interfaces:
   - type:    http
     address: 0.0.0.0
     port:    2826
-  - type: stats
+  - type: http
     address: 0.0.0.0
     port: 9111
+    stats: true
 
 consensus:
   block_interval:

doc/config.example.yaml

@@ -75,9 +75,10 @@ interfaces:
     address: 0.0.0.0 # Any node can bind - default value
     # Port on which we bind
     port:    2826    # 0xB0A, default value
-  - type: stats
+  - type: http
     address: 0.0.0.0
     port: 9110
+    stats: true # Enable stats server on this interface
   - type: tcp
     address: 0.0.0.0
     port: 9220

source/agora/node/Config.d

@@ -30,7 +30,7 @@ import vibe.inet.url;
 import configy.Read;
 
 import std.algorithm.iteration : splitter;
-import std.algorithm.searching : all, count;
+import std.algorithm.searching : all, any;
 import std.exception;
 import std.getopt;
 import std.traits : hasUnsharedAliasing;
@@ -144,8 +144,8 @@ public struct Config
 
         if (this.consensus.quorum_threshold < 1 || this.consensus.quorum_threshold > 100)
             throw new Exception("consensus.quorum_threshold is a percentage and must be between 1 and 100, included");
-        if (this.interfaces.count!(intf => intf.type == InterfaceConfig.Type.stats) > 1)
-            throw new Exception("Can only configure a single stats interface");
+        if (this.interfaces.any!(intf => intf.type == InterfaceConfig.Type.tcp && intf.stats))
+            throw new Exception("TCP interfaces cannot have 'stats' enabled");
     }
 }
 
@@ -165,7 +165,6 @@ public struct InterfaceConfig
         http = 0,
         https = 1,
         tcp = 2,
-        stats = 3,
     }
 
     /// Ditto
@@ -180,6 +179,9 @@ public struct InterfaceConfig
     /// Proxy Protocol V1
     public bool proxy_proto;
 
+    /// Enable StatsServer through this interface
+    public bool stats;
+
     /// Default values when none is given in the config file
     private static immutable InterfaceConfig[Type.max] Default = [
         // Publicly enabled by default

source/agora/node/FullNode.d

@@ -53,7 +53,6 @@ import agora.serialization.Serializer;
 import agora.stats.App;
 import agora.stats.Block;
 import agora.stats.EndpointReq;
-import agora.stats.Server;
 import agora.stats.Tx;
 import agora.stats.Utils;
 import agora.stats.Validator;
@@ -171,18 +170,6 @@ public class FullNode : API
     /// Ditto
     protected ManagedDatabase cacheDB;
 
-    /***************************************************************************
-
-        Stats-related fields
-
-        Those fields are used to expose internal statistics about the node on
-        an HTTP interface that is ultimately queried by a Prometheus server.
-        They may be unused if the stats interface is not enabled in the config.
-
-    ***************************************************************************/
-
-    protected StatsServer stats_server;
-
     /// Ditto
     protected ApplicationStats app_stats;
 
@@ -464,7 +451,6 @@ public class FullNode : API
         }
 
         this.timers[TimersIdx.ClockTick] = this.clock.start(this.taskman);
-        this.stats_server = this.makeStatsServer();
         this.transaction_relayer.start();
 
         this.registry.start();
@@ -726,11 +712,6 @@ public class FullNode : API
         log.info("Shutting down..");
         this.taskman.logStats();
 
-        // The stats server is standalone, but accepts network
-        // requests, hence why we shut it down early on.
-        if (this.stats_server !is null)
-            this.stats_server.shutdown();
-
         // Shut down our timers (discovery, catchup)
         foreach (timer; this.timers)
             timer.stop();
@@ -803,15 +784,6 @@ public class FullNode : API
                 config.consensus);
     }
 
-    /// Returns a newly constructed StatsServer
-    protected StatsServer makeStatsServer ()
-    {
-        auto stat_intf = this.config.interfaces.find!(intf => intf.type == InterfaceConfig.Type.stats);
-        if (stat_intf.empty)
-            return null;
-        return new StatsServer(stat_intf.front.address, stat_intf.front.port);
-    }
-
     /// Returns: The Logger to use for this class
     protected Logger makeLogger ()
     {

source/agora/node/Runner.d

@@ -27,8 +27,9 @@ import agora.node.Config;
 import agora.node.FullNode;
 import agora.node.Registry;
 import agora.node.Validator;
-import agora.serialization.Serializer;
 import agora.script.Engine;
+import agora.serialization.Serializer;
+import agora.stats.Utils;
 import agora.utils.Log;
 
 import vibe.core.core;
@@ -90,7 +91,9 @@ public void runNode (Config config, ref Listeners result)
 
     const bool hasHTTPInterface = config.interfaces.any!(
         i => i.type.among(InterfaceConfig.Type.http, InterfaceConfig.Type.https));
+    const bool hasStatsInterface = config.interfaces.any!(i => i.stats);
     URLRouter router;
+    URLRouter stats_router;
     RestInterfaceSettings settings;
     if (hasHTTPInterface)
     {
@@ -99,6 +102,19 @@ public void runNode (Config config, ref Listeners result)
         settings.errorHandler = toDelegate(&restErrorHandler);
     }
 
+    void handle_metrics (
+        scope HTTPServerRequest, scope HTTPServerResponse res)
+    {
+        res.writeBody(cast(const(ubyte[])) Utils.getCollectorRegistry().collect(),
+                      "text/plain");
+    }
+
+    if (hasStatsInterface)
+    {
+        stats_router = new URLRouter();
+        stats_router.get("/metrics", &handle_metrics);
+    }
+
     if (config.validator.enabled)
     {
         log.trace("Started Validator...");
@@ -108,13 +124,17 @@ public void runNode (Config config, ref Listeners result)
         result.node = inst;
         if (hasHTTPInterface)
             router.registerRestInterface!(agora.api.Validator.API)(inst, settings);
+        if (hasStatsInterface)
+            stats_router.registerRestInterface!(agora.api.Validator.API)(inst, settings);
     }
     else
     {
         log.trace("Started FullNode...");
         result.node = new FullNode(config);
         if (hasHTTPInterface)
             router.registerRestInterface!(agora.api.FullNode.API)(result.node, settings);
+        if (hasStatsInterface)
+            stats_router.registerRestInterface!(agora.api.FullNode.API)(result.node, settings);
     }
 
     if (config.flash.enabled)
@@ -127,7 +147,11 @@ public void runNode (Config config, ref Listeners result)
             config.node.data_dir, params.Genesis.hashFull(), new Engine(),
             result.node.getTaskManager(), &result.node.postTransaction,
             &result.node.getBlock, &result.node.getNetworkManager().makeRegistryClient);
-        router.registerRestInterface!FlashAPI(flash, settings);
+        if (hasHTTPInterface)
+            router.registerRestInterface!FlashAPI(flash, settings);
+        if (hasStatsInterface)
+            stats_router.registerRestInterface!FlashAPI(flash, settings);
+
         result.flash = flash;
     }
 
@@ -155,6 +179,8 @@ public void runNode (Config config, ref Listeners result)
         assert(reg !is null);
         if (hasHTTPInterface)
             router.registerRestInterface(reg, settings);
+        if (hasStatsInterface)
+            stats_router.registerRestInterface(reg, settings);
         /* auto dnstask = */ runTask(() => runDNSServer(config.registry, reg));
         result.tcp ~= listenTCP(config.registry.port, (conn) => conn.runTCPDNSServer(reg),
                 config.registry.address);
@@ -201,9 +227,19 @@ public void runNode (Config config, ref Listeners result)
                     (tls_user_help, interface_.address, httpsettings.port));
             httpsettings.tlsContext = tls_ctx;
         }
+
+        // The following correspond to your scraping interval in Prometheus
+        // The default value for Prometheus is 1 minute:
+        // https://prometheus.io/docs/prometheus/latest/configuration/configuration/
+        // We set it to 70 seconds to avoid a bit of jitter.
+        // See https://github.com/bosagora/agora/issues/2380 and the linked Vibe.d
+        // issue for short-comings of this approach.
+        if (interface_.stats)
+            httpsettings.keepAliveTimeout = 70.seconds;
+
         log.info("Node is listening on interface: http{}://{}:{}",
             interface_.type == InterfaceConfig.Type.https ? "s" : "" , interface_.address, interface_.port);
-        result.http ~= listenHTTP(httpsettings, router);
+        result.http ~= listenHTTP(httpsettings, interface_.stats ? stats_router : router);
     }
 
     // also register the FlashControlAPI