Skip to content

Commit

Permalink
GEOMESA-3407 Micrometer - support easier config overrides (#3222)
Browse files Browse the repository at this point in the history
  • Loading branch information
elahrvivaz authored Oct 25, 2024
1 parent add94b9 commit b7f5d62
Show file tree
Hide file tree
Showing 5 changed files with 162 additions and 32 deletions.
8 changes: 5 additions & 3 deletions docs/user/appendix/metrics.rst
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ Configuration should be under the key ``geomesa.metrics``, and takes the followi
::

geomesa.metrics = {
reporters = []
reporters = {}
instrumentations = {
# jvm classloader metrics
classloader = {
Expand Down Expand Up @@ -232,7 +232,8 @@ Prometheus

::

{
# note: the top-level key here is only for uniqueness - it can be any string
"prometheus" = {
type = "prometheus"
enabled = true
# use prometheus "standard" names - see https://docs.micrometer.io/micrometer/reference/implementations/prometheus.html#_the_prometheus_rename_filter
Expand All @@ -255,7 +256,8 @@ Cloudwatch

::

{
# note: the top-level key here is only for uniqueness - it can be any string
"cloudwatch" = {
type = "cloudwatch"
enabled = true
namespace = "geomesa"
Expand Down
12 changes: 6 additions & 6 deletions docs/user/process.rst
Original file line number Diff line number Diff line change
Expand Up @@ -481,9 +481,9 @@ the result as a json object.
========== ===========
Parameters Description
========== ===========
features The data source feature collection to query. Reference as ``store:layername``.
For an XML file enter ``<wfs:Query typeName=store:layername />``
For interactive WPS request builder select ``VECTOR_LAYER`` & choose ``store:layername``
features The data source feature collection to query. Reference as ``workspace:layername``.
For an XML file enter ``<wfs:Query typeName=workspace:layername />``
For interactive WPS request builder select ``VECTOR_LAYER`` & choose ``workspace:layername``

filter The filter to apply to the feature collection.
For an XML file enter:
Expand Down Expand Up @@ -565,9 +565,9 @@ which are returned as a json object.
=========== ===========
Parameters Description
=========== ===========
features The data source feature collection to query. Reference as ``store:layername``.
For an XML file enter ``<wfs:Query typeName=store:layername />``
For interactive WPS request builder select ``VECTOR_LAYER`` & choose ``store:layername``
features The data source feature collection to query. Reference as ``workspace:layername``.
For an XML file enter ``<wfs:Query typeName=workspace:layername />``
For interactive WPS request builder select ``VECTOR_LAYER`` & choose ``workspace:layername``

attribute The attribute for which unique values will be extracted. Attributes are expressed as a string.
For an XML file enter ``<wps:LiteralData>attribute-name</wps:LiteralData>``
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import org.locationtech.geomesa.metrics.micrometer.prometheus.PrometheusFactory
import pureconfig.{ConfigReader, ConfigSource}

import java.util.Locale
import scala.util.Try
import scala.util.control.NonFatal

object MicrometerSetup extends StrictLogging {

Expand All @@ -35,20 +37,16 @@ object MicrometerSetup extends StrictLogging {
*
* @param conf conf
*/
// noinspection ScalaUnusedSymbol
def configure(conf: Config = ConfigFactory.load()): Unit = synchronized {
if (conf.hasPath(ConfigPath)) {
implicit val r0: ConfigReader[Instrumentation] = deriveReader[Instrumentation]
implicit val r1: ConfigReader[JvmInstrumentations] = deriveReader[JvmInstrumentations]
implicit val r2: ConfigReader[MetricsConfig] = deriveReader[MetricsConfig]
implicit val r3: ConfigReader[RegistryConfig] = deriveReader[RegistryConfig]
val metricsConfig = ConfigSource.fromConfig(conf.getConfig(ConfigPath)).loadOrThrow[MetricsConfig]
metricsConfig.registries.foreach { registryConfig =>
implicit val r0: ConfigReader[RegistryConfig] = deriveReader[RegistryConfig]
val MetricsConfig(registries, instrumentations) = MetricsConfig(conf.getConfig(ConfigPath))
registries.foreach { case (_, registryConfig) =>
val config = ConfigSource.fromConfig(registryConfig).loadOrThrow[RegistryConfig]
if (config.enabled) {
val configString = registryConfig.root().render(ConfigRenderOptions.concise())
if (registries.contains(config.`type`)) {
val existing = registries(config.`type`)
if (this.registries.contains(config.`type`)) {
val existing = this.registries(config.`type`)
if (existing != configString) {
throw new IllegalArgumentException(
s"Registry type ${config.`type`} already registered with a different configuration:" +
Expand All @@ -58,34 +56,34 @@ object MicrometerSetup extends StrictLogging {
val registry = createRegistry(registryConfig)
sys.addShutdownHook(registry.close())
Metrics.addRegistry(registry)
registries.put(config.`type`, configString)
this.registries.put(config.`type`, configString)
}
}
}

if (metricsConfig.instrumentations.classloader.enabled && bindings.add("classloader")) {
if (instrumentations.classloader.enabled && bindings.add("classloader")) {
logger.debug("Enabling JVM classloader metrics")
val tags = metricsConfig.instrumentations.classloader.tags.map { case (k, v) => Tag.of(k, v) }
val tags = instrumentations.classloader.tags.map { case (k, v) => Tag.of(k, v) }
new ClassLoaderMetrics(tags.asJava).bindTo(Metrics.globalRegistry)
}
if (metricsConfig.instrumentations.memory.enabled && bindings.add("memory")) {
if (instrumentations.memory.enabled && bindings.add("memory")) {
logger.debug("Enabling JVM memory metrics")
val tags = metricsConfig.instrumentations.memory.tags.map { case (k, v) => Tag.of(k, v) }
val tags = instrumentations.memory.tags.map { case (k, v) => Tag.of(k, v) }
new JvmMemoryMetrics(tags.asJava).bindTo(Metrics.globalRegistry)
}
if (metricsConfig.instrumentations.gc.enabled && bindings.add("gc")) {
if (instrumentations.gc.enabled && bindings.add("gc")) {
logger.debug("Enabling JVM garbage collection metrics")
val tags = metricsConfig.instrumentations.gc.tags.map { case (k, v) => Tag.of(k, v) }
val tags = instrumentations.gc.tags.map { case (k, v) => Tag.of(k, v) }
new JvmGcMetrics(tags.asJava).bindTo(Metrics.globalRegistry)
}
if (metricsConfig.instrumentations.processor.enabled && bindings.add("processor")) {
if (instrumentations.processor.enabled && bindings.add("processor")) {
logger.debug("Enabling JVM processor metrics")
val tags = metricsConfig.instrumentations.processor.tags.map { case (k, v) => Tag.of(k, v) }
val tags = instrumentations.processor.tags.map { case (k, v) => Tag.of(k, v) }
new ProcessorMetrics(tags.asJava).bindTo(Metrics.globalRegistry)
}
if (metricsConfig.instrumentations.threads.enabled && bindings.add("threads")) {
if (instrumentations.threads.enabled && bindings.add("threads")) {
logger.debug("Enabling JVM thread metrics")
val tags = metricsConfig.instrumentations.threads.tags.map { case (k, v) => Tag.of(k, v) }
val tags = instrumentations.threads.tags.map { case (k, v) => Tag.of(k, v) }
new JvmThreadMetrics(tags.asJava).bindTo(Metrics.globalRegistry)
}
}
Expand All @@ -107,14 +105,38 @@ object MicrometerSetup extends StrictLogging {
}
}

private case class MetricsConfig(
case class MetricsConfig(
registries: Map[String, Config],
instrumentations: JvmInstrumentations,
)

object MetricsConfig {
// noinspection ScalaUnusedSymbol
def apply(conf: Config): MetricsConfig = {
implicit val r0: ConfigReader[Instrumentation] = deriveReader[Instrumentation]
implicit val r1: ConfigReader[JvmInstrumentations] = deriveReader[JvmInstrumentations]
implicit val r2: ConfigReader[MetricsConfig] = deriveReader[MetricsConfig]
val source = ConfigSource.fromConfig(conf)
try { source.loadOrThrow[MetricsConfig] } catch {
case NonFatal(e) =>
// back compatible loading attempt
implicit val r: ConfigReader[MetricsSeqConfig] = deriveReader[MetricsSeqConfig]
val c = Try(source.loadOrThrow[MetricsSeqConfig]).getOrElse(throw e)
val registries = Seq.tabulate(c.registries.length)(i => String.valueOf(i)).zip(c.registries).toMap
MetricsConfig(registries, c.instrumentations)
}
}
}

// back compatible structure
private case class MetricsSeqConfig(
registries: Seq[Config],
instrumentations: JvmInstrumentations,
)

private case class Instrumentation(enabled: Boolean = false, tags: Map[String, String] = Map.empty)
case class Instrumentation(enabled: Boolean = false, tags: Map[String, String] = Map.empty)

private case class JvmInstrumentations(
case class JvmInstrumentations(
classloader: Instrumentation = Instrumentation(),
memory: Instrumentation = Instrumentation(),
gc: Instrumentation = Instrumentation(),
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
/***********************************************************************
* Copyright (c) 2013-2024 Commonwealth Computer Research, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Apache License, Version 2.0
* which accompanies this distribution and is available at
* http://www.opensource.org/licenses/apache2.0.php.
***********************************************************************/

package org.locationtech.geomesa.metrics.micrometer

import com.typesafe.config.ConfigFactory
import org.junit.runner.RunWith
import org.locationtech.geomesa.metrics.micrometer.MicrometerSetup.MetricsConfig
import org.specs2.mutable.Specification
import org.specs2.runner.JUnitRunner

@RunWith(classOf[JUnitRunner])
class MetricsConfigTest extends Specification {

"MetricsConfig" should {
"parse configs" in {
val reg =
"""
|{
| type = "prometheus"
| enabled = false
| rename = true
| common-tags = { "application" = "test" }
| port = 9090
|}
|""".stripMargin
val conf = ConfigFactory.parseString(
s"""{
| enabled = true
| instrumentations = {
| classloader = { enabled = true, tags = {} }
| memory = { enabled = true, tags = {} }
| gc = { enabled = false, tags = {} }
| processor = { enabled = true, tags = { "processor" = "foo" } }
| threads = { enabled = true, tags = {} }
| }
| registries = {
| prometheus = $reg
| }
|}
|""".stripMargin)
val config = MetricsConfig(conf)
config.instrumentations.classloader.enabled must beTrue
config.instrumentations.classloader.tags must beEmpty
config.instrumentations.memory.enabled must beTrue
config.instrumentations.memory.tags must beEmpty
config.instrumentations.gc.enabled must beFalse
config.instrumentations.gc.tags must beEmpty
config.instrumentations.processor.enabled must beTrue
config.instrumentations.processor.tags mustEqual Map("processor" -> "foo")
config.instrumentations.threads.enabled must beTrue
config.instrumentations.threads.tags must beEmpty
config.registries must haveSize(1)
config.registries.get("prometheus") must beSome
config.registries("prometheus") mustEqual ConfigFactory.parseString(reg)
}

"parse deprecated configs" in {
val reg =
"""
|{
| type = "prometheus"
| enabled = false
| rename = true
| common-tags = { "application" = "test" }
| port = 9090
|}
|""".stripMargin
val conf = ConfigFactory.parseString(
s"""{
| enabled = true
| instrumentations = {
| classloader = { enabled = true, tags = {} }
| memory = { enabled = true, tags = {} }
| gc = { enabled = false, tags = {} }
| processor = { enabled = true, tags = { "processor" = "foo" } }
| threads = { enabled = true, tags = {} }
| }
| registries = [
| $reg
| ]
|}
|""".stripMargin)
val config = MetricsConfig(conf)
config.instrumentations.classloader.enabled must beTrue
config.instrumentations.classloader.tags must beEmpty
config.instrumentations.memory.enabled must beTrue
config.instrumentations.memory.tags must beEmpty
config.instrumentations.gc.enabled must beFalse
config.instrumentations.gc.tags must beEmpty
config.instrumentations.processor.enabled must beTrue
config.instrumentations.processor.tags mustEqual Map("processor" -> "foo")
config.instrumentations.threads.enabled must beTrue
config.instrumentations.threads.tags must beEmpty
config.registries must haveSize(1)
config.registries.get("0") must beSome
config.registries("0") mustEqual ConfigFactory.parseString(reg)
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
***********************************************************************/

package org.locationtech.geomesa.metrics.micrometer
package prometheus

import com.typesafe.config.ConfigFactory
import io.micrometer.core.instrument.util.IOUtils
Expand Down

0 comments on commit b7f5d62

Please sign in to comment.