From fd220001439b934412a2c416b2a78bfd99bd0869 Mon Sep 17 00:00:00 2001 From: Andrew Nowak Date: Thu, 5 Sep 2024 14:28:08 +0100 Subject: [PATCH] make an effort to send all metrics before play app shuts down --- collections/app/CollectionsComponents.scala | 2 +- collections/app/lib/CollectionsMetrics.scala | 6 ++++-- .../lib/metrics/CloudWatchMetrics.scala | 16 +++++++++++++++- .../lib/play/RequestMetricFilter.scala | 5 +++-- image-loader/app/ImageLoaderComponents.scala | 2 +- image-loader/app/lib/ImageLoaderMetrics.scala | 5 +++-- media-api/app/MediaApiComponents.scala | 2 +- media-api/app/lib/MediaApiMetrics.scala | 5 +++-- .../lib/elasticsearch/ElasticSearchTest.scala | 12 +++++++----- .../app/MetadataEditorComponents.scala | 2 +- .../app/lib/MetadataEditorMetrics.scala | 6 ++++-- .../mediaservice/lib/play/GridComponents.scala | 2 +- thrall/app/ThrallComponents.scala | 2 +- thrall/app/lib/ThrallMetrics.scala | 5 +++-- usage/app/UsageComponents.scala | 2 +- usage/app/lib/UsageMetrics.scala | 5 +++-- 16 files changed, 52 insertions(+), 27 deletions(-) diff --git a/collections/app/CollectionsComponents.scala b/collections/app/CollectionsComponents.scala index 6c84c113f2..046c99e065 100644 --- a/collections/app/CollectionsComponents.scala +++ b/collections/app/CollectionsComponents.scala @@ -10,7 +10,7 @@ class CollectionsComponents(context: Context) extends GridComponents(context, ne final override val buildInfo = utils.buildinfo.BuildInfo val store = new CollectionsStore(config) - val metrics = new CollectionsMetrics(config, actorSystem) + val metrics = new CollectionsMetrics(config, actorSystem, applicationLifecycle) val notifications = new Notifications(config) val collections = new CollectionsController(auth, config, store, controllerComponents) diff --git a/collections/app/lib/CollectionsMetrics.scala b/collections/app/lib/CollectionsMetrics.scala index 1c3e13c0b2..1c176a72d5 100644 --- a/collections/app/lib/CollectionsMetrics.scala +++ b/collections/app/lib/CollectionsMetrics.scala @@ -2,14 +2,16 @@ package lib import akka.actor.ActorSystem import com.gu.mediaservice.lib.metrics.CloudWatchMetrics +import play.api.inject.ApplicationLifecycle import scala.concurrent.ExecutionContext class CollectionsMetrics( config: CollectionsConfig, - actorSystem: ActorSystem + actorSystem: ActorSystem, + applicationLifecycle: ApplicationLifecycle )(implicit ec: ExecutionContext) - extends CloudWatchMetrics(s"${config.stage}/Collections", config, actorSystem) { + extends CloudWatchMetrics(s"${config.stage}/Collections", config, actorSystem, applicationLifecycle) { val processingLatency = new TimeMetric("ProcessingLatency") diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/metrics/CloudWatchMetrics.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/metrics/CloudWatchMetrics.scala index df0dccc39e..1f49306329 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/metrics/CloudWatchMetrics.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/metrics/CloudWatchMetrics.scala @@ -1,10 +1,13 @@ package com.gu.mediaservice.lib.metrics import akka.actor.{Actor, ActorSystem, Props, Timers} +import akka.pattern.ask +import akka.util.Timeout import com.amazonaws.services.cloudwatch.{AmazonCloudWatch, AmazonCloudWatchClientBuilder} import com.amazonaws.services.cloudwatch.model.{Dimension, MetricDatum, PutMetricDataRequest, StandardUnit, StatisticSet} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.GridLogging +import play.api.inject.ApplicationLifecycle import java.util.Date import scala.collection.JavaConverters._ @@ -20,7 +23,8 @@ trait Metric[A] { abstract class CloudWatchMetrics( namespace: String, config: CommonConfig, - actorSystem: ActorSystem + actorSystem: ActorSystem, + applicationLifecycle: ApplicationLifecycle ) extends GridLogging { class CountMetric(name: String) extends CloudWatchMetric[Long](name) { @@ -43,6 +47,8 @@ abstract class CloudWatchMetrics( private[CloudWatchMetrics] val metricsActor = actorSystem.actorOf(MetricsActor.props(namespace, client), "metricsactor") + applicationLifecycle.addStopHook(() => (metricsActor ? MetricsActor.Shutdown)(Timeout(5.seconds))) + abstract class CloudWatchMetric[A](val name: String) extends Metric[A] { final def recordOne(value: A, dimensions: List[Dimension] = Nil): Unit = metricsActor ! MetricsActor.AddMetrics(List(toDatum(value, dimensions))) @@ -74,6 +80,7 @@ object MetricsActor { Props(new MetricsActor(namespace, client)) private final case object Tick + final case object Shutdown final case class AddMetrics(values: Seq[MetricDatum]) } private class MetricsActor(namespace: String, client: AmazonCloudWatch) extends Actor with Timers with GridLogging { @@ -124,11 +131,18 @@ private class MetricsActor(namespace: String, client: AmazonCloudWatch) extends def receive: Receive = { case AddMetrics(metrics) => become(queued(metrics)) + case Shutdown => become(shutdown) + } + private def shutdown: Receive = { + case message => logger.error(s"metrics actor has shut down, cannot respond to message $message") } private def queued(queuedMetrics: Seq[MetricDatum]): Receive = { case Tick => putData(queuedMetrics) // send metrics off become(receive) + case Shutdown => + putData(queuedMetrics) + become(shutdown) case AddMetrics(metrics) => become(queued(queuedMetrics ++ metrics)) } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/play/RequestMetricFilter.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/play/RequestMetricFilter.scala index 704b620d70..09ea20ff4c 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/play/RequestMetricFilter.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/play/RequestMetricFilter.scala @@ -4,16 +4,17 @@ import akka.actor.ActorSystem import akka.stream.Materializer import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.metrics.CloudWatchMetrics +import play.api.inject.ApplicationLifecycle import play.api.mvc.{Filter, RequestHeader, Result} import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success} -class RequestMetricFilter(val config: CommonConfig, override val mat: Materializer, actorSystem: ActorSystem)(implicit ec: ExecutionContext) extends Filter { +class RequestMetricFilter(val config: CommonConfig, override val mat: Materializer, actorSystem: ActorSystem, applicationLifecycle: ApplicationLifecycle)(implicit ec: ExecutionContext) extends Filter { val namespace: String = s"${config.stage}/${config.appName.split('-').map(_.toLowerCase.capitalize).mkString("")}" val enabled: Boolean = config.requestMetricsEnabled - object RequestMetrics extends CloudWatchMetrics(namespace, config, actorSystem) { + object RequestMetrics extends CloudWatchMetrics(namespace, config, actorSystem, applicationLifecycle) { val totalRequests = new CountMetric("TotalRequests") val successfulRequests = new CountMetric("SuccessfulRequests") val failedRequests = new CountMetric("FailedRequests") diff --git a/image-loader/app/ImageLoaderComponents.scala b/image-loader/app/ImageLoaderComponents.scala index 2cc0493669..ebe3ef69ee 100644 --- a/image-loader/app/ImageLoaderComponents.scala +++ b/image-loader/app/ImageLoaderComponents.scala @@ -41,7 +41,7 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne val services = new Services(config.domainRoot, config.serviceHosts, Set.empty) private val gridClient = GridClient(services)(wsClient) - val metrics = new ImageLoaderMetrics(config, actorSystem) + val metrics = new ImageLoaderMetrics(config, actorSystem, applicationLifecycle) val controller = new ImageLoaderController( auth, downloader, store, maybeIngestQueue, uploadStatusTable, notifications, config, uploader, quarantineUploader, projector, controllerComponents, gridClient, authorisation, metrics) diff --git a/image-loader/app/lib/ImageLoaderMetrics.scala b/image-loader/app/lib/ImageLoaderMetrics.scala index 306a7371c9..d9c93ed6ee 100644 --- a/image-loader/app/lib/ImageLoaderMetrics.scala +++ b/image-loader/app/lib/ImageLoaderMetrics.scala @@ -2,9 +2,10 @@ package lib import akka.actor.ActorSystem import com.gu.mediaservice.lib.metrics.CloudWatchMetrics +import play.api.inject.ApplicationLifecycle -class ImageLoaderMetrics(config: ImageLoaderConfig, actorSystem: ActorSystem) - extends CloudWatchMetrics (namespace = s"${config.stage}/ImageLoader", config, actorSystem){ +class ImageLoaderMetrics(config: ImageLoaderConfig, actorSystem: ActorSystem, applicationLifecycle: ApplicationLifecycle) + extends CloudWatchMetrics (namespace = s"${config.stage}/ImageLoader", config, actorSystem, applicationLifecycle){ val successfulIngestsFromQueue = new CountMetric("SuccessfulIngestsFromQueue") diff --git a/media-api/app/MediaApiComponents.scala b/media-api/app/MediaApiComponents.scala index ce70046d79..d43e5470b5 100644 --- a/media-api/app/MediaApiComponents.scala +++ b/media-api/app/MediaApiComponents.scala @@ -17,7 +17,7 @@ class MediaApiComponents(context: Context) extends GridComponents(context, new M val imageOperations = new ImageOperations(context.environment.rootPath.getAbsolutePath) val messageSender = new ThrallMessageSender(config.thrallKinesisStreamConfig) - val mediaApiMetrics = new MediaApiMetrics(config, actorSystem) + val mediaApiMetrics = new MediaApiMetrics(config, actorSystem, applicationLifecycle) val s3Client = new S3Client(config) diff --git a/media-api/app/lib/MediaApiMetrics.scala b/media-api/app/lib/MediaApiMetrics.scala index d8bd0980a0..48b202e898 100644 --- a/media-api/app/lib/MediaApiMetrics.scala +++ b/media-api/app/lib/MediaApiMetrics.scala @@ -4,11 +4,12 @@ import akka.actor.ActorSystem import com.amazonaws.services.cloudwatch.model.Dimension import com.gu.mediaservice.lib.auth.{ApiAccessor, Syndication} import com.gu.mediaservice.lib.metrics.CloudWatchMetrics +import play.api.inject.ApplicationLifecycle import scala.concurrent.ExecutionContext -class MediaApiMetrics(config: MediaApiConfig, actorSystem: ActorSystem)(implicit ec: ExecutionContext) - extends CloudWatchMetrics(s"${config.stage}/MediaApi", config, actorSystem) { +class MediaApiMetrics(config: MediaApiConfig, actorSystem: ActorSystem, applicationLifecycle: ApplicationLifecycle)(implicit ec: ExecutionContext) + extends CloudWatchMetrics(s"${config.stage}/MediaApi", config, actorSystem, applicationLifecycle) { val searchQueries = new TimeMetric("ElasticSearch") diff --git a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala index 13c334c76a..2352939f2b 100644 --- a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala +++ b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala @@ -32,16 +32,18 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast private val index = "images" + private val applicationLifecycle = new ApplicationLifecycle { + override def addStopHook(hook: () => Future[_]): Unit = {} + override def stop(): Future[_] = Future.successful(()) + } + private val mediaApiConfig = new MediaApiConfig(GridConfigResources( Configuration.from(USED_CONFIGS_IN_TEST ++ MOCK_CONFIG_KEYS.map(_ -> NOT_USED_IN_TEST).toMap), null, - new ApplicationLifecycle { - override def addStopHook(hook: () => Future[_]): Unit = {} - override def stop(): Future[_] = Future.successful(()) - } + applicationLifecycle )) private val actorSystem: ActorSystem = ActorSystem() - private val mediaApiMetrics = new MediaApiMetrics(mediaApiConfig, actorSystem) + private val mediaApiMetrics = new MediaApiMetrics(mediaApiConfig, actorSystem, applicationLifecycle) val elasticConfig = ElasticSearchConfig( aliases = ElasticSearchAliases( current = "Images_Current", diff --git a/metadata-editor/app/MetadataEditorComponents.scala b/metadata-editor/app/MetadataEditorComponents.scala index 554959486f..8020aed28a 100644 --- a/metadata-editor/app/MetadataEditorComponents.scala +++ b/metadata-editor/app/MetadataEditorComponents.scala @@ -14,7 +14,7 @@ class MetadataEditorComponents(context: Context) extends GridComponents(context, val notifications = new Notifications(config) val imageOperations = new ImageOperations(context.environment.rootPath.getAbsolutePath) - val metrics = new MetadataEditorMetrics(config, actorSystem) + val metrics = new MetadataEditorMetrics(config, actorSystem, applicationLifecycle) val messageConsumer = new MetadataSqsMessageConsumer(config, metrics, editsStore) messageConsumer.startSchedule() diff --git a/metadata-editor/app/lib/MetadataEditorMetrics.scala b/metadata-editor/app/lib/MetadataEditorMetrics.scala index 9374898961..3bb35e30c1 100644 --- a/metadata-editor/app/lib/MetadataEditorMetrics.scala +++ b/metadata-editor/app/lib/MetadataEditorMetrics.scala @@ -2,14 +2,16 @@ package lib import akka.actor.ActorSystem import com.gu.mediaservice.lib.metrics.CloudWatchMetrics +import play.api.inject.ApplicationLifecycle import scala.concurrent.ExecutionContext class MetadataEditorMetrics( config: EditsConfig, - actorSystem: ActorSystem + actorSystem: ActorSystem, + applicationLifecycle: ApplicationLifecycle )(implicit ec: ExecutionContext) - extends CloudWatchMetrics(s"${config.stage}/MetadataEditor", config, actorSystem) { + extends CloudWatchMetrics(s"${config.stage}/MetadataEditor", config, actorSystem, applicationLifecycle) { val snsMessage = new CountMetric("SNSMessage") diff --git a/rest-lib/src/main/scala/com/gu/mediaservice/lib/play/GridComponents.scala b/rest-lib/src/main/scala/com/gu/mediaservice/lib/play/GridComponents.scala index 1761515b5e..354613905d 100644 --- a/rest-lib/src/main/scala/com/gu/mediaservice/lib/play/GridComponents.scala +++ b/rest-lib/src/main/scala/com/gu/mediaservice/lib/play/GridComponents.scala @@ -35,7 +35,7 @@ abstract class GridComponents[Config <: CommonConfig](context: Context, val load gzipFilter, new RequestLoggingFilter(materializer), new ConnectionBrokenFilter(materializer), - new RequestMetricFilter(config, materializer, actorSystem) + new RequestMetricFilter(config, materializer, actorSystem, applicationLifecycle) ) final override lazy val corsConfig: CORSConfig = CORSConfig.fromConfiguration(context.initialConfiguration).copy( diff --git a/thrall/app/ThrallComponents.scala b/thrall/app/ThrallComponents.scala index 7e4ed48fcf..2057de575c 100644 --- a/thrall/app/ThrallComponents.scala +++ b/thrall/app/ThrallComponents.scala @@ -26,7 +26,7 @@ class ThrallComponents(context: Context) extends GridComponents(context, new Thr val store = new ThrallStore(config) val metadataEditorNotifications = new MetadataEditorNotifications(config) - val thrallMetrics = new ThrallMetrics(config, actorSystem) + val thrallMetrics = new ThrallMetrics(config, actorSystem, applicationLifecycle) val es = new ElasticSearch(config.esConfig, Some(thrallMetrics), actorSystem.scheduler) es.ensureIndexExistsAndAliasAssigned() diff --git a/thrall/app/lib/ThrallMetrics.scala b/thrall/app/lib/ThrallMetrics.scala index 19faee5dcf..4eb39102df 100644 --- a/thrall/app/lib/ThrallMetrics.scala +++ b/thrall/app/lib/ThrallMetrics.scala @@ -2,11 +2,12 @@ package lib import akka.actor.ActorSystem import com.gu.mediaservice.lib.metrics.CloudWatchMetrics +import play.api.inject.ApplicationLifecycle import scala.concurrent.ExecutionContext -class ThrallMetrics(config: ThrallConfig, actorSystem: ActorSystem)(implicit ec: ExecutionContext) - extends CloudWatchMetrics(s"${config.stage}/Thrall", config, actorSystem) { +class ThrallMetrics(config: ThrallConfig, actorSystem: ActorSystem, applicationLifecycle: ApplicationLifecycle)(implicit ec: ExecutionContext) + extends CloudWatchMetrics(s"${config.stage}/Thrall", config, actorSystem, applicationLifecycle) { val indexedImages = new CountMetric("IndexedImages") diff --git a/usage/app/UsageComponents.scala b/usage/app/UsageComponents.scala index 0260ddc53b..692e10502e 100644 --- a/usage/app/UsageComponents.scala +++ b/usage/app/UsageComponents.scala @@ -18,7 +18,7 @@ class UsageComponents(context: Context) extends GridComponents(context, new Usag val liveContentApi = new LiveContentApi(config)(ScheduledExecutor()) val usageGroupOps = new UsageGroupOps(config, mediaWrapper) val usageTable = new UsageTable(config) - val usageMetrics = new UsageMetrics(config, actorSystem) + val usageMetrics = new UsageMetrics(config, actorSystem, applicationLifecycle) val usageNotifier = new UsageNotifier(config, usageTable) val usageRecorder = new UsageRecorder(usageMetrics, usageTable, usageNotifier, usageNotifier) val notifications = new Notifications(config) diff --git a/usage/app/lib/UsageMetrics.scala b/usage/app/lib/UsageMetrics.scala index 85e2364208..4b41687ac5 100644 --- a/usage/app/lib/UsageMetrics.scala +++ b/usage/app/lib/UsageMetrics.scala @@ -2,11 +2,12 @@ package lib import akka.actor.ActorSystem import com.gu.mediaservice.lib.metrics.CloudWatchMetrics +import play.api.inject.ApplicationLifecycle import scala.concurrent.ExecutionContext -class UsageMetrics(config: UsageConfig, actorSystem: ActorSystem)(implicit ec: ExecutionContext) - extends CloudWatchMetrics(s"${config.stage}/Usage", config, actorSystem) { +class UsageMetrics(config: UsageConfig, actorSystem: ActorSystem, applicationLifecycle: ApplicationLifecycle)(implicit ec: ExecutionContext) + extends CloudWatchMetrics(s"${config.stage}/Usage", config, actorSystem, applicationLifecycle) { def incrementUpdated = updates.increment() def incrementErrors = errors.increment()