fix(cluster): Fixes for clusterV2 in kubernetes setup (#1680)

* Fixes for ClusterV2, using the hostname config

coordinator/src/main/scala/filodb.coordinator/ShardHealthStats.scala

@@ -33,8 +33,8 @@ class ShardHealthStats(ref: DatasetRef,
   def update(mapper: ShardMapper, skipUnassigned: Boolean = false): Unit = {
     numActive.update(mapper.statuses.count(_ == ShardStatusActive))
     numRecovering.update(mapper.statuses.count(_.isInstanceOf[ShardStatusRecovery]))
-    numUnassigned.update(mapper.statuses.count(_ == ShardStatusUnassigned))
-    if (!skipUnassigned) numAssigned.update(mapper.statuses.count(_ == ShardStatusAssigned))
+    numAssigned.update(mapper.statuses.count(_ == ShardStatusAssigned))
+    if (!skipUnassigned) numUnassigned.update(mapper.statuses.count(_ == ShardStatusUnassigned))
     numError.update(mapper.statuses.count(_ == ShardStatusError))
     numStopped.update(mapper.statuses.count(_ == ShardStatusStopped))
     numDown.update(mapper.statuses.count(_ == ShardStatusDown))

coordinator/src/main/scala/filodb/coordinator/v2/FiloDbClusterDiscovery.scala

@@ -71,12 +71,17 @@ class FiloDbClusterDiscovery(settings: FilodbSettings,
   def shardsForLocalhost(numShards: Int): Seq[Int] = shardsForOrdinal(ordinalOfLocalhost, numShards)
 
   lazy private val hostNames = {
-    require(settings.minNumNodes.isDefined, "Minimum Number of Nodes config not provided")
+    require(settings.minNumNodes.isDefined, "[ClusterV2] Minimum Number of Nodes config not provided")
     if (settings.k8sHostFormat.isDefined) {
-      (0 until settings.minNumNodes.get).map(i => String.format(settings.k8sHostFormat.get, i.toString))
+      // This is used in kubernetes setup. We read the host format config and resolve the FQDN using env variables
+      val hosts = (0 until settings.minNumNodes.get)
+        .map(i => String.format(settings.k8sHostFormat.get, i.toString))
+      logger.info(s"[ClusterV2] hosts to communicate: " + hosts)
+      hosts.sorted
     } else if (settings.hostList.isDefined) {
+      // All the required hosts are provided manually in the config. usually used for local runs/setup
       settings.hostList.get.sorted // sort to make order consistent on all nodes of cluster
-    } else throw new IllegalArgumentException("Cluster Discovery mechanism not defined")
+    } else throw new IllegalArgumentException("[ClusterV2] Cluster Discovery mechanism not defined")
   }
 
   lazy private val nodeCoordActorSelections = {