fix unit test

filodb · Dec 2, 2024 · aff7f80 · aff7f80
1 parent 1b04f67
commit aff7f80
Show file tree

Hide file tree

Showing 2 changed files with 72 additions and 13 deletions.
diff --git a/memory/src/test/scala/filodb.memory/format/vectors/HistogramTest.scala b/memory/src/test/scala/filodb.memory/format/vectors/HistogramTest.scala
@@ -13,10 +13,10 @@ object HistogramTest {
   )
 
   val rawHistBuckets2 = Seq(
-    Array[Double](0, 25, 17, 20, 25, 34, 76, 82),
-    Array[Double](0, 22, 26, 26, 36, 38, 56, 59),
-    Array[Double](0, 27, 26, 27, 33, 42, 46, 55),
-    Array[Double](0, 8, 5, 33, 35, 67, 91, 121)
+    Array[Double](0, 15, 17, 20, 25, 34, 76, 82),
+    Array[Double](0, 16, 26, 26, 36, 38, 56, 59),
+    Array[Double](0, 16, 26, 27, 33, 42, 46, 55),
+    Array[Double](0, 4, 5, 33, 35, 67, 91, 121)
   )
 
   val rawLongBuckets = rawHistBuckets.map(_.map(_.toLong))
@@ -256,27 +256,29 @@ class HistogramTest extends NativeVectorTest {
     it("should calculate more accurate quantile for otel exponential histogram using max and min column") {
       // bucket boundaries for scale 3, range -3 to 3, 7 buckets
       // zeroBucket: 0.0 -3: 0.840896, -2: 0.917004, -1: 1.000000, 0; 1.090508, 1: 1.189207, 2: 1.296840, 3: 1.414214
-      val exp95thWithoutMinMax = Seq(1.3329136609345256, 1.2987136172035367, 1.3772644080792311, 1.3897175222720994)
+      val expected95thWithoutMinMax = Seq(1.3329136609345256, 1.2987136172035367, 1.3772644080792311, 1.3897175222720994)
       otelExpHistograms.map { h =>
         h.quantile(0.95)
-      }.zip(exp95thWithoutMinMax).foreach { case (q, e) => q shouldEqual e +- 0.0001 }
+      }.zip(expected95thWithoutMinMax).foreach { case (q, e) => q shouldEqual e +- 0.0001 }
 
       // notice how the quantiles are calculated within the max now, unlike before
-      val exp95thWithMinMax = Seq(1.2978395301558558, 1.296892165727062, 1.2990334920692943, 1.2993620241156902)
+      val expected95thWithMinMax = Seq(1.2978395301558558, 1.296892165727062, 1.2990334920692943, 1.2993620241156902)
       otelExpHistograms.map { h =>
         h.quantile(0.95, 0.78, 1.3) // notice 1.3 max is less than last bucket
-      }.zip(exp95thWithMinMax).foreach { case (q, e) => q shouldEqual e +- 0.0001 }
+      }.zip(expected95thWithMinMax).foreach { case (q, e) => q shouldEqual e +- 0.0001 }
 
-      val exp5thWithoutMinMax = Seq(0.13790701210160922, 0.11275656477265718, 0.08564685710917462, 0.6359279140356217)
+      val expected5thWithoutMinMax = Seq(0.22984502016934866, 0.15504027656240363, 0.14452907137173218, 0.9199883517494387)
       otelExpHistograms.map { h =>
         h.quantile(0.05)
-      }.zip(exp5thWithoutMinMax).foreach { case (q, e) => q shouldEqual e +- 0.0001 }
+      }.zip(expected5thWithoutMinMax).foreach { case (q, e) => q shouldEqual e +- 0.0001 }
 
       // notice how the quantiles are calculated within the min now, unlike before
-      val exp5thWithMinMax = Seq(0.7896758526873408, 0.7879023377662122, 0.7859951235344664, 0.825628309567315)
+      val expected5thWithMinMax = Seq(0.7961930120386448, 0.7908863115573832, 0.7901434789531481, 0.9199883517494387)
       otelExpHistograms.map { h =>
-        h.quantile(0.05, 0.78, 1.3) // notice 0.78 min is less than first non-zero bucket, but bigger than previous otel bucket if it existed
-      }.zip(exp5thWithMinMax).foreach { case (q, e) => q shouldEqual e +- 0.0001 }
+        // notice 0.78 min is less than first non-zero bucketTop,
+        // but bigger than previous otel bucketTop if it existed
+        h.quantile(0.05, 0.78, 1.3)
+      }.zip(expected5thWithMinMax).foreach { case (q, e) => q shouldEqual e +- 0.0001 }
     }
 
     it("should serialize to and from BinaryHistograms and compare correctly") {

diff --git a/memory/src/test/scala/filodb.memory/format/vectors/HistogramVectorTest.scala b/memory/src/test/scala/filodb.memory/format/vectors/HistogramVectorTest.scala
@@ -87,6 +87,63 @@ class HistogramVectorTest extends NativeVectorTest {
     }
   }
 
+  // This test confirms that we can store 1-minutely exponential histogram samples of 160 buckets
+  // easily in a histogram appender / write buffer, and we can still hold samples for 1-hr flush interval
+  it("should be able to store sufficient hist samples in one 15k byte vector, the histogram appender default size") {
+    val str = "0.0=0.0, 0.0012664448775888738=0.0, 0.0013810679320049727=0.0, 0.001506065259187439=0.0, " +
+      "0.0016423758110424079=0.0, 0.0017910235218841198=0.0, 0.001953124999999996=0.0, 0.002129897915361827=0.0, " +
+      "0.002322670146489685=0.0, 0.0025328897551777484=0.0, 0.002762135864009946=0.0, 0.0030121305183748786=0.0, " +
+      "0.0032847516220848166=0.0, 0.0035820470437682404=0.0, 0.003906249999999993=0.0, 0.004259795830723655=0.0, " +
+      "0.004645340292979371=0.0, 0.005065779510355498=0.0, 0.005524271728019893=0.0, 0.006024261036749759=0.0, " +
+      "0.006569503244169634=0.0, 0.007164094087536483=0.0, 0.007812499999999988=0.0, 0.008519591661447312=0.0, " +
+      "0.009290680585958744=0.0, 0.010131559020710997=0.0, 0.011048543456039788=0.0, 0.012048522073499521=0.0, " +
+      "0.013139006488339272=0.0, 0.014328188175072969=0.0, 0.01562499999999998=0.0, 0.017039183322894627=0.0, " +
+      "0.018581361171917492=0.0, 0.020263118041422=0.0, 0.022097086912079584=0.0, 0.024097044146999046=0.0, " +
+      "0.026278012976678547=0.0, 0.028656376350145944=0.0, 0.031249999999999965=0.0, 0.03407836664578927=0.0, " +
+      "0.037162722343835=0.0, 0.04052623608284401=0.0, 0.044194173824159175=0.0, 0.0481940882939981=0.0, " +
+      "0.05255602595335711=0.0, 0.0573127527002919=0.0, 0.062499999999999944=0.0, 0.06815673329157855=0.0, " +
+      "0.07432544468767001=0.0, 0.08105247216568803=0.0, 0.08838834764831838=0.0, 0.09638817658799623=0.0, " +
+      "0.10511205190671424=0.0, 0.11462550540058382=0.0, 0.12499999999999992=0.0, 0.13631346658315713=1.0, " +
+      "0.14865088937534005=1.0, 0.16210494433137612=1.0, 0.17677669529663678=1.0, 0.1927763531759925=1.0, " +
+      "0.21022410381342854=1.0, 0.2292510108011677=1.0, 0.2499999999999999=2.0, 0.2726269331663143=2.0, " +
+      "0.29730177875068015=3.0, 0.3242098886627523=3.0, 0.3535533905932736=3.0, 0.3855527063519851=3.0, " +
+      "0.42044820762685714=4.0, 0.4585020216023355=5.0, 0.4999999999999999=5.0, 0.5452538663326287=5.0, " +
+      "0.5946035575013604=6.0, 0.6484197773255047=6.0, 0.7071067811865475=8.0, 0.7711054127039704=8.0, " +
+      "0.8408964152537145=9.0, 0.9170040432046712=9.0, 1.0=11.0, 1.0905077326652577=12.0, 1.189207115002721=14.0, " +
+      "1.2968395546510099=15.0, 1.4142135623730951=17.0, 1.542210825407941=19.0, 1.6817928305074294=20.0, " +
+      "1.8340080864093429=22.0, 2.0000000000000004=23.0, 2.181015465330516=26.0, 2.378414230005443=28.0, " +
+      "2.59367910930202=31.0, 2.8284271247461907=34.0, 3.084421650815883=37.0, 3.3635856610148593=41.0, " +
+      "3.6680161728186866=45.0, 4.000000000000002=48.0, 4.3620309306610325=53.0, 4.756828460010887=58.0, " +
+      "5.187358218604041=64.0, 5.656854249492383=70.0, 6.168843301631767=76.0, 6.7271713220297205=84.0, " +
+      "7.336032345637374=90.0, 8.000000000000005=99.0, 8.724061861322067=108.0, 9.513656920021775=118.0, " +
+      "10.374716437208086=129.0, 11.31370849898477=140.0, 12.337686603263537=152.0, 13.454342644059444=167.0, " +
+      "14.672064691274752=182.0, 16.000000000000014=199.0, 17.448123722644137=217.0, 19.027313840043554=237.0, " +
+      "20.749432874416176=258.0, 22.627416997969544=282.0, 24.675373206527077=308.0, 26.908685288118896=336.0, " +
+      "29.34412938254951=367.0, 32.000000000000036=400.0, 34.89624744528828=435.0, 38.05462768008712=474.0, " +
+      "41.49886574883236=517.0, 45.254833995939094=565.0, 49.35074641305417=617.0, 53.8173705762378=672.0, " +
+      "58.688258765099036=732.0, 64.00000000000009=749.0"
+
+    val appender = HistogramVector.appending(memFactory, 15000) // 15k bytes is default blob size
+    val bucketScheme = Base2ExpHistogramBuckets(3, -78, 126)
+    var counts = str.split(", ").map { s =>
+      val kv = s.split("=")
+      kv(1).toDouble.toLong
+    }
+
+    (0 until 206).foreach { buckets =>
+      val hist = LongHistogram(bucketScheme, counts)
+      hist.serialize(Some(buffer))
+      if (buckets < 205) appender.addData(buffer) shouldEqual Ack
+      // should fail for 206th histogram because it crosses the size of write buffer
+      if (buckets >= 205) appender.addData(buffer) shouldEqual VectorTooSmall(73,46)
+      counts = counts.map(_ + 10)
+    }
+
+    val reader = appender.reader.asInstanceOf[RowHistogramReader]
+    reader.length shouldEqual 205
+
+  }
+
   it("should accept MutableHistograms with otel exp scheme and query them back") {
     val appender = HistogramVector.appending(memFactory, 1024)
     val mutHistograms = otelExpHistograms.map(h => MutableHistogram(h.buckets, h.valueArray))