From 4228a44db42c4aa83624fadd6f8ac6429b8b4897 Mon Sep 17 00:00:00 2001
From: Seth Digel <digel@stanford.edu>
Date: Thu, 8 Jun 2017 04:53:00 -0700
Subject: [PATCH 1/9] Adding flatnesshalfband_95 and frac_outside

---
 .../v0/validator_absolute_height_offline.py   | 19 ++++++++++++++++++-
 .../v0/validator_flatness_offline.py          |  3 +++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/harnessed_jobs/absolute_height_offline/v0/validator_absolute_height_offline.py b/harnessed_jobs/absolute_height_offline/v0/validator_absolute_height_offline.py
index 3aa6908..561ed0f 100755
--- a/harnessed_jobs/absolute_height_offline/v0/validator_absolute_height_offline.py
+++ b/harnessed_jobs/absolute_height_offline/v0/validator_absolute_height_offline.py
@@ -21,10 +21,27 @@
     z_median_m_13 = np.median(sensorData.sensor.z) - 13000.
     z_quantile_0025 = sensorData.quantiles['0.025']
     z_quantile_0975 = sensorData.quantiles['0.975']
+
+    zvalues, quantiles = [], []
+    for key in sensorData.quantiles:
+        quantiles.append(float(key))
+        zvalues.append(sensorData.quantiles[key])
+
+    zvalues.sort()
+    quantiles.sort()
+
+    znom = 12.992  # for ITL
+    zbounds = (-0.009, 0.009)  # from CCD-030
+    quant_low = np.interp(znom + 0.009, zvalues, quantiles)
+    quant_high = np.interp(znom - 0.009, zvalues, quantiles)
+    frac_outside = 1. - (quant_high - quant_low)
+
     results.append(lcatr.schema.valid(lcatr.schema.get('sensor_abs_height'),
                                       z_median_m_13=z_median_m_13,
                                       z_quantile_0025=z_quantile_0025,
-                                      z_quantile_0975=z_quantile_0975))
+                                      z_quantile_0975=z_quantile_0975,
+                                      frac_low=frac_low,
+                                      flatnesshalfband_95=flatnesshalfband_95))
 
 results.extend(siteUtils.packageVersions())
 
diff --git a/harnessed_jobs/flatness_offline/v0/validator_flatness_offline.py b/harnessed_jobs/flatness_offline/v0/validator_flatness_offline.py
index 3a4e7c9..8b8ed2a 100755
--- a/harnessed_jobs/flatness_offline/v0/validator_flatness_offline.py
+++ b/harnessed_jobs/flatness_offline/v0/validator_flatness_offline.py
@@ -10,8 +10,11 @@
 
 sensorData = md_factory.load('flatness.pickle')
 peak_valley_95 = sensorData.quantiles['0.975'] - sensorData.quantiles['0.025']
+flatnesshalfband_95 = peak_valley_95/2.
+
 results.append(lcatr.schema.valid(lcatr.schema.get('sensor_flatness'),
                                   peak_valley_95=peak_valley_95))
+                                  flatnesshalfband_95=flatnesshalfband_95))
 
 results.extend(siteUtils.packageVersions())
 

From 86c56ec2671b344b35607254c99b60401d9664e5 Mon Sep 17 00:00:00 2001
From: Seth Digel <digel@stanford.edu>
Date: Fri, 9 Jun 2017 20:36:40 -0700
Subject: [PATCH 2/9] Making schemas directory, removing unused schema files
 for offline tasks

---
 schemas/sensor_abs_height.schema | 10 ++++++++++
 schemas/sensor_flatness.schema   |  7 +++++++
 2 files changed, 17 insertions(+)
 create mode 100644 schemas/sensor_abs_height.schema
 create mode 100644 schemas/sensor_flatness.schema

diff --git a/schemas/sensor_abs_height.schema b/schemas/sensor_abs_height.schema
new file mode 100644
index 0000000..7d27ab6
--- /dev/null
+++ b/schemas/sensor_abs_height.schema
@@ -0,0 +1,10 @@
+# -*- python -*-
+{
+    'schema_name' : 'sensor_abs_height',
+    'schema_version' : 0,
+    'z_median_m_13' : float,
+    'z_quantile_0975' : float,
+    'z_quantile_0025' : float,
+    'frac_low' : float,
+    'flatnesshalfband_95' : float}
+}
diff --git a/schemas/sensor_flatness.schema b/schemas/sensor_flatness.schema
new file mode 100644
index 0000000..671580e
--- /dev/null
+++ b/schemas/sensor_flatness.schema
@@ -0,0 +1,7 @@
+# -*- python -*-
+{
+    'schema_name' : 'sensor_flatness',
+    'schema_version' : 0,
+    'peak_valley_95' : float,
+    'flatnesshalfband_95' : float}
+}

From 7720f1c4c1e7516788373ed900554cc3b8c662af Mon Sep 17 00:00:00 2001
From: Seth Digel <digel@stanford.edu>
Date: Sun, 18 Jun 2017 21:24:01 -0700
Subject: [PATCH 3/9] Updates based on fake eTraveler run

---
 .../v0/producer_absolute_height_offline.py                 | 3 +++
 .../v0/validator_absolute_height_offline.py                | 7 ++++---
 schemas/sensor_abs_height.schema                           | 4 ++--
 schemas/sensor_flatness.schema                             | 2 +-
 4 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/harnessed_jobs/absolute_height_offline/v0/producer_absolute_height_offline.py b/harnessed_jobs/absolute_height_offline/v0/producer_absolute_height_offline.py
index b645083..ddb88df 100755
--- a/harnessed_jobs/absolute_height_offline/v0/producer_absolute_height_offline.py
+++ b/harnessed_jobs/absolute_height_offline/v0/producer_absolute_height_offline.py
@@ -7,6 +7,9 @@
 sensor_id = siteUtils.getUnitId()
 ccd_vendor = siteUtils.getCcdVendor()
 
+print("sensor_id:  " + sensor_id)
+print("ccd_vendor:  " + ccd_vendor)
+
 if ccd_vendor == 'e2v':
     print "Absolute height analysis cannot not be performed with e2v vendor data.  Exiting producer script."
     sys.exit(0)
diff --git a/harnessed_jobs/absolute_height_offline/v0/validator_absolute_height_offline.py b/harnessed_jobs/absolute_height_offline/v0/validator_absolute_height_offline.py
index 561ed0f..0bcbf04 100755
--- a/harnessed_jobs/absolute_height_offline/v0/validator_absolute_height_offline.py
+++ b/harnessed_jobs/absolute_height_offline/v0/validator_absolute_height_offline.py
@@ -21,6 +21,7 @@
     z_median_m_13 = np.median(sensorData.sensor.z) - 13000.
     z_quantile_0025 = sensorData.quantiles['0.025']
     z_quantile_0975 = sensorData.quantiles['0.975']
+    flatnesshalfband_95 = (z_quantile_0975 - z_quantile_0025)/2.
 
     zvalues, quantiles = [], []
     for key in sensorData.quantiles:
@@ -32,15 +33,15 @@
 
     znom = 12.992  # for ITL
     zbounds = (-0.009, 0.009)  # from CCD-030
-    quant_low = np.interp(znom + 0.009, zvalues, quantiles)
-    quant_high = np.interp(znom - 0.009, zvalues, quantiles)
+    quant_low = np.interp(znom - zbounds[0], zvalues, quantiles)
+    quant_high = np.interp(znom - zbounds[1], zvalues, quantiles)
     frac_outside = 1. - (quant_high - quant_low)
 
     results.append(lcatr.schema.valid(lcatr.schema.get('sensor_abs_height'),
                                       z_median_m_13=z_median_m_13,
                                       z_quantile_0025=z_quantile_0025,
                                       z_quantile_0975=z_quantile_0975,
-                                      frac_low=frac_low,
+                                      frac_outside=frac_outside,
                                       flatnesshalfband_95=flatnesshalfband_95))
 
 results.extend(siteUtils.packageVersions())
diff --git a/schemas/sensor_abs_height.schema b/schemas/sensor_abs_height.schema
index 7d27ab6..dbd96fb 100644
--- a/schemas/sensor_abs_height.schema
+++ b/schemas/sensor_abs_height.schema
@@ -5,6 +5,6 @@
     'z_median_m_13' : float,
     'z_quantile_0975' : float,
     'z_quantile_0025' : float,
-    'frac_low' : float,
-    'flatnesshalfband_95' : float}
+    'frac_outside' : float,
+    'flatnesshalfband_95' : float
 }
diff --git a/schemas/sensor_flatness.schema b/schemas/sensor_flatness.schema
index 671580e..271e767 100644
--- a/schemas/sensor_flatness.schema
+++ b/schemas/sensor_flatness.schema
@@ -3,5 +3,5 @@
     'schema_name' : 'sensor_flatness',
     'schema_version' : 0,
     'peak_valley_95' : float,
-    'flatnesshalfband_95' : float}
+    'flatnesshalfband_95' : float
 }

From feaee87e1f59d8f1bda28b8d3f22138203216155 Mon Sep 17 00:00:00 2001
From: Seth Digel <digel@stanford.edu>
Date: Thu, 14 Sep 2017 13:46:16 -0700
Subject: [PATCH 4/9] New schemas folder with schemas for absolute height and
 flatness analysis

---
 schemas/abs_height_sensor.schema | 10 ++++++++++
 schemas/absolute_height.schema   | 12 ++++++++++++
 schemas/flatness.schema          |  7 +++++++
 schemas/flatness_sensor.schema   |  7 +++++++
 4 files changed, 36 insertions(+)
 create mode 100644 schemas/abs_height_sensor.schema
 create mode 100644 schemas/absolute_height.schema
 create mode 100644 schemas/flatness.schema
 create mode 100644 schemas/flatness_sensor.schema

diff --git a/schemas/abs_height_sensor.schema b/schemas/abs_height_sensor.schema
new file mode 100644
index 0000000..dbd96fb
--- /dev/null
+++ b/schemas/abs_height_sensor.schema
@@ -0,0 +1,10 @@
+# -*- python -*-
+{
+    'schema_name' : 'sensor_abs_height',
+    'schema_version' : 0,
+    'z_median_m_13' : float,
+    'z_quantile_0975' : float,
+    'z_quantile_0025' : float,
+    'frac_outside' : float,
+    'flatnesshalfband_95' : float
+}
diff --git a/schemas/absolute_height.schema b/schemas/absolute_height.schema
new file mode 100644
index 0000000..fb30df3
--- /dev/null
+++ b/schemas/absolute_height.schema
@@ -0,0 +1,12 @@
+# -*- python -*-
+{
+    'schema_name' : 'absolute_height',
+    'schema_version' : 0,
+    'dzdx' : float,
+    'dzdy' : float,
+    'z0' : float,
+    'zmean' : float,
+    'znom' : float,
+    'znom_residual_025' : float,
+    'znom_residual_975' : float
+}
diff --git a/schemas/flatness.schema b/schemas/flatness.schema
new file mode 100644
index 0000000..bc6de1f
--- /dev/null
+++ b/schemas/flatness.schema
@@ -0,0 +1,7 @@
+# -*- python -*-
+{
+    'schema_name' : 'flatness',
+    'schema_version' : 0,
+    'residual_025' : float,
+    'residual_975' : float
+}
diff --git a/schemas/flatness_sensor.schema b/schemas/flatness_sensor.schema
new file mode 100644
index 0000000..271e767
--- /dev/null
+++ b/schemas/flatness_sensor.schema
@@ -0,0 +1,7 @@
+# -*- python -*-
+{
+    'schema_name' : 'sensor_flatness',
+    'schema_version' : 0,
+    'peak_valley_95' : float,
+    'flatnesshalfband_95' : float
+}

From 3c221ce6643bc9677308f0022c134c3aa9346d89 Mon Sep 17 00:00:00 2001
From: Seth Digel <digel@stanford.edu>
Date: Thu, 14 Sep 2017 15:44:06 -0700
Subject: [PATCH 5/9] Fixed schema names

---
 schemas/abs_height_sensor.schema | 2 +-
 schemas/flatness_sensor.schema   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/schemas/abs_height_sensor.schema b/schemas/abs_height_sensor.schema
index dbd96fb..186d200 100644
--- a/schemas/abs_height_sensor.schema
+++ b/schemas/abs_height_sensor.schema
@@ -1,6 +1,6 @@
 # -*- python -*-
 {
-    'schema_name' : 'sensor_abs_height',
+    'schema_name' : 'abs_height_sensor',
     'schema_version' : 0,
     'z_median_m_13' : float,
     'z_quantile_0975' : float,
diff --git a/schemas/flatness_sensor.schema b/schemas/flatness_sensor.schema
index 271e767..912641f 100644
--- a/schemas/flatness_sensor.schema
+++ b/schemas/flatness_sensor.schema
@@ -1,6 +1,6 @@
 # -*- python -*-
 {
-    'schema_name' : 'sensor_flatness',
+    'schema_name' : 'flatness_sensor',
     'schema_version' : 0,
     'peak_valley_95' : float,
     'flatnesshalfband_95' : float

From 22fad342cdca244cc9410103e871776f53c5b6b5 Mon Sep 17 00:00:00 2001
From: Seth Digel <digel@stanford.edu>
Date: Thu, 14 Sep 2017 16:12:35 -0700
Subject: [PATCH 6/9] Modified absolute height schema and validator to remove
 flatnesshalfband_95

---
 .../v0/validator_absolute_height_offline.py                 | 6 ++----
 .../flatness_offline/v0/validator_flatness_offline.py       | 2 +-
 schemas/abs_height_sensor.schema                            | 3 +--
 3 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/harnessed_jobs/absolute_height_offline/v0/validator_absolute_height_offline.py b/harnessed_jobs/absolute_height_offline/v0/validator_absolute_height_offline.py
index 0bcbf04..6bc9132 100755
--- a/harnessed_jobs/absolute_height_offline/v0/validator_absolute_height_offline.py
+++ b/harnessed_jobs/absolute_height_offline/v0/validator_absolute_height_offline.py
@@ -21,7 +21,6 @@
     z_median_m_13 = np.median(sensorData.sensor.z) - 13000.
     z_quantile_0025 = sensorData.quantiles['0.025']
     z_quantile_0975 = sensorData.quantiles['0.975']
-    flatnesshalfband_95 = (z_quantile_0975 - z_quantile_0025)/2.
 
     zvalues, quantiles = [], []
     for key in sensorData.quantiles:
@@ -37,12 +36,11 @@
     quant_high = np.interp(znom - zbounds[1], zvalues, quantiles)
     frac_outside = 1. - (quant_high - quant_low)
 
-    results.append(lcatr.schema.valid(lcatr.schema.get('sensor_abs_height'),
+    results.append(lcatr.schema.valid(lcatr.schema.get('abs_height_sensor'),
                                       z_median_m_13=z_median_m_13,
                                       z_quantile_0025=z_quantile_0025,
                                       z_quantile_0975=z_quantile_0975,
-                                      frac_outside=frac_outside,
-                                      flatnesshalfband_95=flatnesshalfband_95))
+                                      frac_outside=frac_outside))
 
 results.extend(siteUtils.packageVersions())
 
diff --git a/harnessed_jobs/flatness_offline/v0/validator_flatness_offline.py b/harnessed_jobs/flatness_offline/v0/validator_flatness_offline.py
index 8b8ed2a..e441456 100755
--- a/harnessed_jobs/flatness_offline/v0/validator_flatness_offline.py
+++ b/harnessed_jobs/flatness_offline/v0/validator_flatness_offline.py
@@ -12,7 +12,7 @@
 peak_valley_95 = sensorData.quantiles['0.975'] - sensorData.quantiles['0.025']
 flatnesshalfband_95 = peak_valley_95/2.
 
-results.append(lcatr.schema.valid(lcatr.schema.get('sensor_flatness'),
+results.append(lcatr.schema.valid(lcatr.schema.get('flatness_sensor'),
                                   peak_valley_95=peak_valley_95))
                                   flatnesshalfband_95=flatnesshalfband_95))
 
diff --git a/schemas/abs_height_sensor.schema b/schemas/abs_height_sensor.schema
index 186d200..4ec5b4a 100644
--- a/schemas/abs_height_sensor.schema
+++ b/schemas/abs_height_sensor.schema
@@ -5,6 +5,5 @@
     'z_median_m_13' : float,
     'z_quantile_0975' : float,
     'z_quantile_0025' : float,
-    'frac_outside' : float,
-    'flatnesshalfband_95' : float
+    'frac_outside' : float
 }

From a7494f04d314e96616016edcf81d21a6b5be214b Mon Sep 17 00:00:00 2001
From: Seth Digel <digel@stanford.edu>
Date: Fri, 15 Sep 2017 00:18:55 -0700
Subject: [PATCH 7/9] Fixed unbalanced () in validator

---
 .../flatness_offline/v0/validator_flatness_offline.py           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/harnessed_jobs/flatness_offline/v0/validator_flatness_offline.py b/harnessed_jobs/flatness_offline/v0/validator_flatness_offline.py
index e441456..136f6b2 100755
--- a/harnessed_jobs/flatness_offline/v0/validator_flatness_offline.py
+++ b/harnessed_jobs/flatness_offline/v0/validator_flatness_offline.py
@@ -13,7 +13,7 @@
 flatnesshalfband_95 = peak_valley_95/2.
 
 results.append(lcatr.schema.valid(lcatr.schema.get('flatness_sensor'),
-                                  peak_valley_95=peak_valley_95))
+                                  peak_valley_95=peak_valley_95,
                                   flatnesshalfband_95=flatnesshalfband_95))
 
 results.extend(siteUtils.packageVersions())

From 6ee0483996e21a28fe8f045308c8761a9c9f38e1 Mon Sep 17 00:00:00 2001
From: Seth Digel <digel@stanford.edu>
Date: Fri, 15 Sep 2017 12:51:44 -0700
Subject: [PATCH 8/9] Removed print statements from producer script

---
 .../v0/producer_absolute_height_offline.py                     | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/harnessed_jobs/absolute_height_offline/v0/producer_absolute_height_offline.py b/harnessed_jobs/absolute_height_offline/v0/producer_absolute_height_offline.py
index ddb88df..b645083 100755
--- a/harnessed_jobs/absolute_height_offline/v0/producer_absolute_height_offline.py
+++ b/harnessed_jobs/absolute_height_offline/v0/producer_absolute_height_offline.py
@@ -7,9 +7,6 @@
 sensor_id = siteUtils.getUnitId()
 ccd_vendor = siteUtils.getCcdVendor()
 
-print("sensor_id:  " + sensor_id)
-print("ccd_vendor:  " + ccd_vendor)
-
 if ccd_vendor == 'e2v':
     print "Absolute height analysis cannot not be performed with e2v vendor data.  Exiting producer script."
     sys.exit(0)

From c186caef414b2d386a05a154e8312bade14f43f1 Mon Sep 17 00:00:00 2001
From: Seth Digel <digel@stanford.edu>
Date: Fri, 15 Sep 2017 12:55:45 -0700
Subject: [PATCH 9/9] Removed old schema files

---
 schemas/absolute_height.schema   | 12 ------------
 schemas/flatness.schema          |  7 -------
 schemas/sensor_abs_height.schema | 10 ----------
 schemas/sensor_flatness.schema   |  7 -------
 4 files changed, 36 deletions(-)
 delete mode 100644 schemas/absolute_height.schema
 delete mode 100644 schemas/flatness.schema
 delete mode 100644 schemas/sensor_abs_height.schema
 delete mode 100644 schemas/sensor_flatness.schema

diff --git a/schemas/absolute_height.schema b/schemas/absolute_height.schema
deleted file mode 100644
index fb30df3..0000000
--- a/schemas/absolute_height.schema
+++ /dev/null
@@ -1,12 +0,0 @@
-# -*- python -*-
-{
-    'schema_name' : 'absolute_height',
-    'schema_version' : 0,
-    'dzdx' : float,
-    'dzdy' : float,
-    'z0' : float,
-    'zmean' : float,
-    'znom' : float,
-    'znom_residual_025' : float,
-    'znom_residual_975' : float
-}
diff --git a/schemas/flatness.schema b/schemas/flatness.schema
deleted file mode 100644
index bc6de1f..0000000
--- a/schemas/flatness.schema
+++ /dev/null
@@ -1,7 +0,0 @@
-# -*- python -*-
-{
-    'schema_name' : 'flatness',
-    'schema_version' : 0,
-    'residual_025' : float,
-    'residual_975' : float
-}
diff --git a/schemas/sensor_abs_height.schema b/schemas/sensor_abs_height.schema
deleted file mode 100644
index dbd96fb..0000000
--- a/schemas/sensor_abs_height.schema
+++ /dev/null
@@ -1,10 +0,0 @@
-# -*- python -*-
-{
-    'schema_name' : 'sensor_abs_height',
-    'schema_version' : 0,
-    'z_median_m_13' : float,
-    'z_quantile_0975' : float,
-    'z_quantile_0025' : float,
-    'frac_outside' : float,
-    'flatnesshalfband_95' : float
-}
diff --git a/schemas/sensor_flatness.schema b/schemas/sensor_flatness.schema
deleted file mode 100644
index 271e767..0000000
--- a/schemas/sensor_flatness.schema
+++ /dev/null
@@ -1,7 +0,0 @@
-# -*- python -*-
-{
-    'schema_name' : 'sensor_flatness',
-    'schema_version' : 0,
-    'peak_valley_95' : float,
-    'flatnesshalfband_95' : float
-}