#1: Add constraint importance and data recovery plots

README.md

@@ -71,15 +71,18 @@ let tmeaResult =
 
 # Plots
 
+All plot functions have a `generate*` analog, which generates the Chart object without rendering it (in case you want to fine tune styles etc.).
 Currently, the following plots are provided by the package:
 
-**Surprisal Analysis:**
+### Surprisal Analysis:
 
 Given the following example input:
 
 ```F#
+open TMEA
+
 let data =
-    TMEA.IO.readDataFrame 
+    IO.readDataFrame 
         "TranscriptIdentifier" 
         "\t"
         @"path/to/data.txt"
@@ -88,40 +91,66 @@ let data =
     |> JaggedArray.transpose
 
 let SaRes = 
-    TMEA.IO.readDataFrame 
+    IO.readDataFrame 
         "TranscriptIdentifier" 
         "\t"
         @"path/to/data.txt"
-    |> TMEA.SurprisalAnalysis.computeOfDataFrame
+    |> SurprisalAnalysis.computeOfDataFrame
 ```
 
- - `TMEA.Plots.SurprisalAnalysis.plotConstraintTimecourses` plots the constraint potential time courses of the given surprisal analysis result:
+##### Potential Time Course:
+
+ - `plotConstraintTimecourses` plots the constraint potential time courses of the given surprisal analysis result:
 
     ```F#
     SaRes
-    |> TMEA.Plots.SurprisalAnalysis.plotConstraintTimecourses true //true -> will use style presets
+    |> Plots.SurprisalAnalysis.plotConstraintTimecourses true //true -> will use style presets
     ```
 
     ![](./docs/img/cpTimeCourse.png)
 
- - `TMEA.Plots.SurprisalAnalysis.plotPotentialHeatmap` is a more visually pleasing version of above plot (it omits the baseline state per default):
+ - `plotPotentialHeatmap` is a more visually pleasing version of above plot (it omits the baseline state per default):
 
     ```F#
     SaRes
-    |> TMEA.Plots.SurprisalAnalysis.plotPotentialHeatmap true
+    |> Plots.SurprisalAnalysis.plotPotentialHeatmap true
     ```
 
     ![](./docs/img/cpHeatmap.png)
 
- - `Plots.SurprisalAnalysis.plotFreeEnergyLandscape` plots the free energy landscape of the given surprisal analysis result:
+##### Free Energy Landscape:
+
+ - `plotFreeEnergyLandscape` plots the free energy landscape of the given surprisal analysis result:
 
     ```
     SaRes
-    |> TMEA.Plots.SurprisalAnalysis.plotFreeEnergyLandscape true data
+    |> Plots.SurprisalAnalysis.plotFreeEnergyLandscape true data
     ```
 
     ![](./docs/img/EnergyLandscape.png)
 
+##### Constraint importance:
+
+ - `plotConstraintImportance` plots the singular values of all constraints (except the baseline state) and the 'importance loss' between them. 
+
+    ```
+    SaRes
+    |> Plots.SurprisalAnalysis.plotConstraintImportance true
+    ```
+
+    ![](./docs/img/ConstraintImportance.png)
+
+##### Data recovery:
+
+ - `plotDataRecovery` plots the gradual reconstruction of the original data when using only n (in the example below, n = 3) constraints from the given Surprisal Analysis result:
+
+    ```
+    SaRes
+    |> Plots.SurprisalAnalysis.plotDataRecovery true 3 data
+    ```
+
+    ![](./docs/img/DataRecovery.png)
+
 # License acknowlegments
 
 This library contains [Netlib LAPACK](http://www.netlib.org/lapack/) binaries compiled from source, thanks to all the authors of it:

RELEASE_NOTES.md

@@ -1,5 +1,8 @@
 #### 0.0.3-alpha - Friday, September 4, 2020 (WIP)
 - added Surprisal Analysis cp Heatmap Plot
+- added Surprisal Analysis analytic plots: 
+  - Constraint importance
+  - Gradual data recovery
 
 #### 0.0.2-alpha - Wednesday, September 2, 2020
 - added Plot functions for Surprisal Analysis results.

src/TMEA/Playground.fsx

@@ -156,6 +156,12 @@ SaRes
 SaRes
 |> TMEA.Plots.SurprisalAnalysis.plotFreeEnergyLandscape true data
 
+SaRes
+|> TMEA.Plots.SurprisalAnalysis.plotConstraintImportance true
+
+SaRes
+|> TMEA.Plots.SurprisalAnalysis.plotDataRecovery true 3 data
+
 readDataFrame 
     "TranscriptIdentifier" 
     "\t"

src/TMEA/Plots.fs

@@ -34,7 +34,7 @@ module Plots =
 
     module SurprisalAnalysis =
 
-        ///generates a Plot object containing the constraint time courses of the given Surprisal Analysis result
+        ///generates a Chart object containing the constraint time courses of the given Surprisal Analysis result
         let generateConstraintTimeCoursePlot (useStylePreset:bool) (saRes:SurprisalAnalysis.SAResult) =
             saRes.Potentials
             |> Matrix.toJaggedArray
@@ -53,13 +53,12 @@ module Plots =
                     |> Chart.withY_AxisStyle "λ(t)"
                     |> Chart.withTitle "Constraint Potential TimeCourse"
 
-
-        ///generates a Plot object containing the constraint time courses of the given Surprisal Analysis result and renders it in the browser
+        ///generates a Chart object containing the constraint time courses of the given Surprisal Analysis result and renders it in the browser
         let plotConstraintTimecourses (useStylePreset:bool) (saRes:SurprisalAnalysis.SAResult) =
              generateConstraintTimeCoursePlot useStylePreset saRes
              |> Chart.Show
 
-        ///generates a Plot object containing the free energy landscape of the given Surprisal Analysis result
+        ///generates a Chart object containing the free energy landscape of the given Surprisal Analysis result
         let generateFreeEnergyLandscapePlot (useStylePreset:bool) (data:float [] []) (saRes:SurprisalAnalysis.SAResult) =
             let freeEnergies =
                 [|1..data.Length-1|]
@@ -90,24 +89,23 @@ module Plots =
                     |> Chart.withY_AxisStyle "Free Energy / k<sub>b</sub>T"
                     |> Chart.withTitle "Free Energy Landscape"
 
-        ///generates a Plot object containing the free energy landscape of the given Surprisal Analysis result and renders it in the browser
+        ///generates a Chart object containing the free energy landscape of the given Surprisal Analysis result and renders it in the browser
         let plotFreeEnergyLandscape (useStylePreset:bool) (data:float [] []) (saRes:SurprisalAnalysis.SAResult) =
             generateFreeEnergyLandscapePlot useStylePreset data saRes
             |> Chart.Show
 
-
+        ///generates a Chart object containing the constraint time courses of the given Surprisal Analysis result via heatmap. omits the baseline state.
         let generatePotentialHeatmap (useStylePreset:bool) (saRes:SurprisalAnalysis.SAResult) =
             saRes.Potentials
                |> Matrix.toArray2D
                |> JaggedArray.ofArray2D
                |> Array.tail
                |> Array.rev
-               |> JaggedArray.map (fun x -> x*10.)
                |> fun potentials ->
                     Chart.Heatmap(
                         potentials,
                         ColNames=[0 .. potentials.Length-1],
-                        RowNames = ([1..potentials.Length-1] |> List.map (sprintf "Constraint_%i") |> List.rev),
+                        RowNames = ([1..potentials.Length] |> List.map (sprintf "Constraint_%i") |> List.rev),
                         Ygap=10,
                         Colorscale= Presets.colorscale,zSmooth=StyleParam.SmoothAlg.Best
                         )
@@ -123,8 +121,142 @@ module Plots =
                    else
                        c
 
-
-
+        ///generates a Chart object containing the constraint time courses of the given Surprisal Analysis result via heatmap and renders it in the browser. omits the baseline state.
         let plotPotentialHeatmap (useStylePreset:bool) (saRes:SurprisalAnalysis.SAResult) =
             generatePotentialHeatmap useStylePreset saRes
+            |> Chart.Show
+
+        ///generates a Chart object containing charts to help with selection of an importance threshold for constraints of the given surprisal analysis result.
+        let generateConstraintImportancePlot (useStylePreset:bool) (saRes:SurprisalAnalysis.SAResult) =
+            [
+                saRes.SingularValues 
+                |> Vector.toArray 
+                |> Array.indexed
+                |> Array.tail
+                |> Array.mapi (fun i (cI,x) -> 
+                    Chart.Column([(i+1),x])
+                    |> Chart.withTraceName (sprintf "σ(C_%i)" (i+1))
+                )
+                |> Chart.Combine
+                |> fun c -> 
+                    if useStylePreset then
+                        c
+                        |> Presets.applyPresetStyle "ConstraintIndex α" "Singular Value σ(α)"
+                    else
+                        c
+
+                saRes.SingularValues 
+                |> Vector.toArray 
+                |> Array.tail
+                |> fun a -> 
+                    a 
+                    |> Array.mapi (fun i x -> 
+                        [(i+1) , (
+                            if (i=0) then 
+                                0. 
+                            else
+                                1. - (x/a.[i-1])
+                        )]
+                        |> fun c -> Chart.Column(c)
+                        |> Chart.withTraceName (sprintf "σ loss (C_%i)" (i+1))
+
+                    )
+                |> Chart.Combine
+                |> fun c -> 
+                    if useStylePreset then
+                        c
+                        |> Presets.applyPresetStyle "ConstraintIndex α" "σ loss (1 - (σ(α) / σ(α-1))"
+                    else
+                        c
+            ]
+            |> List.rev
+            |> fun x -> Chart.SingleStack (x,true)
+            |> Chart.withTitle "Constraint importance"
+            |> fun c -> 
+                if useStylePreset then
+                    c
+                    |> Chart.withSize(750.,1000.)
+                    |> Chart.withConfig Presets.standardConfig
+                else
+                    c
+
+        ///generates a Chart object containing charts to help with selection of an importance threshold for constraints of the given surprisal analysis result and renders it in the browser.
+        let plotConstraintImportance (useStylePreset:bool) (saRes:SurprisalAnalysis.SAResult) =
+            generateConstraintImportancePlot useStylePreset saRes
+            |> Chart.Show
+
+        ///generates a Chart object containing a plot that shows the gradual reconstruction of the original data when using only n constraints from the given Surprisal Analysis result.
+        let generateDataRecoveryPlot (useStylePreset:bool) (constraintCutoff:int) (data:float[][]) (saRes:SurprisalAnalysis.SAResult) =
+
+            let patterns =
+                saRes.MolecularPhenotypes
+                |> Matrix.toJaggedArray
+                |> JaggedArray.transpose
+
+            let potentials = 
+                saRes.Potentials
+                |> Matrix.toJaggedArray
+
+            [0 .. constraintCutoff]
+            |> List.map (fun recIndex ->
+                patterns.[0..10]
+                |> Array.mapi (fun i pattern ->
+                    pattern
+                    |> Array.map (fun g ->
+                        potentials.[i] |> Array.map (fun p -> p * g)
+                    )
+                )
+                |> fun x ->
+                    let outerLen = x.[0].Length
+                    let innerLen = x.[0].[0].Length
+                    Array.init outerLen (fun outerI ->
+                        Array.init innerLen (fun innerI ->
+                            x.[0..recIndex]
+                            |> Array.map (fun r ->
+                                r.[outerI].[innerI]
+                            )
+                            |> Array.sum
+                        )
+                    )
+                |> JaggedArray.transpose
+                |> Array.concat
+                |> fun x ->  Array.zip x (data |> Array.concat)
+                |> fun xy -> 
+                    let xy' = xy |> Array.filter (fun (x,y) -> not (nan.Equals(x) || nan.Equals(y) || infinity.Equals(-x) || infinity.Equals(-y) || infinity.Equals(x) || infinity.Equals(y)))
+                    let x,y = xy' |> Array.unzip
+                    let coeffs = FSharp.Stats.Fitting.LinearRegression.OrdinaryLeastSquares.Polynomial.coefficient 1 (x|> vector) (y|> vector)
+                    let fit = FSharp.Stats.Fitting.LinearRegression.OrdinaryLeastSquares.Linear.Univariable.fit coeffs
+                    let sos = FSharp.Stats.Fitting.GoodnessOfFit.calculateSumOfSquares fit x y
+                    let rSquared = FSharp.Stats.Fitting.GoodnessOfFit.calculateDetermination sos
+
+                    xy,rSquared
+
+                |> fun (xy,rS) -> 
+                    Chart.Point(xy,UseWebGL=true,Name= (sprintf "C0-%i : R<sup>2</sup> = %.5f" recIndex rS)) 
+                    |> Chart.withTitle (sprintf "R² = %.5f" rS)
+                    |> Chart.withMarkerStyle (
+                        Size = 3,
+                        Opacity = 0.7,
+                        Symbol = StyleParam.Symbol.Star
+
+                    )
+                |> fun c -> 
+                    if useStylePreset then
+                        c |> Presets.applyPresetStyle "&#8721;<sup>&#945;</sup>(&#955;<sub>&#945;</sub>G<sub>i&#945;</sub>)" "ln(X<sub>i</sub>)"
+                    else
+                        c
+            )
+            |> Chart.Combine
+            |> Chart.withTitle (sprintf "Data recovery using Constraints 0-%i" constraintCutoff)
+            |> fun c ->
+                if useStylePreset then
+                    c
+                    |> Chart.withSize (1000.,1000.)
+                    |> Chart.withConfig Presets.standardConfig
+                else
+                    c
+
+        ///generates a Chart object containing a plot that shows the gradual reconstruction of the original data when using only n constraints from the given Surprisal Analysis result and renders it in the browser
+        let plotDataRecovery (useStylePreset:bool) (constraintCutoff:int) (data:float[][]) (saRes:SurprisalAnalysis.SAResult) =
+            generateDataRecoveryPlot useStylePreset constraintCutoff data saRes
             |> Chart.Show