Add Two-way ANOVA without replication (#29)

* add * add * add * add * add * add * add * re * add * add * add * add * add * add * add * add * add * add * add * minor
kampersanda · Sep 28, 2024 · 77c6c9a · 77c6c9a
1 parent 93e2e71
commit 77c6c9a
Show file tree

Hide file tree

Showing 3 changed files with 810 additions and 18 deletions.
diff --git a/examples/two_way_anova_without_replication.rs b/examples/two_way_anova_without_replication.rs
@@ -0,0 +1,70 @@
+use anyhow::Result;
+use elinor::statistical_tests::TwoWayAnovaWithoutReplication;
+
+fn main() -> Result<()> {
+    // From Table 5.1 in Sakai's book, "情報アクセス評価方法論".
+    let a = vec![
+        0.70, 0.30, 0.20, 0.60, 0.40, 0.40, 0.00, 0.70, 0.10, 0.30, //
+        0.50, 0.40, 0.00, 0.60, 0.50, 0.30, 0.10, 0.50, 0.20, 0.10,
+    ];
+    let b = vec![
+        0.50, 0.10, 0.00, 0.20, 0.40, 0.30, 0.00, 0.50, 0.30, 0.30, //
+        0.40, 0.40, 0.10, 0.40, 0.20, 0.10, 0.10, 0.60, 0.30, 0.20,
+    ];
+    let c = vec![
+        0.00, 0.00, 0.20, 0.10, 0.30, 0.30, 0.10, 0.20, 0.40, 0.40, //
+        0.40, 0.30, 0.30, 0.20, 0.20, 0.20, 0.10, 0.50, 0.40, 0.30,
+    ];
+
+    let tupled_samples = a
+        .iter()
+        .zip(b.iter())
+        .zip(c.iter())
+        .map(|((&a, &b), &c)| [a, b, c]);
+    let stat = TwoWayAnovaWithoutReplication::from_tupled_samples(tupled_samples, 3)?;
+    println!("n_systems: {}", stat.n_systems());
+    println!("n_topics: {}", stat.n_topics());
+    println!(
+        "between_system_variation: {:.4}",
+        stat.between_system_variation()
+    );
+    println!(
+        "between_topic_variation: {:.4}",
+        stat.between_topic_variation()
+    );
+    println!("residual_variation: {:.4}", stat.residual_variation());
+    println!(
+        "between_system_variance: {:.4}",
+        stat.between_system_variance()
+    );
+    println!(
+        "between_topic_variance: {:.4}",
+        stat.between_topic_variance()
+    );
+    println!("residual_variance: {:.4}", stat.residual_variance());
+    println!("between_system_f_stat: {:.4}", stat.between_system_f_stat());
+    println!("between_topic_f_stat: {:.4}", stat.between_topic_f_stat());
+    println!(
+        "between_system_p_value: {:.4}",
+        stat.between_system_p_value()
+    );
+    println!("between_topic_p_value: {:.4}", stat.between_topic_p_value());
+
+    let moe95 = stat.margin_of_error(0.05)?;
+    let system_means = stat.system_means();
+    for (i, mean) in system_means.iter().enumerate() {
+        let ci95_btm = mean - moe95;
+        let ci95_top = mean + moe95;
+        println!("Mean and 95% CI of system {i}: {mean:.4} [{ci95_btm:.4}, {ci95_top:.4}]");
+    }
+
+    let effect_sizes = stat.between_system_effect_sizes();
+    for i in 0..stat.n_systems() {
+        for j in (i + 1)..stat.n_systems() {
+            let effect_size = effect_sizes[i][j];
+            println!("Effect size between system {i} and {j}: {effect_size:.4}");
+        }
+    }
+
+    Ok(())
+}
diff --git a/src/statistical_tests.rs b/src/statistical_tests.rs
@@ -5,6 +5,7 @@
 //! * [Student's t-test](StudentTTest) for comparing two systems.
 //! * [Bootstrap test](BootstrapTest) for comparing two systems.
 //! * [Randomized Tukey HSD test](RandomizedTukeyHsdTest) for comparing two or more systems.
+//! * [Two-way ANOVA without replication](TwoWayAnovaWithoutReplication) for comparing three or more systems.
 //!
 //! # Example: Statistical tests for comparing two systems
 //!
@@ -47,26 +48,26 @@
 //!
 //! // Perform Student's t-test.
 //! let paired_scores = elinor::paired_scores_from_evaluated(&evaluated_a, &evaluated_b)?;
-//! let result = StudentTTest::from_paired_samples(paired_scores)?;
+//! let stat = StudentTTest::from_paired_samples(paired_scores)?;
 //!
 //! // Various statistics can be obtained from the t-test result.
-//! assert!(result.mean() > 0.0);
-//! assert!(result.var() > 0.0);
-//! assert!(result.effect_size() > 0.0);
-//! assert!(result.t_stat() > 0.0);
-//! assert!((0.0..=1.0).contains(&result.p_value()));
+//! assert!(stat.mean() > 0.0);
+//! assert!(stat.var() > 0.0);
+//! assert!(stat.effect_size() > 0.0);
+//! assert!(stat.t_stat() > 0.0);
+//! assert!((0.0..=1.0).contains(&stat.p_value()));
 //!
 //! // Margin of error at a 95% confidence level.
-//! let moe95 = result.margin_of_error(0.05)?;
+//! let moe95 = stat.margin_of_error(0.05)?;
 //! assert!(moe95 > 0.0);
 //!
 //! // Confidence interval at a 95% confidence level.
-//! let (ci95_btm, ci95_top) = result.confidence_interval(0.05)?;
-//! assert_relative_eq!(ci95_btm, result.mean() - moe95);
-//! assert_relative_eq!(ci95_top, result.mean() + moe95);
+//! let (ci95_btm, ci95_top) = stat.confidence_interval(0.05)?;
+//! assert_relative_eq!(ci95_btm, stat.mean() - moe95);
+//! assert_relative_eq!(ci95_top, stat.mean() + moe95);
 //!
 //! // Check if the difference is significant at a 95% confidence level.
-//! assert_eq!(result.is_significant(0.05), result.p_value() <= 0.05);
+//! assert_eq!(stat.is_significant(0.05), stat.p_value() <= 0.05);
 //! # Ok(())
 //! # }
 //! ```
@@ -78,7 +79,7 @@
 //! ```
 //! # fn main() -> Result<(), Box<dyn std::error::Error>> {
 //! use elinor::{GoldRelStoreBuilder, PredRelStoreBuilder, Metric};
-//! use elinor::statistical_tests::RandomizedTukeyHsdTest;
+//! use elinor::statistical_tests::{RandomizedTukeyHsdTest, TwoWayAnovaWithoutReplication};
 //!
 //! // Prepare gold relevance scores.
 //! let mut b = GoldRelStoreBuilder::new();
@@ -110,25 +111,39 @@
 //! b.add_score("q_2", "d_4", 0.2.into())?;
 //! let pred_rels_c = b.build();
 //!
-//! // Evaluate Precision for both systems.
+//! // Evaluate Precision for all systems.
 //! let metric = Metric::Precision { k: 0 };
 //! let evaluated_a = elinor::evaluate(&gold_rels, &pred_rels_a, metric)?;
 //! let evaluated_b = elinor::evaluate(&gold_rels, &pred_rels_b, metric)?;
 //! let evaluated_c = elinor::evaluate(&gold_rels, &pred_rels_c, metric)?;
 //!
-//! // Perform Randomized Tukey HSD test.
+//! // Perform Randomized Tukey HSD test and Two-way ANOVA without replication.
 //! let tupled_scores = elinor::tupled_scores_from_evaluated(&[evaluated_a, evaluated_b, evaluated_c])?;
-//! let result = RandomizedTukeyHsdTest::from_tupled_samples(tupled_scores, 3)?;
-//! assert!((0.0..=1.0).contains(&result.p_value(0, 1)?));  // A vs. B
-//! assert!((0.0..=1.0).contains(&result.p_value(0, 2)?));  // A vs. C
-//! assert!((0.0..=1.0).contains(&result.p_value(1, 2)?));  // B vs. C
+//! let hsd_stat = RandomizedTukeyHsdTest::from_tupled_samples(tupled_scores.iter(), 3)?;
+//! let anova_stat = TwoWayAnovaWithoutReplication::from_tupled_samples(tupled_scores.iter(), 3)?;
+//!
+//! // p-values and effect sizes for all pairs of systems.
+//! let effect_sizes = anova_stat.between_system_effect_sizes();
+//! for (i, j) in [(0, 1), (0, 2), (1, 2)] {
+//!     assert!((0.0..=1.0).contains(&hsd_stat.p_value(i, j)?));
+//!     assert!(effect_sizes[i][j] != 0.0);
+//! }
+//!
+//! // 95% CI of system means.
+//! let moe95 = anova_stat.margin_of_error(0.05)?;
+//! let system_means = anova_stat.system_means();
+//! for (i, mean) in system_means.iter().enumerate() {
+//!     assert!(mean - moe95 <= mean + moe95);
+//! }
 //! # Ok(())
 //! # }
 //! ```
 pub mod bootstrap_test;
 pub mod randomized_tukey_hsd_test;
 pub mod student_t_test;
+pub mod two_way_anova_without_replication;
 
 pub use bootstrap_test::BootstrapTest;
 pub use randomized_tukey_hsd_test::RandomizedTukeyHsdTest;
 pub use student_t_test::StudentTTest;
+pub use two_way_anova_without_replication::TwoWayAnovaWithoutReplication;