forked from awslabs/deequ
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Adding Wilson Score Confidence Interval Strategy (awslabs#567)
* Configurable RetainCompletenessRule * Add doc string * Add default completeness const * Add ConfidenceIntervalStrategy * Add Separate Wilson and Wald Interval Test * Add License information, Fix formatting * Add License information * formatting fix * Update documentation * Make WaldInterval the default strategy for now * Formatting import to per line * Separate group import to per line import
- Loading branch information
Showing
9 changed files
with
299 additions
and
54 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
55 changes: 55 additions & 0 deletions
55
src/main/scala/com/amazon/deequ/suggestions/rules/interval/ConfidenceIntervalStrategy.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| /** | ||
| * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"). You may not | ||
| * use this file except in compliance with the License. A copy of the License | ||
| * is located at | ||
| * | ||
| * http://aws.amazon.com/apache2.0/ | ||
| * | ||
| * or in the "license" file accompanying this file. This file is distributed on | ||
| * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either | ||
| * express or implied. See the License for the specific language governing | ||
| * permissions and limitations under the License. | ||
| * | ||
| */ | ||
|
|
||
| package com.amazon.deequ.suggestions.rules.interval | ||
|
|
||
| import breeze.stats.distributions.{Gaussian, Rand} | ||
| import com.amazon.deequ.suggestions.rules.interval.ConfidenceIntervalStrategy._ | ||
|
|
||
| /** | ||
| * Strategy for calculate confidence interval | ||
| * */ | ||
| trait ConfidenceIntervalStrategy { | ||
|
|
||
| /** | ||
| * Generated confidence interval interval | ||
| * @param pHat sample of the population that share a trait | ||
| * @param numRecords overall number of records | ||
| * @param confidence confidence level of method used to estimate the interval. | ||
| * @return | ||
| */ | ||
| def calculateTargetConfidenceInterval( | ||
| pHat: Double, | ||
| numRecords: Long, | ||
| confidence: Double = defaultConfidence | ||
| ): ConfidenceInterval | ||
|
|
||
| def validateInput(pHat: Double, confidence: Double): Unit = { | ||
| require(0.0 <= pHat && pHat <= 1.0, "pHat must be between 0.0 and 1.0") | ||
| require(0.0 <= confidence && confidence <= 1.0, "confidence must be between 0.0 and 1.0") | ||
| } | ||
|
|
||
| def calculateZScore(confidence: Double): Double = Gaussian(0, 1)(Rand).inverseCdf(1 - ((1.0 - confidence)/ 2.0)) | ||
| } | ||
|
|
||
| object ConfidenceIntervalStrategy { | ||
| val defaultConfidence = 0.95 | ||
| val defaultIntervalStrategy: ConfidenceIntervalStrategy = WaldIntervalStrategy() | ||
|
|
||
| case class ConfidenceInterval(lowerBound: Double, upperBound: Double) | ||
| } | ||
|
|
||
|
|
47 changes: 47 additions & 0 deletions
47
src/main/scala/com/amazon/deequ/suggestions/rules/interval/WaldIntervalStrategy.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| /** | ||
| * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"). You may not | ||
| * use this file except in compliance with the License. A copy of the License | ||
| * is located at | ||
| * | ||
| * http://aws.amazon.com/apache2.0/ | ||
| * | ||
| * or in the "license" file accompanying this file. This file is distributed on | ||
| * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either | ||
| * express or implied. See the License for the specific language governing | ||
| * permissions and limitations under the License. | ||
| * | ||
| */ | ||
|
|
||
| package com.amazon.deequ.suggestions.rules.interval | ||
|
|
||
| import com.amazon.deequ.suggestions.rules.interval.ConfidenceIntervalStrategy.ConfidenceInterval | ||
| import com.amazon.deequ.suggestions.rules.interval.ConfidenceIntervalStrategy.defaultConfidence | ||
|
|
||
| import scala.math.BigDecimal.RoundingMode | ||
|
|
||
| /** | ||
| * Implements the Wald Interval method for creating a binomial proportion confidence interval. Provided for backwards | ||
| * compatibility. using [[WaldIntervalStrategy]] for calculating confidence interval can be problematic when dealing | ||
| * with small sample sizes or proportions close to 0 or 1. It also have poorer coverage and might produce confidence | ||
| * limit outside the range of [0,1] | ||
| * @see <a | ||
| * href="http://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval#Normal_approximation_interval"> | ||
| * Normal approximation interval (Wikipedia)</a> | ||
| */ | ||
| @deprecated("WilsonScoreIntervalStrategy is recommended for calculating confidence interval") | ||
| case class WaldIntervalStrategy() extends ConfidenceIntervalStrategy { | ||
| def calculateTargetConfidenceInterval( | ||
| pHat: Double, | ||
| numRecords: Long, | ||
| confidence: Double = defaultConfidence | ||
| ): ConfidenceInterval = { | ||
| validateInput(pHat, confidence) | ||
| val successRatio = BigDecimal(pHat) | ||
| val marginOfError = BigDecimal(calculateZScore(confidence) * math.sqrt(pHat * (1 - pHat) / numRecords)) | ||
| val lowerBound = (successRatio - marginOfError).setScale(2, RoundingMode.DOWN).toDouble | ||
| val upperBound = (successRatio + marginOfError).setScale(2, RoundingMode.UP).toDouble | ||
| ConfidenceInterval(lowerBound, upperBound) | ||
| } | ||
| } |
47 changes: 47 additions & 0 deletions
47
src/main/scala/com/amazon/deequ/suggestions/rules/interval/WilsonScoreIntervalStrategy.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| /** | ||
| * Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"). You may not | ||
| * use this file except in compliance with the License. A copy of the License | ||
| * is located at | ||
| * | ||
| * http://aws.amazon.com/apache2.0/ | ||
| * | ||
| * or in the "license" file accompanying this file. This file is distributed on | ||
| * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either | ||
| * express or implied. See the License for the specific language governing | ||
| * permissions and limitations under the License. | ||
| * | ||
| */ | ||
|
|
||
| package com.amazon.deequ.suggestions.rules.interval | ||
|
|
||
| import com.amazon.deequ.suggestions.rules.interval.ConfidenceIntervalStrategy.ConfidenceInterval | ||
| import com.amazon.deequ.suggestions.rules.interval.ConfidenceIntervalStrategy.defaultConfidence | ||
|
|
||
| import scala.math.BigDecimal.RoundingMode | ||
|
|
||
| /** | ||
| * Using Wilson score method for creating a binomial proportion confidence interval. | ||
| * | ||
| * @see <a | ||
| * href="http://en.wikipedia.org/wiki/Binomial_proportion_confidence_interval#Wilson_score_interval"> | ||
| * Wilson score interval (Wikipedia)</a> | ||
| */ | ||
| case class WilsonScoreIntervalStrategy() extends ConfidenceIntervalStrategy { | ||
|
|
||
| def calculateTargetConfidenceInterval( | ||
| pHat: Double, numRecords: Long, | ||
| confidence: Double = defaultConfidence | ||
| ): ConfidenceInterval = { | ||
| validateInput(pHat, confidence) | ||
| val zScore = calculateZScore(confidence) | ||
| val zSquareOverN = math.pow(zScore, 2) / numRecords | ||
| val factor = 1.0 / (1 + zSquareOverN) | ||
| val adjustedSuccessRatio = pHat + zSquareOverN/2 | ||
| val marginOfError = zScore * math.sqrt(pHat * (1 - pHat)/numRecords + zSquareOverN/(4 * numRecords)) | ||
| val lowerBound = BigDecimal(factor * (adjustedSuccessRatio - marginOfError)).setScale(2, RoundingMode.DOWN).toDouble | ||
| val upperBound = BigDecimal(factor * (adjustedSuccessRatio + marginOfError)).setScale(2, RoundingMode.UP).toDouble | ||
| ConfidenceInterval(lowerBound, upperBound) | ||
| } | ||
| } |
Oops, something went wrong.