From 7aafec6e86d3d9b26c424793c5bf0c8f0370ff51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Ekeberg?= Date: Thu, 14 May 2020 01:14:02 +0200 Subject: [PATCH] Version 1.1.0 --- CHANGELOG.md | 16 +++++++++++ CONTRIBUTING.md | 4 +-- LICENSE | 2 +- README.md | 29 +++++++++++--------- docs/Experiment.md | 20 +++++++------- docs/Group.md | 24 ++++++++--------- docs/README.md | 2 +- docs/Result.md | 6 ++--- docs/Token.md | 4 +-- docs/User.md | 51 +++++++++++++++++++++++++++++++---- src/Experiment.php | 20 +++++++------- src/Group.php | 22 ++++++++-------- src/Result.php | 39 ++++++++++++++++----------- src/User.php | 66 ++++++++++++++++++++++++++++++++++++++-------- 14 files changed, 205 insertions(+), 100 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77ef30f..434ad56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,24 @@ All notable changes to this project will be documented in this file. +## [1.1.0] - 2020-05-14 + +### Added + +- Added [`User::hasViewed`](docs/User.md#userhasviewed) +- Added [`User::setViewed`](docs/User.md#usersetviewed) + +### Breaking changes + +- Updated [`User::addExperiment`](docs/User.md#useraddexperiment) to include a `$viewed` argument before `$converted` +- Renamed `Group::getSize` to [`Group::getViews`](docs/Group.md#groupgetviews) +- Renamed `Group::setSize` to [`Group::setViews`](docs/Group.md#groupsetviews) +- Renamed `Experiment::getCoverage` to [`Experiment::getAllocation`](docs/Experiment.md#experimentgetallocation) +- Renamed `Experiment::setCoverage` to [`Experiment::setAllocation`](docs/Experiment.md#experimentsetallocation) + ## [1.0.0] - 2020-04-29 Initial release +[1.1.0]: https://github.com/andreekeberg/abby/releases/tag/1.1.0 [1.0.0]: https://github.com/andreekeberg/abby/releases/tag/1.0.0 \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7cd5811..1e9a5e2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing to Abby -I want to make contributing to this project as easy and transparent as possible, whether it's: +This document contains basic guidelines to make contributing to this project as easy and transparent as possible, whether it's: - Reporting a bug - Discussing the current state of the code @@ -38,7 +38,7 @@ All bugs are tracked using GitHub issues to track public bugs. Report a bug by [ ## Use a Consistent Coding Style -I'm using the automatic formatter in the [PHP Intelephense](https://marketplace.visualstudio.com/items?itemName=bmewburn.vscode-intelephense-client) Visual Studio Code extension. +All code should follow the [PSR-12: Extended Coding Style](https://www.php-fig.org/psr/psr-12/) * 4 spaces for indentation rather than tabs * Newlines after opening curly brackets in classes and methods diff --git a/LICENSE b/LICENSE index f49a05f..4fbe095 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2020 André Ekeberg +Copyright (c) 2015-2020 André Ekeberg Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index f93fcdb..2e3414c 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ $data = [ [ 'id' => 1, 'group' => 1, + 'viewed' => true, 'converted' => false ] ]; @@ -64,9 +65,8 @@ foreach ($data as $item) { 'type' => $item['group'] ]); - // Add the experiment (including their group and whether they have - // already converted) to our user instance - $user->addExperiment($experiment, $group, $item['converted']); + // Add the experiment (including their group, and whether they have viewed and converted) + $user->addExperiment($experiment, $group, $item['viewed'], $item['converted']); } ``` @@ -105,7 +105,10 @@ $data = [ 'id' => 1 ]; -// If the user is part of the variation in our experiment +// Record a view for the experiment in question +$user->setViewed($data['id']); + +// If the user is part of the variation group if ($user->inVariation($data['id'])) { // Apply a custom class to an element, load a script, etc. } @@ -119,8 +122,8 @@ $data = [ 'id' => 1 ]; -// On a custom experiment goal, check if user is a participant and define a conversion -if ($user->isParticipant($data['id'])) { +// On a custom goal, check if user has viewed the experiment and define a conversion +if ($user->hasViewed($data['id'])) { $user->setConverted($data['id']); } @@ -138,12 +141,12 @@ $experiment = new Abby\Experiment([ 'groups' => [ [ 'name' => 'Control', - 'size' => 3000, + 'views' => 3000, 'conversions' => 300 ], [ 'name' => 'Variation', - 'size' => 3000, + 'views' => 3000, 'conversions' => 364 ] ] @@ -157,18 +160,18 @@ $winner = $result->getWinner(); /** * Get whether we can be confident of the result (even if we haven't - * reached the minimum group size for each variant) + * reached the minimum number of views for each variant) */ $confident = $result->isConfident(); /** - * Get the minimum sample size required for each group to reach statistical - * significance, given the control groups current conversion rate (based on - * the configured minimumDetectableEffect) + * Get the minimum sample size (number of views) required for each group to + * reach statistical significance, given the control groups current conversion + * rate (based on the configured minimumDetectableEffect) */ -$minimum = $result->getMinimumGroupSize(); +$minimum = $result->getMinimumSampleSize(); /** * Get whether the results are statistically significant diff --git a/docs/Experiment.md b/docs/Experiment.md index 46538a6..7780164 100644 --- a/docs/Experiment.md +++ b/docs/Experiment.md @@ -15,8 +15,8 @@ This class is in charge of managing your experiment, including configuring the n |[setControl](#experimentsetcontrol)|Define the control| |[getVariation](#experimentgetvariation)|Get the variation| |[setVariation](#experimentsetvariation)|Define the variation| -|[getCoverage](#experimentgetcoverage)|Get experiment coverage| -|[setCoverage](#experimentsetcoverage)|Set experiment coverage| +|[getAllocation](#experimentgetallocation)|Get experiment allocation| +|[setAllocation](#experimentsetallocation)|Set experiment allocation| |[getResult](#experimentgetresult)|Return a Result instance from the current experiment| ## Experiment::getID @@ -240,15 +240,15 @@ Define the variation
-## Experiment::getCoverage +## Experiment::getAllocation **Description** ```php -public getCoverage (void) +public getAllocation (void) ``` -Get experiment coverage +Get experiment allocation This is the percentual chance that a new user will be included in the experiment @@ -262,15 +262,15 @@ This is the percentual chance that a new user will be included in the experiment
-## Experiment::setCoverage +## Experiment::setAllocation **Description** ```php -public setCoverage (int $percent) +public setAllocation (int $percent) ``` -Set experiment coverage +Set experiment allocation This is the percentual chance that a new user will be included in the experiment @@ -300,6 +300,4 @@ Return a Result instance from the current experiment **Return Values** -`Result` - -
\ No newline at end of file +`Result` \ No newline at end of file diff --git a/docs/Group.md b/docs/Group.md index 3831c35..1494ebf 100644 --- a/docs/Group.md +++ b/docs/Group.md @@ -1,6 +1,6 @@ # Group -This class is in charge of creating and handling the control and variation groups of an experiment, including getting and setting their values such as group type, size, number of conversions, and whether the group is the winning or losing variant. +This class is in charge of creating and handling the control and variation groups of an experiment, including getting and setting their values such as group type, numbers of views and conversions, and whether the group is the winning or losing variant. | Name | Description | |------|-------------| @@ -9,8 +9,8 @@ This class is in charge of creating and handling the control and variation group |[setValue](#groupsetvalue)|Set property value of group| |[getName](#groupgetname)|Get group name| |[setName](#groupsetname)|Set group name| -|[getSize](#groupgetsize)|Get group size| -|[setSize](#groupsetsize)|Set group size| +|[getViews](#groupgetviews)|Get group views| +|[setViews](#groupsetviews)|Set group views| |[getConversions](#groupgetconversions)|Get number of conversions for the group| |[setConversions](#groupsetconversions)|Set number of conversions for the group| |[getConversionRate](#groupgetconversionrate)|Get conversion rate for the group| @@ -127,15 +127,15 @@ Set group name
-## Group::getSize +## Group::getViews **Description** ```php -public getSize (void) +public getViews (void) ``` -Get group size +Get group views **Parameters** @@ -147,19 +147,19 @@ Get group size
-## Group::setSize +## Group::setViews **Description** ```php -public setSize (int $size) +public setViews (int $views) ``` -Set group size +Set group views **Parameters** -* `(int) $size` +* `(int) $views` **Return Values** @@ -445,6 +445,4 @@ Define the group as the variation **Return Values** -`self` - -
\ No newline at end of file +`self` \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index 8f61829..a6e1304 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ # Documentation -Full documentation of all the available classes and methods +Full documentation of all the available classes and methods. * [Experiment](Experiment.md) * [Token](Token.md) diff --git a/docs/Result.md b/docs/Result.md index ef880a6..19dabca 100644 --- a/docs/Result.md +++ b/docs/Result.md @@ -235,7 +235,7 @@ public isSignificant (void) Return whether the result is statistically significant -This requires both the confidence (alternative hypothesis, i.e the inverse of the p-value) to be high enough (based on the specified minimum confidence) that the null hypothesis can be rejected, and the size of both groups to be greater than or equal to the minimum sample size (which is in turn calculated based on the minimum detectable effect, and the conversion rate of the control group) +This requires both the confidence (alternative hypothesis, i.e the inverse of the p-value) to be high enough (based on the specified minimum confidence) that the null hypothesis can be rejected, and the number of views for both groups to be greater than or equal to the minimum sample size (which is in turn calculated based on the minimum detectable effect, and the conversion rate of the control group) **Parameters** @@ -263,6 +263,4 @@ Get complete results **Return Values** -`object` - -
\ No newline at end of file +`object` \ No newline at end of file diff --git a/docs/Token.md b/docs/Token.md index da9126b..cdc8b0c 100644 --- a/docs/Token.md +++ b/docs/Token.md @@ -161,6 +161,4 @@ This sends a cookie to the users browser with the configured settings, but with **Return Values** -`self` - -
\ No newline at end of file +`self` \ No newline at end of file diff --git a/docs/User.md b/docs/User.md index 0a4f04c..9252a6e 100644 --- a/docs/User.md +++ b/docs/User.md @@ -14,6 +14,8 @@ This class is in charge of managing your users, including associating them with |[addExperiment](#useraddexperiment)|Add an experiment to the user's list of experiments| |[shouldParticipate](#usershouldparticipate)|Determine is user should be a participant in an experiment| |[assignGroup](#userassigngroup)|Get a group that the user should be asssiged to| +|[hasViewed](#userhasviewed)|Get whether a user has viewed a specific experiment| +|[setViewed](#usersetviewed)|Set whether a user has viewed a specific experiment| |[hasConverted](#userhasconverted)|Get whether a user has converted in a specific experiment| |[setConverted](#usersetconverted)|Set whether a user has converted in a specific experiment| @@ -233,6 +235,49 @@ If the experiments coverage is below 100, it's percentage value will be used to
+## User::hasViewed + +**Description** + +```php +public hasViewed (int $id) +``` + +Get whether a user has viewed a specific experiment + +**Parameters** + +* `(int) $id` +: Experiment ID + +**Return Values** + +`bool` + +
+ +## User::setViewed + +**Description** + +```php +public setViewed (int $id, bool $viewed) +``` + +Set whether a user has viewed a specific experiment + +**Parameters** + +* `(int) $id` +: Experiment ID +* `(bool) $viewed` + +**Return Values** + +`self` + +
+ ## User::hasConverted **Description** @@ -270,8 +315,4 @@ Set whether a user has converted in a specific experiment : Experiment ID * `(bool) $converted` -**Return Values** - -`self` - -
\ No newline at end of file +**Return Values** \ No newline at end of file diff --git a/src/Experiment.php b/src/Experiment.php index f3adb1e..a822785 100644 --- a/src/Experiment.php +++ b/src/Experiment.php @@ -6,7 +6,7 @@ class Experiment { private $id = null; private $name = null; - private $coverage = 100; + private $allocation = 100; private $groups = []; private $defaultGroups = [ [ @@ -52,9 +52,9 @@ public function __construct($config = []) $this->setName($config['name']); } - // Set coverage (if defined) - if (isset($config['coverage'])) { - $this->setCoverage($config['coverage']); + // Set allocation (if defined) + if (isset($config['allocation'])) { + $this->setAllocation($config['allocation']); } } @@ -238,17 +238,17 @@ public function setVariation($group) } /** - * Get experiment coverage + * Get experiment allocation * * @return int */ - public function getCoverage() + public function getAllocation() { - return $this->coverage; + return $this->allocation; } /** - * Set experiment coverage + * Set experiment allocation * * This is the percentual chance that a new user will be included in the experiment * @@ -256,13 +256,13 @@ public function getCoverage() * * @return self */ - public function setCoverage($percent) + public function setAllocation($percent) { if ($percent < 0 || $percent > 100) { throw new \Exception('Invalid $percent (value must be between 0 and 100)'); } - $this->coverage = round($percent); + $this->allocation = round($percent); return $this; } diff --git a/src/Group.php b/src/Group.php index be25633..d404168 100644 --- a/src/Group.php +++ b/src/Group.php @@ -18,7 +18,7 @@ public function __construct($group = []) { $this->group = array_merge([ 'type' => null, 'name' => null, - 'size' => 0, + 'views' => 0, 'conversions' => 0, 'winner' => null ], (array)$group); @@ -90,25 +90,25 @@ public function setName($name) } /** - * Get group size + * Get group views * * @return int */ - public function getSize() + public function getViews() { - return $this->getValue('size'); + return $this->getValue('views'); } /** - * Set group size + * Set group views * - * @param int $size + * @param int $views * * @return self */ - public function setSize($size) + public function setViews($views) { - return $this->setValue('size', $size); + return $this->setValue('views', $views); } /** @@ -140,14 +140,14 @@ public function setConversions($conversions) */ public function getConversionRate() { - $size = $this->getSize(); + $views = $this->getViews(); $conversions = $this->getConversions(); - if (!$size || !$conversions) { + if (!$views || !$conversions) { return 0; } - return $conversions / $size; + return $conversions / $views; } /** diff --git a/src/Result.php b/src/Result.php index 113f24c..8433041 100644 --- a/src/Result.php +++ b/src/Result.php @@ -49,16 +49,16 @@ private function calculate() $control = $this->experiment->getControl(); $variation = $this->experiment->getVariation(); - $controlSize = $control->getSize(); + $controlViews = $control->getViews(); $controlConversions = $control->getConversions(); $controlConversionRate = $control->getConversionRate(); - $variationSize = $variation->getSize(); + $variationViews = $variation->getViews(); $variationConversions = $variation->getConversions(); $variationConversionRate = $variation->getConversionRate(); if ( - ($controlSize != 0 && $variationSize != 0) && + ($controlViews != 0 && $variationViews != 0) && !($controlConversions == 0 && $variationConversions == 0) && $controlConversionRate !== $variationConversionRate ) { @@ -74,13 +74,13 @@ private function calculate() // If control is winner, flip experiment and control for the remaining calculations if ($winner === 0) { - list($controlSize, $variationSize) = array($variationSize, $controlSize); + list($controlViews, $variationViews) = array($variationViews, $controlViews); list($crA, $crB) = array($crB, $crA); } // Calculate standard error - $seA = sqrt(($crA * (1 - $crA)) / $controlSize); - $seB = sqrt(($crB * (1 - $crB)) / $variationSize); + $seA = sqrt(($crA * (1 - $crA)) / $controlViews); + $seB = sqrt(($crB * (1 - $crB)) / $variationViews); $seDiff = sqrt(pow($seA, 2) + pow($seB, 2)); @@ -105,18 +105,18 @@ private function calculate() * to consider the result statistically significant */ - $significant = min($controlSize, $variationSize) >= $sampleSize ? $confident : false; + $significant = min($controlViews, $variationViews) >= $sampleSize ? $confident : false; $this->winner = $winner; $this->confidence = $confidence; $this->confident = $confident; - $this->sampleSize = $sampleSize; + $this->sampleSize = $sampleSize; $this->significant = $significant; } else { $this->winner = null; $this->confidence = 0; $this->confident = false; - $this->sampleSize = INF; + $this->sampleSize = INF; $this->significant = false; } @@ -193,31 +193,40 @@ private function cdf($zScore, $mean, $std) */ private function calculateSampleSize($controlConversionRate, $minimumConfidence, $miminumEffect) { + // We can't calculate a sample size without a conversion rate, so return an infinte number if ($controlConversionRate <= 0) { return INF; } + // Set conficence and effect for our calculations $confidence = 1 - $minimumConfidence; $effect = $controlConversionRate * ($miminumEffect / 100); + // Create a two-tailed test based on the minimum confidence and effect $c1 = $controlConversionRate; $c2 = $controlConversionRate - $effect; $c3 = $controlConversionRate + $effect; - $t = abs($effect); + // Get absolute value of the effect + $theta = abs($effect); - $v1 = $c1 * (1 - $c1) + $c2 * (1 - $c2); - $v2 = $c1 * (1 - $c1) + $c3 * (1 - $c3); + // Create to variances by swapping $c1 and $c2 + $variance1 = $c1 * (1 - $c1) + $c2 * (1 - $c2); + $variance2 = $c1 * (1 - $c1) + $c3 * (1 - $c3); - $e1 = 2 * (1 - $confidence) * $v1 * log(1 + sqrt($v1) / $t) / ($t * $t); - $e2 = 2 * (1 - $confidence) * $v2 * log(1 + sqrt($v2) / $t) / ($t * $t); + // Look for the greatest absolute value of two possible sample estimates + $estimate1 = 2 * (1 - $confidence) * $variance1 * log(1 + sqrt($variance1) / $theta) / ($theta * $theta); + $estimate2 = 2 * (1 - $confidence) * $variance2 * log(1 + sqrt($variance2) / $theta) / ($theta * $theta); - $sampleSize = abs($e1) >= abs($e2) ? $e1 : $e2; + // Settle on the larger of the two calculated sizes to control the false discovery rate + $sampleSize = abs($estimate1) >= abs($estimate2) ? $estimate1 : $estimate2; + // If the calculated sample size is below zero or not a number, return an infinite number if ($sampleSize < 0 || !is_numeric($sampleSize)) { return INF; } + // Round result to a significant figure $n = round($sampleSize); $m = pow(10, 2 - floor(log($n) / log(10)) - 1); diff --git a/src/User.php b/src/User.php index d97edf4..16d73b8 100644 --- a/src/User.php +++ b/src/User.php @@ -179,8 +179,12 @@ public function inVariation($experiment) * * @return self */ - public function addExperiment(Experiment $experiment, $group = null, $converted = false) - { + public function addExperiment( + Experiment $experiment, + $group = null, + $viewed = false, + $converted = false + ) { // Setup user experiment array $userExperiment = (object)[ 'id' => $experiment->getID() @@ -189,8 +193,9 @@ public function addExperiment(Experiment $experiment, $group = null, $converted // Add group type if a group is set $userExperiment->group = $group ? $group->getType() : null; - // Only set converted value if the user is part of an experiment group + // Only set viewed and converted values if the user is part of an experiment group if ($group) { + $userExperiment->viewed = $viewed; $userExperiment->converted = $converted; } @@ -202,7 +207,7 @@ public function addExperiment(Experiment $experiment, $group = null, $converted /** * Determine is user should be a participant in an experiment, based on the - * experiments configured percentual coverage + * experiments configured percentual allocation * * @param Experiment $experiment * @@ -210,18 +215,18 @@ public function addExperiment(Experiment $experiment, $group = null, $converted */ public function shouldParticipate(Experiment $experiment) { - // Get experiments percentual coverage - $coverage = $experiment->getCoverage(); + // Get experiments percentual allocation + $allocation = $experiment->getAllocation(); - // Return true if coverage is 100% - if ($coverage == 100) { + // Return true if allocation is 100% + if ($allocation == 100) { return true; } // Create an array of a 100 positive and negative numbers, based - // on the value of $coverage + // on the value of $allocation $slots = str_split( - str_repeat(1, $coverage) . str_repeat(0, 100 - $coverage) + str_repeat(1, $allocation) . str_repeat(0, 100 - $allocation) ); // Shuffle all the slots @@ -235,7 +240,7 @@ public function shouldParticipate(Experiment $experiment) * Get a group that the used should be asssiged to, either the control * or variation (with a 50/50 chance) * - * If the experiments coverage is below 100, its percentage value will be + * If the experiments allocation is below 100, its percentage value will be * used to determine if the user is assigned an experiment group at all * * @param Experiment $experiment @@ -253,6 +258,45 @@ public function assignGroup(Experiment $experiment) } } + /** + * Get whether a user has viewed a specific experiment + * + * @param int $id Experiment ID + * + * @return bool + */ + public function hasViewed($id) + { + foreach ($this->experiments as $i => $experiment) { + if ($experiment->id == $id) { + if ($this->experiments[$i]->viewed) { + return true; + } + } + } + + return false; + } + + /** + * Set whether a user has viewed a specific experiment + * + * @param int $id Experiment ID + * @param bool $viewed + * + * @return self + */ + public function setViewed($id, $viewed = true) + { + foreach ($this->experiments as $i => $experiment) { + if ($experiment->id == $id) { + $this->experiments[$i]->viewed = $viewed; + } + } + + return $this; + } + /** * Get whether a user has converted in a specific experiment *