diff --git a/README.md b/README.md index d264e3a..05e96ac 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Table of Contents * [installation](#installation) * [type declarations](#type-declarations) * [annotations](#annotations) + * [tabulate](#tabulate) * [generators](#generators) * [assertions](#assertions) * [examples](#examples) @@ -48,6 +49,79 @@ Parameter tags (use them in the description of a parameter, usually the end): Method tags: * `@maxSuccess` sets the number of successful checks for a successful result. The default is 100. +* `@tabulate` and `@coverTable` (dicussed below). + +### tabulate + +If you decorate a check method with `tabulate`, information about test case distribution is collected into a table. + +The arguments to `tabulate` are the table's name and a _list_ of values associated with the current check. An example: + + /** + * @tabulate "Values" [$value] + */ + public function checkBooleans(bool $value) + { + return true; + } + +If you run this check, everything passes and a table is output at the end of the check run: + + Tables + + 1) Datashaman\PHPCheck\Checks\GeneratorCheck::checkBooleans + + Values (100 total) + + 52% true + 48% false + +We'd like to check that the coverage is correct for the generator, so we use a `@coverTable` method annotation: + + /** + * @coverTable "Values" [[true, 49], [false, 49]] + * @tabulate "Values" [$value] + */ + public function checkBooleans(bool $value) + { + return true; + } + +The arguments to the annotation are the name of the table, and a list of key value pairs where the value is the expected percentage distribution. + +Here's a sample output from the above: + + 1) Datashaman\PHPCheck\Checks\GeneratorCheck::checkBooleans + + Values (100 total) + + 54% true + 46% false + + Table 'Values' had only 46.0% false, but expected 49.0% + +We are now warned when the distribution does not fall within the accepted percentage of generated values. + +In this case, we would benefit from running the checks a lot more times so we approach the expected _50/50_ average for a boolean: + + /** + * @coverTable "Values" [[true, 49], [false, 49]] + * @maxSuccess 10000 + * @tabulate "Values" [$value] + */ + public function checkBooleans(bool $value) + { + return true; + } + +Now with _10000_ successful iterations, the warning disappears from the output and the percentage is within the acceptable _1%_ margin of error: + + 1) Datashaman\PHPCheck\Checks\GeneratorCheck::checkBooleans + + Values (10000 total) + + 50.5% true + 49.5% false ## generators @@ -174,9 +248,10 @@ The `phpcheck` program accept a number of arguments and options: --coverage-html[=COVERAGE-HTML] Generate HTML code coverage report [default: false] --coverage-text[=COVERAGE-TEXT] Generate text code coverage report [default: false] -f, --filter[=FILTER] Filter the checks that will be run - -i, --iterations=ITERATIONS How many times each check will be run [default: 100] -j, --log-junit[=LOG-JUNIT] Log check execution to JUnit XML file [default: false] -t, --log-text[=LOG-TEXT] Log check execution to text file [default: false] + --max-success=MAX-SUCCESS Maximum number of successful checks before succeeding. Testing stops at the first failure. + If all tests are passing and you want to run more tests, increase this number. [default: 100] -d, --no-defects[=NO-DEFECTS] Ignore previous defects [default: false] -h, --help Display this help message -q, --quiet Do not output any message @@ -244,8 +319,6 @@ Using `---verbose 3` or `-vvv` enables a list of the checks as they are run: OK (Checks: 13, Iterations: 120006, Failures: 0, Errors: 0) -The above output is from _10000_ successes per check. The heavy use of generators throughout the architecture ensures low memory usage throughout the run process despite large numbers of iterations. - ## storage of results `PHPCheck` stores results of check execution in the `.phpcheck` folder of the project. diff --git a/src/CheckCommand.php b/src/CheckCommand.php index 7c423ca..29869f2 100644 --- a/src/CheckCommand.php +++ b/src/CheckCommand.php @@ -67,27 +67,6 @@ protected function configure(): void 'Log check execution to text file', false ) - ->addOption( - 'max-discard-ratio', - null, - InputOption::VALUE_REQUIRED, - 'Maximum number of discarded checks per successful check before giving up', - Runner::MAX_DISCARD_RATIO - ) - ->addOption( - 'max-shrinks', - null, - InputOption::VALUE_REQUIRED, - "Maximum number of shrinks to before giving up.\nSetting this to zero turns shrinking off.", - Runner::MAX_SIZE - ) - ->addOption( - 'max-size', - null, - InputOption::VALUE_REQUIRED, - 'Size to use for the biggest test cases', - Runner::MAX_SIZE - ) ->addOption( 'max-success', null, @@ -103,12 +82,6 @@ protected function configure(): void 'Ignore previous defects', false ) - ->addOption( - 'replay', - 'r', - InputOption::VALUE_OPTIONAL, - 'Replay execution with a specific seed' - ) ->addArgument( 'path', InputArgument::OPTIONAL, diff --git a/src/Runner.php b/src/Runner.php index e7c15fd..773c0fd 100644 --- a/src/Runner.php +++ b/src/Runner.php @@ -91,10 +91,6 @@ public function execute(InputInterface $input, OutputInterface $output): void $this->input = $input; $this->output = $output; - if ($replay = $input->getOption('replay')) { - $this->setSeed((int) $replay); - } - if ($input->getOption('coverage-html') !== false) { if (null === $input->getOption('coverage-html')) { $output->writeln('You must specify a directory for coverage-html');