stdout reporter

examples/example1.c++

@@ -0,0 +1,15 @@
+#include <nonius/nonius.h++>
+
+#include <iterator>
+#include <string>
+
+int main() {
+    nonius::configuration cfg;
+    nonius::benchmark benchmarks[] = {
+        { "to_string(42)", []{ return std::to_string(42); } },
+        { "to_string(4.2)", []{ return std::to_string(4.2); } },
+    };
+
+    nonius::go(cfg, std::begin(benchmarks), std::end(benchmarks));
+}
+

include/nonius/benchmark.h++

@@ -44,12 +44,12 @@ namespace nonius {
         }
 
         template <typename Clock>
-        execution_plan<FloatDuration<Clock>> prepare(configuration cfg, environment<FloatDuration<Clock>> env) const {
-            auto min_time = env.clock_resolution * detail::minimum_ticks;
+        execution_plan<FloatDuration<Clock>> prepare(configuration, environment<FloatDuration<Clock>> env) const {
+            auto min_time = env.clock_resolution.mean * detail::minimum_ticks;
             auto run_time = std::min(min_time, decltype(min_time)(detail::warmup_time));
             auto&& test = detail::run_for_at_least<Clock>(std::chrono::duration_cast<Duration<Clock>>(run_time), 1, *this);
             int new_iters = std::ceil(min_time * test.iterations / test.elapsed);
-            return { cfg.samples, new_iters, test.elapsed / test.iterations * new_iters };
+            return { new_iters, test.elapsed / test.iterations * new_iters };
         }
 
         template <typename Clock>
@@ -61,7 +61,7 @@ namespace nonius {
             times.reserve(cfg.samples);
             std::generate_n(std::back_inserter(times), cfg.samples, [this, env, plan]{
                     auto t = detail::measure<Clock>(*this, plan.iterations_per_sample).elapsed;
-                    return ((t - env.clock_cost) / plan.iterations_per_sample);
+                    return ((t - env.clock_cost.mean) / plan.iterations_per_sample);
             });
             return times;
         }

include/nonius/configuration.h++

@@ -14,20 +14,12 @@
 #ifndef NONIUS_CONFIGURATION_HPP
 #define NONIUS_CONFIGURATION_HPP
 
-#include <nonius/reporter.h++>
-
 namespace nonius {
     struct configuration {
     public:
-        configuration() {
-            static nonius::reporter null_reporter;
-            reporter = &null_reporter;
-        }
-
         int samples = 100;
         double confidence_interval = 0.95;
         int resamples = 100000;
-        nonius::reporter* reporter;
     };
 } // namespace nonius
 

include/nonius/detail/analyse.h++

@@ -15,7 +15,7 @@
 #define NONIUS_DETAIL_ANALYSE_HPP
 
 #include <nonius/clock.h++>
-#include <nonius/benchmark_results.h++>
+#include <nonius/sample_analysis.h++>
 #include <nonius/detail/stats.h++>
 
 #include <algorithm>
@@ -25,7 +25,7 @@
 namespace nonius {
     namespace detail {
         template <typename Duration, typename Iterator>
-        sample_analysis<Duration> analyse(configuration cfg, environment<Duration> env, Iterator first, Iterator last) {
+        sample_analysis<Duration> analyse(configuration cfg, environment<Duration>, Iterator first, Iterator last) {
             std::vector<double> samples;
             samples.reserve(last - first);
             std::transform(first, last, std::back_inserter(samples), [](Duration d) { return d.count(); });
@@ -41,11 +41,15 @@ namespace nonius {
                     e.confidence_interval,
                 };
             };
+            std::vector<Duration> samples2;
+            samples2.reserve(samples.size());
+            std::transform(samples.begin(), samples.end(), std::back_inserter(samples2), [](double d) { return Duration(d); });
             return {
-                std::move(samples),
+                std::move(samples2),
                 wrap_estimate(analysis.mean),
                 wrap_estimate(analysis.standard_deviation),
-                analysis.outlier_variance
+                outliers,
+                analysis.outlier_variance,
             };
         }
     } // namespace detail

include/nonius/detail/estimate_clock.h++

@@ -30,7 +30,7 @@
 namespace nonius {
     namespace detail {
         template <typename Clock>
-        std::tuple<std::vector<double>, int> resolution(int k) {
+        std::vector<double> resolution(int k) {
             std::vector<TimePoint<Clock>> times;
             times.reserve(k+1);
             std::generate_n(std::back_inserter(times), k+1, now<Clock>{});
@@ -42,7 +42,7 @@ namespace nonius {
                               [](TimePoint<Clock> a, TimePoint<Clock> b) { return (a - b).count(); },
                               [](double d) { return d > 0; });
 
-            return std::make_tuple(std::move(deltas), times.size());
+            return deltas;
         }
 
         constexpr auto warmup_seed = 10000;
@@ -58,13 +58,16 @@ namespace nonius {
                     .iterations;
         }
         template <typename Clock>
-        FloatDuration<Clock> estimate_clock_resolution(int iterations) {
+        environment_estimate<FloatDuration<Clock>> estimate_clock_resolution(int iterations) {
             auto r = run_for_at_least<Clock>(std::chrono::duration_cast<Duration<Clock>>(clock_resolution_estimation_time), iterations, &resolution<Clock>)
                     .result;
-            return FloatDuration<Clock>(mean(std::get<0>(r).begin(), std::get<0>(r).end()));
+            return {
+                FloatDuration<Clock>(mean(r.begin(), r.end())),
+                classify_outliers(r.begin(), r.end()),
+            };
         }
         template <typename Clock>
-        FloatDuration<Clock> estimate_clock_cost(FloatDuration<Clock> resolution) {
+        environment_estimate<FloatDuration<Clock>> estimate_clock_cost(FloatDuration<Clock> resolution) {
             auto time_limit = std::min(resolution * clock_cost_estimation_tick_limit, FloatDuration<Clock>(clock_cost_estimation_time_limit));
             auto time_clock = [](int k) {
                 return detail::measure<Clock>([k]{
@@ -83,7 +86,10 @@ namespace nonius {
             std::generate_n(std::back_inserter(times), nsamples, [time_clock, &r]{
                         return (time_clock(r.iterations) / r.iterations).count();
                     });
-            return FloatDuration<Clock>(mean(times.begin(), times.end()));
+            return {
+                FloatDuration<Clock>(mean(times.begin(), times.end())),
+                classify_outliers(times.begin(), times.end()),
+            };
         }
     } // namespace detail
 } // namespace nonius

include/nonius/detail/stats.h++

@@ -185,8 +185,8 @@ namespace nonius {
             int lo = std::max(cumn(a1), 0);
             int hi = std::min(cumn(a2), n - 1);
 
-            if(n_samples == 1) return { point, point, point };
-            else return { point, resample[lo], resample[hi] };
+            if(n_samples == 1) return { point, point, point, confidence_level };
+            else return { point, resample[lo], resample[hi], confidence_level };
         }
 
         template <typename Iterator, int... I, typename... Estimators>
@@ -227,14 +227,14 @@ namespace nonius {
             return std::min(var_out(1), var_out(std::min(c_max(0.), c_max(mg_min)))) / sb2;
         }
 
-        struct sample_analysis {
+        struct bootstrap_analysis {
             estimate<double> mean;
             estimate<double> standard_deviation;
             double outlier_variance;
         };
 
         template <typename Iterator>
-        sample_analysis analyse_samples(double confidence_level, int n_resamples, Iterator first, Iterator last) {
+        bootstrap_analysis analyse_samples(double confidence_level, int n_resamples, Iterator first, Iterator last) {
             static std::random_device entropy;
 
             int n = last - first;
@@ -244,7 +244,7 @@ namespace nonius {
             auto stddev = &detail::standard_deviation<Iterator>;
             auto resamples = resample(rng, n_resamples, first, last, mean, stddev);
             auto estimates = bootstrap(confidence_level, first, last, resamples, mean, stddev);
-            double outlier_variance = outlier_variance(estimates[0], estimates[1], n);
+            double outlier_variance = detail::outlier_variance(estimates[0], estimates[1], n);
 
             return { estimates[0], estimates[1], outlier_variance };
         }

include/nonius/environment.h++

@@ -19,12 +19,15 @@
 
 namespace nonius {
     template <typename Duration>
-    using environment_estimate = Duration;
-    //TODO
-    //struct environment_estimate {
-    //    Duration mean;
-    //    outlier_classification outliers;
-    //};
+    struct environment_estimate {
+        Duration mean;
+        outlier_classification outliers;
+
+        template <typename Duration2>
+        operator environment_estimate<Duration2>() const {
+            return { mean, outliers };
+        }
+    };
     template <typename Clock = default_clock>
     struct environment {
         environment_estimate<FloatDuration<Clock>> clock_resolution;

include/nonius/estimate.h++

@@ -15,12 +15,17 @@
 #define NONIUS_ESTIMATE_HPP
 
 namespace nonius {
-    template <typename T>
+    template <typename Duration>
     struct estimate {
-        T point;
-        T lower_bound;
-        T upper_bound;
+        Duration point;
+        Duration lower_bound;
+        Duration upper_bound;
         double confidence_interval;
+
+        template <typename Duration2>
+        operator estimate<Duration2>() const {
+            return { point, lower_bound, upper_bound, confidence_interval };
+        }
     };
 } // namespace nonius
 

include/nonius/execution_plan.h++

@@ -17,13 +17,12 @@
 namespace nonius {
     template <typename Duration>
     struct execution_plan {
-        int samples;
         int iterations_per_sample;
         Duration estimated_duration;
 
         template <typename Duration2>
         operator execution_plan<Duration2>() const {
-            return { samples, iterations_per_sample, estimated_duration };
+            return { iterations_per_sample, estimated_duration };
         }
     };
 } // namespace nonius

include/nonius/go.h++

@@ -19,42 +19,58 @@
 #include <nonius/configuration.h++>
 #include <nonius/environment.h++>
 #include <nonius/reporter.h++>
+#include <nonius/stdout_reporter.h++>
 #include <nonius/detail/estimate_clock.h++>
+#include <nonius/detail/analyse.h++>
 
 namespace nonius {
     namespace detail {
         template <typename Clock>
-        environment<FloatDuration<Clock>> measure_environment(reporter& r) {
-            r.estimate_clock_resolution_start();
+        environment<FloatDuration<Clock>> measure_environment(reporter& rep) {
+            rep.estimate_clock_resolution_start();
             auto iters = detail::warmup<Clock>();
             auto resolution = detail::estimate_clock_resolution<Clock>(iters);
-            r.estimate_clock_resolution_complete(resolution);
+            rep.estimate_clock_resolution_complete(resolution);
 
-            r.estimate_clock_cost_start();
-            auto cost = detail::estimate_clock_cost<Clock>(resolution);
-            r.estimate_clock_cost_complete(cost);
+            rep.estimate_clock_cost_start();
+            auto cost = detail::estimate_clock_cost<Clock>(resolution.mean);
+            rep.estimate_clock_cost_complete(cost);
 
             return { resolution, cost };
         }
     } // namespace detail
     template <typename Clock = default_clock, typename Iterator>
-    void go(configuration cfg, Iterator first, Iterator last) {
-        auto env = detail::measure_environment<Clock>(*cfg.reporter);
+    void go(configuration cfg, Iterator first, Iterator last, reporter& rep) {
+        rep.configure(cfg);
+
+        auto env = detail::measure_environment<Clock>(rep);
 
-        cfg.reporter->suite_start();
+        rep.suite_start();
 
         for(; first != last; ++first) {
-            cfg.reporter->benchmark_start(first->name);
+            rep.benchmark_start(first->name);
 
             auto plan = first->template prepare<Clock>(cfg, env);
-            cfg.reporter->measurement_start(plan);
+            rep.measurement_start(plan);
             auto samples = first->template run<Clock>(cfg, env, plan);
-            cfg.reporter->measurement_complete(std::vector<fp_seconds>(samples.begin(), samples.end()));
+            rep.measurement_complete(std::vector<fp_seconds>(samples.begin(), samples.end()));
 
-            cfg.reporter->benchmark_complete();
+            rep.analysis_start();
+            auto analysis = detail::analyse(cfg, env, samples.begin(), samples.end());
+            rep.analysis_complete(analysis);
+
+            rep.benchmark_complete();
         }
 
-        cfg.reporter->suite_complete();
+        rep.suite_complete();
+    }
+    template <typename Clock = default_clock, typename Iterator>
+    void go(configuration cfg, Iterator first, Iterator last, reporter&& rep) {
+        go(cfg, first, last, rep);
+    }
+    template <typename Clock = default_clock, typename Iterator>
+    void go(configuration cfg, Iterator first, Iterator last) {
+        go(cfg, first, last, stdout_reporter{});
     }
 } // namespace nonius
 

include/nonius/nonius.h++

@@ -0,0 +1,24 @@
+// Nonius - C++ benchmarking tool
+//
+// Written in 2014 by Martinho Fernandes <[email protected]>
+//
+// To the extent possible under law, the author(s) have dedicated all copyright and related
+// and neighboring rights to this software to the public domain worldwide. This software is
+// distributed without any warranty.
+//
+// You should have received a copy of the CC0 Public Domain Dedication along with this software.
+// If not, see <http://creativecommons.org/publicdomain/zero/1.0/>
+
+// Main header
+
+#ifndef NONIUS_HPP
+#define NONIUS_HPP
+
+#include <nonius/clock.h++>
+#include <nonius/benchmark.h++>
+#include <nonius/configuration.h++>
+#include <nonius/stdout_reporter.h++>
+#include <nonius/go.h++>
+
+#endif // NONIUS_HPP
+

include/nonius/reporter.h++

@@ -18,12 +18,15 @@
 #include <nonius/outlier_classification.h++>
 #include <nonius/environment.h++>
 #include <nonius/execution_plan.h++>
+#include <nonius/sample_analysis.h++>
 
 #include <vector>
 #include <string>
 
 namespace nonius {
     struct reporter {
+        virtual void configure(configuration /*cfg*/) {}
+
         virtual void estimate_clock_resolution_start() {}
         virtual void estimate_clock_resolution_complete(environment_estimate<fp_seconds> /*estimate*/) {}
 
@@ -36,6 +39,9 @@ namespace nonius {
         virtual void measurement_start(execution_plan<fp_seconds> /*measurement conditions*/) {}
         virtual void measurement_complete(std::vector<fp_seconds> const& /*samples*/) {}
 
+        virtual void analysis_start() {} // TODO make generic?
+        virtual void analysis_complete(sample_analysis<fp_seconds> const& /*analysis*/) {}
+
         virtual void benchmark_complete() {}
         virtual void suite_complete() {}