diff mbox series

[v7,15/21] scripts/simplebench: use standard deviation for +- error

Message ID 20201021145859.11201-16-vsementsov@virtuozzo.com
State Accepted
Commit f52e1af0b08af93b5354fe2648eccaec6bb8a2b2
Headers show
Series preallocate filter | expand

Commit Message

Vladimir Sementsov-Ogievskiy Oct. 21, 2020, 2:58 p.m. UTC 
Standard deviation is more usual to see after +- than current maximum
of deviations.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
---
 scripts/simplebench/simplebench.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

Comments

Max Reitz Nov. 13, 2020, 3:30 p.m. UTC | #1 
On 21.10.20 16:58, Vladimir Sementsov-Ogievskiy wrote:
> Standard deviation is more usual to see after +- than current maximum

> of deviations.

> 

> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

> ---

>   scripts/simplebench/simplebench.py | 11 ++++++-----

>   1 file changed, 6 insertions(+), 5 deletions(-)


Reviewed-by: Max Reitz <mreitz@redhat.com>
diff mbox series

Patch

diff --git a/scripts/simplebench/simplebench.py b/scripts/simplebench/simplebench.py
index 2251cd34ea..55ec1ad5db 100644
--- a/scripts/simplebench/simplebench.py
+++ b/scripts/simplebench/simplebench.py
@@ -18,6 +18,8 @@ 
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+import statistics
+
 
 def bench_one(test_func, test_env, test_case, count=5, initial_run=True):
     """Benchmark one test-case
@@ -40,7 +42,7 @@  def bench_one(test_func, test_env, test_case, count=5, initial_run=True):
         'dimension': dimension of results, may be 'seconds' or 'iops'
         'average':  average value (iops or seconds) per run (exists only if at
                     least one run succeeded)
-        'delta':    maximum delta between test_func result and the average
+        'stdev':    standard deviation of results
                     (exists only if at least one run succeeded)
         'n-failed': number of failed runs (exists only if at least one run
                     failed)
@@ -67,10 +69,9 @@  def bench_one(test_func, test_env, test_case, count=5, initial_run=True):
             assert all('seconds' in r for r in succeeded)
             assert all('iops' not in r for r in succeeded)
             dim = 'seconds'
-        avg = sum(r[dim] for r in succeeded) / len(succeeded)
         result['dimension'] = dim
-        result['average'] = avg
-        result['delta'] = max(abs(r[dim] - avg) for r in succeeded)
+        result['average'] = statistics.mean(r[dim] for r in succeeded)
+        result['stdev'] = statistics.stdev(r[dim] for r in succeeded)
 
     if len(succeeded) < count:
         result['n-failed'] = count - len(succeeded)
@@ -81,7 +82,7 @@  def bench_one(test_func, test_env, test_case, count=5, initial_run=True):
 def ascii_one(result):
     """Return ASCII representation of bench_one() returned dict."""
     if 'average' in result:
-        s = '{:.2f} +- {:.2f}'.format(result['average'], result['delta'])
+        s = '{:.2f} +- {:.2f}'.format(result['average'], result['stdev'])
         if 'n-failed' in result:
             s += '\n({} failed)'.format(result['n-failed'])
         return s