From d86fb670e6d78ca38dbaedfdde35180e3b8f4bb3 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 18 Jun 2019 23:44:16 +1000 Subject: [PATCH 1/6] tests: Rename "bench" tests to "internal_bench" and run-internalbench.py To emphasise these benchmark tests compare the internal performance of features amongst themselves, rather than absolute performance testing. --- tests/{bench => internal_bench}/arrayop-1-list_inplace.py | 0 tests/{bench => internal_bench}/arrayop-2-list_map.py | 0 tests/{bench => internal_bench}/arrayop-3-bytearray_inplace.py | 0 tests/{bench => internal_bench}/arrayop-4-bytearray_map.py | 0 tests/{bench => internal_bench}/bench.py | 0 tests/{bench => internal_bench}/bytealloc-1-bytes_n.py | 0 tests/{bench => internal_bench}/bytealloc-2-repeat.py | 0 tests/{bench => internal_bench}/bytebuf-1-inplace.py | 0 tests/{bench => internal_bench}/bytebuf-2-join_map_bytes.py | 0 tests/{bench => internal_bench}/bytebuf-3-bytarray_map.py | 0 tests/{bench => internal_bench}/from_iter-1-list_bound.py | 0 tests/{bench => internal_bench}/from_iter-2-list_unbound.py | 0 tests/{bench => internal_bench}/from_iter-3-tuple_bound.py | 0 tests/{bench => internal_bench}/from_iter-4-tuple_unbound.py | 0 tests/{bench => internal_bench}/from_iter-5-bytes_bound.py | 0 tests/{bench => internal_bench}/from_iter-6-bytes_unbound.py | 0 tests/{bench => internal_bench}/from_iter-7-bytearray_bound.py | 0 .../{bench => internal_bench}/from_iter-8-bytearray_unbound.py | 0 tests/{bench => internal_bench}/func_args-1.1-pos_1.py | 0 tests/{bench => internal_bench}/func_args-1.2-pos_3.py | 0 .../{bench => internal_bench}/func_args-2-pos_default_2_of_3.py | 0 tests/{bench => internal_bench}/func_args-3.1-kw_1.py | 0 tests/{bench => internal_bench}/func_args-3.2-kw_3.py | 0 tests/{bench => internal_bench}/func_builtin-1-enum_pos.py | 0 tests/{bench => internal_bench}/func_builtin-2-enum_kw.py | 0 tests/{bench => internal_bench}/funcall-1-inline.py | 0 tests/{bench => internal_bench}/funcall-2-funcall.py | 0 tests/{bench => internal_bench}/funcall-3-funcall-local.py | 0 tests/{bench => internal_bench}/loop_count-1-range.py | 0 tests/{bench => internal_bench}/loop_count-2-range_iter.py | 0 tests/{bench => internal_bench}/loop_count-3-while_up.py | 0 tests/{bench => internal_bench}/loop_count-4-while_down_gt.py | 0 tests/{bench => internal_bench}/loop_count-5-while_down_ne.py | 0 .../loop_count-5.1-while_down_ne_localvar.py | 0 tests/{bench => internal_bench}/var-1-constant.py | 0 tests/{bench => internal_bench}/var-2-global.py | 0 tests/{bench => internal_bench}/var-3-local.py | 0 tests/{bench => internal_bench}/var-4-arg.py | 0 tests/{bench => internal_bench}/var-5-class-attr.py | 0 tests/{bench => internal_bench}/var-6-instance-attr.py | 0 tests/{bench => internal_bench}/var-6.1-instance-attr-5.py | 0 tests/{bench => internal_bench}/var-7-instance-meth.py | 0 tests/{bench => internal_bench}/var-8-namedtuple-1st.py | 0 tests/{bench => internal_bench}/var-8.1-namedtuple-5th.py | 0 tests/{run-bench-tests => run-internalbench.py} | 2 +- 45 files changed, 1 insertion(+), 1 deletion(-) rename tests/{bench => internal_bench}/arrayop-1-list_inplace.py (100%) rename tests/{bench => internal_bench}/arrayop-2-list_map.py (100%) rename tests/{bench => internal_bench}/arrayop-3-bytearray_inplace.py (100%) rename tests/{bench => internal_bench}/arrayop-4-bytearray_map.py (100%) rename tests/{bench => internal_bench}/bench.py (100%) rename tests/{bench => internal_bench}/bytealloc-1-bytes_n.py (100%) rename tests/{bench => internal_bench}/bytealloc-2-repeat.py (100%) rename tests/{bench => internal_bench}/bytebuf-1-inplace.py (100%) rename tests/{bench => internal_bench}/bytebuf-2-join_map_bytes.py (100%) rename tests/{bench => internal_bench}/bytebuf-3-bytarray_map.py (100%) rename tests/{bench => internal_bench}/from_iter-1-list_bound.py (100%) rename tests/{bench => internal_bench}/from_iter-2-list_unbound.py (100%) rename tests/{bench => internal_bench}/from_iter-3-tuple_bound.py (100%) rename tests/{bench => internal_bench}/from_iter-4-tuple_unbound.py (100%) rename tests/{bench => internal_bench}/from_iter-5-bytes_bound.py (100%) rename tests/{bench => internal_bench}/from_iter-6-bytes_unbound.py (100%) rename tests/{bench => internal_bench}/from_iter-7-bytearray_bound.py (100%) rename tests/{bench => internal_bench}/from_iter-8-bytearray_unbound.py (100%) rename tests/{bench => internal_bench}/func_args-1.1-pos_1.py (100%) rename tests/{bench => internal_bench}/func_args-1.2-pos_3.py (100%) rename tests/{bench => internal_bench}/func_args-2-pos_default_2_of_3.py (100%) rename tests/{bench => internal_bench}/func_args-3.1-kw_1.py (100%) rename tests/{bench => internal_bench}/func_args-3.2-kw_3.py (100%) rename tests/{bench => internal_bench}/func_builtin-1-enum_pos.py (100%) rename tests/{bench => internal_bench}/func_builtin-2-enum_kw.py (100%) rename tests/{bench => internal_bench}/funcall-1-inline.py (100%) rename tests/{bench => internal_bench}/funcall-2-funcall.py (100%) rename tests/{bench => internal_bench}/funcall-3-funcall-local.py (100%) rename tests/{bench => internal_bench}/loop_count-1-range.py (100%) rename tests/{bench => internal_bench}/loop_count-2-range_iter.py (100%) rename tests/{bench => internal_bench}/loop_count-3-while_up.py (100%) rename tests/{bench => internal_bench}/loop_count-4-while_down_gt.py (100%) rename tests/{bench => internal_bench}/loop_count-5-while_down_ne.py (100%) rename tests/{bench => internal_bench}/loop_count-5.1-while_down_ne_localvar.py (100%) rename tests/{bench => internal_bench}/var-1-constant.py (100%) rename tests/{bench => internal_bench}/var-2-global.py (100%) rename tests/{bench => internal_bench}/var-3-local.py (100%) rename tests/{bench => internal_bench}/var-4-arg.py (100%) rename tests/{bench => internal_bench}/var-5-class-attr.py (100%) rename tests/{bench => internal_bench}/var-6-instance-attr.py (100%) rename tests/{bench => internal_bench}/var-6.1-instance-attr-5.py (100%) rename tests/{bench => internal_bench}/var-7-instance-meth.py (100%) rename tests/{bench => internal_bench}/var-8-namedtuple-1st.py (100%) rename tests/{bench => internal_bench}/var-8.1-namedtuple-5th.py (100%) rename tests/{run-bench-tests => run-internalbench.py} (98%) diff --git a/tests/bench/arrayop-1-list_inplace.py b/tests/internal_bench/arrayop-1-list_inplace.py similarity index 100% rename from tests/bench/arrayop-1-list_inplace.py rename to tests/internal_bench/arrayop-1-list_inplace.py diff --git a/tests/bench/arrayop-2-list_map.py b/tests/internal_bench/arrayop-2-list_map.py similarity index 100% rename from tests/bench/arrayop-2-list_map.py rename to tests/internal_bench/arrayop-2-list_map.py diff --git a/tests/bench/arrayop-3-bytearray_inplace.py b/tests/internal_bench/arrayop-3-bytearray_inplace.py similarity index 100% rename from tests/bench/arrayop-3-bytearray_inplace.py rename to tests/internal_bench/arrayop-3-bytearray_inplace.py diff --git a/tests/bench/arrayop-4-bytearray_map.py b/tests/internal_bench/arrayop-4-bytearray_map.py similarity index 100% rename from tests/bench/arrayop-4-bytearray_map.py rename to tests/internal_bench/arrayop-4-bytearray_map.py diff --git a/tests/bench/bench.py b/tests/internal_bench/bench.py similarity index 100% rename from tests/bench/bench.py rename to tests/internal_bench/bench.py diff --git a/tests/bench/bytealloc-1-bytes_n.py b/tests/internal_bench/bytealloc-1-bytes_n.py similarity index 100% rename from tests/bench/bytealloc-1-bytes_n.py rename to tests/internal_bench/bytealloc-1-bytes_n.py diff --git a/tests/bench/bytealloc-2-repeat.py b/tests/internal_bench/bytealloc-2-repeat.py similarity index 100% rename from tests/bench/bytealloc-2-repeat.py rename to tests/internal_bench/bytealloc-2-repeat.py diff --git a/tests/bench/bytebuf-1-inplace.py b/tests/internal_bench/bytebuf-1-inplace.py similarity index 100% rename from tests/bench/bytebuf-1-inplace.py rename to tests/internal_bench/bytebuf-1-inplace.py diff --git a/tests/bench/bytebuf-2-join_map_bytes.py b/tests/internal_bench/bytebuf-2-join_map_bytes.py similarity index 100% rename from tests/bench/bytebuf-2-join_map_bytes.py rename to tests/internal_bench/bytebuf-2-join_map_bytes.py diff --git a/tests/bench/bytebuf-3-bytarray_map.py b/tests/internal_bench/bytebuf-3-bytarray_map.py similarity index 100% rename from tests/bench/bytebuf-3-bytarray_map.py rename to tests/internal_bench/bytebuf-3-bytarray_map.py diff --git a/tests/bench/from_iter-1-list_bound.py b/tests/internal_bench/from_iter-1-list_bound.py similarity index 100% rename from tests/bench/from_iter-1-list_bound.py rename to tests/internal_bench/from_iter-1-list_bound.py diff --git a/tests/bench/from_iter-2-list_unbound.py b/tests/internal_bench/from_iter-2-list_unbound.py similarity index 100% rename from tests/bench/from_iter-2-list_unbound.py rename to tests/internal_bench/from_iter-2-list_unbound.py diff --git a/tests/bench/from_iter-3-tuple_bound.py b/tests/internal_bench/from_iter-3-tuple_bound.py similarity index 100% rename from tests/bench/from_iter-3-tuple_bound.py rename to tests/internal_bench/from_iter-3-tuple_bound.py diff --git a/tests/bench/from_iter-4-tuple_unbound.py b/tests/internal_bench/from_iter-4-tuple_unbound.py similarity index 100% rename from tests/bench/from_iter-4-tuple_unbound.py rename to tests/internal_bench/from_iter-4-tuple_unbound.py diff --git a/tests/bench/from_iter-5-bytes_bound.py b/tests/internal_bench/from_iter-5-bytes_bound.py similarity index 100% rename from tests/bench/from_iter-5-bytes_bound.py rename to tests/internal_bench/from_iter-5-bytes_bound.py diff --git a/tests/bench/from_iter-6-bytes_unbound.py b/tests/internal_bench/from_iter-6-bytes_unbound.py similarity index 100% rename from tests/bench/from_iter-6-bytes_unbound.py rename to tests/internal_bench/from_iter-6-bytes_unbound.py diff --git a/tests/bench/from_iter-7-bytearray_bound.py b/tests/internal_bench/from_iter-7-bytearray_bound.py similarity index 100% rename from tests/bench/from_iter-7-bytearray_bound.py rename to tests/internal_bench/from_iter-7-bytearray_bound.py diff --git a/tests/bench/from_iter-8-bytearray_unbound.py b/tests/internal_bench/from_iter-8-bytearray_unbound.py similarity index 100% rename from tests/bench/from_iter-8-bytearray_unbound.py rename to tests/internal_bench/from_iter-8-bytearray_unbound.py diff --git a/tests/bench/func_args-1.1-pos_1.py b/tests/internal_bench/func_args-1.1-pos_1.py similarity index 100% rename from tests/bench/func_args-1.1-pos_1.py rename to tests/internal_bench/func_args-1.1-pos_1.py diff --git a/tests/bench/func_args-1.2-pos_3.py b/tests/internal_bench/func_args-1.2-pos_3.py similarity index 100% rename from tests/bench/func_args-1.2-pos_3.py rename to tests/internal_bench/func_args-1.2-pos_3.py diff --git a/tests/bench/func_args-2-pos_default_2_of_3.py b/tests/internal_bench/func_args-2-pos_default_2_of_3.py similarity index 100% rename from tests/bench/func_args-2-pos_default_2_of_3.py rename to tests/internal_bench/func_args-2-pos_default_2_of_3.py diff --git a/tests/bench/func_args-3.1-kw_1.py b/tests/internal_bench/func_args-3.1-kw_1.py similarity index 100% rename from tests/bench/func_args-3.1-kw_1.py rename to tests/internal_bench/func_args-3.1-kw_1.py diff --git a/tests/bench/func_args-3.2-kw_3.py b/tests/internal_bench/func_args-3.2-kw_3.py similarity index 100% rename from tests/bench/func_args-3.2-kw_3.py rename to tests/internal_bench/func_args-3.2-kw_3.py diff --git a/tests/bench/func_builtin-1-enum_pos.py b/tests/internal_bench/func_builtin-1-enum_pos.py similarity index 100% rename from tests/bench/func_builtin-1-enum_pos.py rename to tests/internal_bench/func_builtin-1-enum_pos.py diff --git a/tests/bench/func_builtin-2-enum_kw.py b/tests/internal_bench/func_builtin-2-enum_kw.py similarity index 100% rename from tests/bench/func_builtin-2-enum_kw.py rename to tests/internal_bench/func_builtin-2-enum_kw.py diff --git a/tests/bench/funcall-1-inline.py b/tests/internal_bench/funcall-1-inline.py similarity index 100% rename from tests/bench/funcall-1-inline.py rename to tests/internal_bench/funcall-1-inline.py diff --git a/tests/bench/funcall-2-funcall.py b/tests/internal_bench/funcall-2-funcall.py similarity index 100% rename from tests/bench/funcall-2-funcall.py rename to tests/internal_bench/funcall-2-funcall.py diff --git a/tests/bench/funcall-3-funcall-local.py b/tests/internal_bench/funcall-3-funcall-local.py similarity index 100% rename from tests/bench/funcall-3-funcall-local.py rename to tests/internal_bench/funcall-3-funcall-local.py diff --git a/tests/bench/loop_count-1-range.py b/tests/internal_bench/loop_count-1-range.py similarity index 100% rename from tests/bench/loop_count-1-range.py rename to tests/internal_bench/loop_count-1-range.py diff --git a/tests/bench/loop_count-2-range_iter.py b/tests/internal_bench/loop_count-2-range_iter.py similarity index 100% rename from tests/bench/loop_count-2-range_iter.py rename to tests/internal_bench/loop_count-2-range_iter.py diff --git a/tests/bench/loop_count-3-while_up.py b/tests/internal_bench/loop_count-3-while_up.py similarity index 100% rename from tests/bench/loop_count-3-while_up.py rename to tests/internal_bench/loop_count-3-while_up.py diff --git a/tests/bench/loop_count-4-while_down_gt.py b/tests/internal_bench/loop_count-4-while_down_gt.py similarity index 100% rename from tests/bench/loop_count-4-while_down_gt.py rename to tests/internal_bench/loop_count-4-while_down_gt.py diff --git a/tests/bench/loop_count-5-while_down_ne.py b/tests/internal_bench/loop_count-5-while_down_ne.py similarity index 100% rename from tests/bench/loop_count-5-while_down_ne.py rename to tests/internal_bench/loop_count-5-while_down_ne.py diff --git a/tests/bench/loop_count-5.1-while_down_ne_localvar.py b/tests/internal_bench/loop_count-5.1-while_down_ne_localvar.py similarity index 100% rename from tests/bench/loop_count-5.1-while_down_ne_localvar.py rename to tests/internal_bench/loop_count-5.1-while_down_ne_localvar.py diff --git a/tests/bench/var-1-constant.py b/tests/internal_bench/var-1-constant.py similarity index 100% rename from tests/bench/var-1-constant.py rename to tests/internal_bench/var-1-constant.py diff --git a/tests/bench/var-2-global.py b/tests/internal_bench/var-2-global.py similarity index 100% rename from tests/bench/var-2-global.py rename to tests/internal_bench/var-2-global.py diff --git a/tests/bench/var-3-local.py b/tests/internal_bench/var-3-local.py similarity index 100% rename from tests/bench/var-3-local.py rename to tests/internal_bench/var-3-local.py diff --git a/tests/bench/var-4-arg.py b/tests/internal_bench/var-4-arg.py similarity index 100% rename from tests/bench/var-4-arg.py rename to tests/internal_bench/var-4-arg.py diff --git a/tests/bench/var-5-class-attr.py b/tests/internal_bench/var-5-class-attr.py similarity index 100% rename from tests/bench/var-5-class-attr.py rename to tests/internal_bench/var-5-class-attr.py diff --git a/tests/bench/var-6-instance-attr.py b/tests/internal_bench/var-6-instance-attr.py similarity index 100% rename from tests/bench/var-6-instance-attr.py rename to tests/internal_bench/var-6-instance-attr.py diff --git a/tests/bench/var-6.1-instance-attr-5.py b/tests/internal_bench/var-6.1-instance-attr-5.py similarity index 100% rename from tests/bench/var-6.1-instance-attr-5.py rename to tests/internal_bench/var-6.1-instance-attr-5.py diff --git a/tests/bench/var-7-instance-meth.py b/tests/internal_bench/var-7-instance-meth.py similarity index 100% rename from tests/bench/var-7-instance-meth.py rename to tests/internal_bench/var-7-instance-meth.py diff --git a/tests/bench/var-8-namedtuple-1st.py b/tests/internal_bench/var-8-namedtuple-1st.py similarity index 100% rename from tests/bench/var-8-namedtuple-1st.py rename to tests/internal_bench/var-8-namedtuple-1st.py diff --git a/tests/bench/var-8.1-namedtuple-5th.py b/tests/internal_bench/var-8.1-namedtuple-5th.py similarity index 100% rename from tests/bench/var-8.1-namedtuple-5th.py rename to tests/internal_bench/var-8.1-namedtuple-5th.py diff --git a/tests/run-bench-tests b/tests/run-internalbench.py similarity index 98% rename from tests/run-bench-tests rename to tests/run-internalbench.py index f4a6776cbc29a..f6294572f0091 100755 --- a/tests/run-bench-tests +++ b/tests/run-internalbench.py @@ -74,7 +74,7 @@ def main(): if len(args.files) == 0: if pyb is None: # run PC tests - test_dirs = ('bench',) + test_dirs = ('internal_bench',) else: # run pyboard tests test_dirs = ('basics', 'float', 'pyb') From e92c9aa9c94eae7971c8e82f4e875fc53ef52f07 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 26 Jun 2019 14:23:03 +1000 Subject: [PATCH 2/6] tests: Add performance benchmarking test-suite framework. This benchmarking test suite is intended to be run on any MicroPython target. As such all tests are parameterised with N and M: N is the approximate CPU frequency (in MHz) of the target and M is the approximate amount of heap memory (in kbytes) available on the target. When running the benchmark suite these parameters must be specified and then each test is tuned to run on that target in a reasonable time (<1 second). The test scripts are not standalone: they require adding some extra code at the end to run the test with the appropriate parameters. This is done automatically by the run-perfbench.py script, in such a way that imports are minimised (so the tests can be run on targets without filesystem support). To interface with the benchmarking framework, each test provides a bm_params dict and a bm_setup function, with the later taking a set of parameters (chosen based on N, M) and returning a pair of functions, one to run the test and one to get the results. When running the test the number of microseconds taken by the test are recorded. Then this is converted into a benchmark score by inverting it (so higher number is faster) and normalising it with an appropriate factor (based roughly on the amount of work done by the test, eg number of iterations). Test outputs are also compared against a "truth" value, computed by running the test with CPython. This provides a basic way of making sure the test actually ran correctly. Each test is run multiple times and the results averaged and standard deviation computed. This is output as a summary of the test. To make comparisons of performance across different runs the run-perfbench.py script also includes a diff mode that reads in the output of two previous runs and computes the difference in performance. Reports are given as a percentage change in performance with a combined standard deviation to give an indication if the noise in the benchmarking is less than the thing that is being measured. Example invocations for PC, pyboard and esp8266 targets respectively: $ ./run-perfbench.py 1000 1000 $ ./run-perfbench.py --pyboard 100 100 $ ./run-perfbench.py --pyboard --device /dev/ttyUSB0 50 25 --- tests/perf_bench/benchrun.py | 26 ++++ tests/run-perfbench.py | 241 +++++++++++++++++++++++++++++++++++ 2 files changed, 267 insertions(+) create mode 100644 tests/perf_bench/benchrun.py create mode 100755 tests/run-perfbench.py diff --git a/tests/perf_bench/benchrun.py b/tests/perf_bench/benchrun.py new file mode 100644 index 0000000000000..9cbc9695aca7b --- /dev/null +++ b/tests/perf_bench/benchrun.py @@ -0,0 +1,26 @@ +def bm_run(N, M): + try: + from utime import ticks_us, ticks_diff + except ImportError: + import time + ticks_us = lambda: int(time.perf_counter() * 1000000) + ticks_diff = lambda a, b: a - b + + # Pick sensible parameters given N, M + cur_nm = (0, 0) + param = None + for nm, p in bm_params.items(): + if 10 * nm[0] <= 12 * N and nm[1] <= M and nm > cur_nm: + cur_nm = nm + param = p + if param is None: + print(-1, -1, 'no matching params') + return + + # Run and time benchmark + run, result = bm_setup(param) + t0 = ticks_us() + run() + t1 = ticks_us() + norm, out = result() + print(ticks_diff(t1, t0), norm, out) diff --git a/tests/run-perfbench.py b/tests/run-perfbench.py new file mode 100755 index 0000000000000..d8100ef81d194 --- /dev/null +++ b/tests/run-perfbench.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 + +# This file is part of the MicroPython project, http://micropython.org/ +# The MIT License (MIT) +# Copyright (c) 2019 Damien P. George + +import os +import subprocess +import sys +import argparse +from glob import glob + +sys.path.append('../tools') +import pyboard + +# Paths for host executables +if os.name == 'nt': + CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3.exe') + MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../ports/windows/micropython.exe') +else: + CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3') + MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../ports/unix/micropython') + +PYTHON_TRUTH = CPYTHON3 + +BENCH_SCRIPT_DIR = 'perf_bench/' + +def compute_stats(lst): + avg = 0 + var = 0 + for x in lst: + avg += x + var += x * x + avg /= len(lst) + var = max(0, var / len(lst) - avg ** 2) + return avg, var ** 0.5 + +def run_script_on_target(target, script): + output = b'' + err = None + + if isinstance(target, pyboard.Pyboard): + # Run via pyboard interface + try: + target.enter_raw_repl() + output = target.exec_(script) + except pyboard.PyboardError as er: + err = er + else: + # Run local executable + try: + p = subprocess.run([target], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, input=script) + output = p.stdout + except subprocess.CalledProcessError as er: + err = er + + return str(output.strip(), 'ascii'), err + +def run_feature_test(target, test): + with open('feature_check/' + test + '.py', 'rb') as f: + script = f.read() + output, err = run_script_on_target(target, script) + if err is None: + return output + else: + return 'CRASH: %r' % err + +def run_benchmark_on_target(target, script): + output, err = run_script_on_target(target, script) + if err is None: + time, norm, result = output.split(None, 2) + try: + return int(time), int(norm), result + except ValueError: + return -1, -1, 'CRASH: %r' % output + else: + return -1, -1, 'CRASH: %r' % err + +def run_benchmarks(target, param_n, param_m, n_average, test_list): + skip_native = run_feature_test(target, 'native_check') != '' + + for test_file in sorted(test_list): + print(test_file + ': ', end='') + + # Check if test should be skipped + skip = skip_native and test_file.find('viper_') != -1 + if skip: + print('skip') + continue + + # Create test script + with open(test_file, 'rb') as f: + test_script = f.read() + with open(BENCH_SCRIPT_DIR + 'benchrun.py', 'rb') as f: + test_script += f.read() + test_script += b'bm_run(%u, %u)\n' % (param_n, param_m) + + # Write full test script if needed + if 0: + with open('%s.full' % test_file, 'wb') as f: + f.write(test_script) + + # Run MicroPython a given number of times + times = [] + scores = [] + error = None + result_out = None + for _ in range(n_average): + time, norm, result = run_benchmark_on_target(target, test_script) + if time < 0 or norm < 0: + error = result + break + if result_out is None: + result_out = result + elif result != result_out: + error = 'FAIL self' + break + times.append(time) + scores.append(1e6 * norm / time) + + # Check result against truth if needed + if error is None and result_out != 'None': + _, _, result_exp = run_benchmark_on_target(PYTHON_TRUTH, test_script) + if result_out != result_exp: + error = 'FAIL truth' + break + + if error is not None: + print(error) + else: + t_avg, t_sd = compute_stats(times) + s_avg, s_sd = compute_stats(scores) + print('{:.2f} {:.4f} {:.2f} {:.4f}'.format(t_avg, 100 * t_sd / t_avg, s_avg, 100 * s_sd / s_avg)) + if 0: + print(' times: ', times) + print(' scores:', scores) + + sys.stdout.flush() + +def parse_output(filename): + with open(filename) as f: + params = f.readline() + n, m, _ = params.strip().split() + n = int(n.split('=')[1]) + m = int(m.split('=')[1]) + data = [] + for l in f: + if l.find(': ') != -1 and l.find(': skip') == -1 and l.find('CRASH: ') == -1: + name, values = l.strip().split(': ') + values = tuple(float(v) for v in values.split()) + data.append((name,) + values) + return n, m, data + +def compute_diff(file1, file2, diff_score): + # Parse output data from previous runs + n1, m1, d1 = parse_output(file1) + n2, m2, d2 = parse_output(file2) + + # Print header + if diff_score: + print('diff of scores (higher is better)') + else: + print('diff of microsecond times (lower is better)') + if n1 == n2 and m1 == m2: + hdr = 'N={} M={}'.format(n1, m1) + else: + hdr = 'N={} M={} vs N={} M={}'.format(n1, m1, n2, m2) + print('{:24} {:>10} -> {:>10} {:>10} {:>7}% (error%)'.format(hdr, file1, file2, 'diff', 'diff')) + + # Print entries + while d1 and d2: + if d1[0][0] == d2[0][0]: + # Found entries with matching names + entry1 = d1.pop(0) + entry2 = d2.pop(0) + name = entry1[0].rsplit('/')[-1] + av1, sd1 = entry1[1 + 2 * diff_score], entry1[2 + 2 * diff_score] + av2, sd2 = entry2[1 + 2 * diff_score], entry2[2 + 2 * diff_score] + sd1 *= av1 / 100 # convert from percent sd to absolute sd + sd2 *= av2 / 100 # convert from percent sd to absolute sd + av_diff = av2 - av1 + sd_diff = (sd1 ** 2 + sd2 ** 2) ** 0.5 + percent = 100 * av_diff / av1 + percent_sd = 100 * sd_diff / av1 + print('{:24} {:10.2f} -> {:10.2f} : {:+10.2f} = {:+7.3f}% (+/-{:.2f}%)'.format(name, av1, av2, av_diff, percent, percent_sd)) + elif d1[0][0] < d2[0][0]: + d1.pop(0) + else: + d2.pop(0) + +def main(): + cmd_parser = argparse.ArgumentParser(description='Run benchmarks for MicroPython') + cmd_parser.add_argument('-t', '--diff-time', action='store_true', help='diff time outputs from a previous run') + cmd_parser.add_argument('-s', '--diff-score', action='store_true', help='diff score outputs from a previous run') + cmd_parser.add_argument('-p', '--pyboard', action='store_true', help='run tests via pyboard.py') + cmd_parser.add_argument('-d', '--device', default='/dev/ttyACM0', help='the device for pyboard.py') + cmd_parser.add_argument('-a', '--average', default='8', help='averaging number') + cmd_parser.add_argument('N', nargs=1, help='N parameter (approximate target CPU frequency)') + cmd_parser.add_argument('M', nargs=1, help='M parameter (approximate target heap in kbytes)') + cmd_parser.add_argument('files', nargs='*', help='input test files') + args = cmd_parser.parse_args() + + if args.diff_time or args.diff_score: + compute_diff(args.N[0], args.M[0], args.diff_score) + sys.exit(0) + + # N, M = 50, 25 # esp8266 + # N, M = 100, 100 # pyboard, esp32 + # N, M = 1000, 1000 # PC + N = int(args.N[0]) + M = int(args.M[0]) + n_average = int(args.average) + + if args.pyboard: + target = pyboard.Pyboard(args.device) + target.enter_raw_repl() + else: + target = MICROPYTHON + + if len(args.files) == 0: + tests_skip = ('benchrun.py',) + if M <= 25: + # These scripts are too big to be compiled by the target + tests_skip += ('bm_chaos.py', 'bm_hexiom.py', 'misc_raytrace.py') + tests = sorted( + BENCH_SCRIPT_DIR + test_file for test_file in os.listdir(BENCH_SCRIPT_DIR) + if test_file.endswith('.py') and test_file not in tests_skip + ) + else: + tests = sorted(args.files) + + print('N={} M={} n_average={}'.format(N, M, n_average)) + + run_benchmarks(target, N, M, n_average, tests) + + if isinstance(target, pyboard.Pyboard): + target.exit_raw_repl() + target.close() + +if __name__ == "__main__": + main() From 127714c3af73633ba723dad2cf7267131e7ac6c2 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 26 Jun 2019 14:24:13 +1000 Subject: [PATCH 3/6] tests/perf_bench: Add some benchmarks from python-performance. From https://github.com/python/pyperformance commit 6690642ddeda46fc5ee6e97c3ef4b2f292348ab8 --- tests/perf_bench/bm_chaos.py | 274 ++++++++++++++ tests/perf_bench/bm_fannkuch.py | 67 ++++ tests/perf_bench/bm_float.py | 70 ++++ tests/perf_bench/bm_hexiom.py | 647 ++++++++++++++++++++++++++++++++ tests/perf_bench/bm_nqueens.py | 62 +++ tests/perf_bench/bm_pidigits.py | 62 +++ 6 files changed, 1182 insertions(+) create mode 100644 tests/perf_bench/bm_chaos.py create mode 100644 tests/perf_bench/bm_fannkuch.py create mode 100644 tests/perf_bench/bm_float.py create mode 100644 tests/perf_bench/bm_hexiom.py create mode 100644 tests/perf_bench/bm_nqueens.py create mode 100644 tests/perf_bench/bm_pidigits.py diff --git a/tests/perf_bench/bm_chaos.py b/tests/perf_bench/bm_chaos.py new file mode 100644 index 0000000000000..04e04531cab88 --- /dev/null +++ b/tests/perf_bench/bm_chaos.py @@ -0,0 +1,274 @@ +# Source: https://github.com/python/pyperformance +# License: MIT + +# create chaosgame-like fractals +# Copyright (C) 2005 Carl Friedrich Bolz + +import math +import random + + +class GVector(object): + + def __init__(self, x=0, y=0, z=0): + self.x = x + self.y = y + self.z = z + + def Mag(self): + return math.sqrt(self.x ** 2 + self.y ** 2 + self.z ** 2) + + def dist(self, other): + return math.sqrt((self.x - other.x) ** 2 + + (self.y - other.y) ** 2 + + (self.z - other.z) ** 2) + + def __add__(self, other): + if not isinstance(other, GVector): + raise ValueError("Can't add GVector to " + str(type(other))) + v = GVector(self.x + other.x, self.y + other.y, self.z + other.z) + return v + + def __sub__(self, other): + return self + other * -1 + + def __mul__(self, other): + v = GVector(self.x * other, self.y * other, self.z * other) + return v + __rmul__ = __mul__ + + def linear_combination(self, other, l1, l2=None): + if l2 is None: + l2 = 1 - l1 + v = GVector(self.x * l1 + other.x * l2, + self.y * l1 + other.y * l2, + self.z * l1 + other.z * l2) + return v + + def __str__(self): + return "<%f, %f, %f>" % (self.x, self.y, self.z) + + def __repr__(self): + return "GVector(%f, %f, %f)" % (self.x, self.y, self.z) + + +class Spline(object): + """Class for representing B-Splines and NURBS of arbitrary degree""" + + def __init__(self, points, degree, knots): + """Creates a Spline. + + points is a list of GVector, degree is the degree of the Spline. + """ + if len(points) > len(knots) - degree + 1: + raise ValueError("too many control points") + elif len(points) < len(knots) - degree + 1: + raise ValueError("not enough control points") + last = knots[0] + for cur in knots[1:]: + if cur < last: + raise ValueError("knots not strictly increasing") + last = cur + self.knots = knots + self.points = points + self.degree = degree + + def GetDomain(self): + """Returns the domain of the B-Spline""" + return (self.knots[self.degree - 1], + self.knots[len(self.knots) - self.degree]) + + def __call__(self, u): + """Calculates a point of the B-Spline using de Boors Algorithm""" + dom = self.GetDomain() + if u < dom[0] or u > dom[1]: + raise ValueError("Function value not in domain") + if u == dom[0]: + return self.points[0] + if u == dom[1]: + return self.points[-1] + I = self.GetIndex(u) + d = [self.points[I - self.degree + 1 + ii] + for ii in range(self.degree + 1)] + U = self.knots + for ik in range(1, self.degree + 1): + for ii in range(I - self.degree + ik + 1, I + 2): + ua = U[ii + self.degree - ik] + ub = U[ii - 1] + co1 = (ua - u) / (ua - ub) + co2 = (u - ub) / (ua - ub) + index = ii - I + self.degree - ik - 1 + d[index] = d[index].linear_combination(d[index + 1], co1, co2) + return d[0] + + def GetIndex(self, u): + dom = self.GetDomain() + for ii in range(self.degree - 1, len(self.knots) - self.degree): + if u >= self.knots[ii] and u < self.knots[ii + 1]: + I = ii + break + else: + I = dom[1] - 1 + return I + + def __len__(self): + return len(self.points) + + def __repr__(self): + return "Spline(%r, %r, %r)" % (self.points, self.degree, self.knots) + + +def write_ppm(im, w, h, filename): + with open(filename, "wb") as f: + f.write(b'P6\n%i %i\n255\n' % (w, h)) + for j in range(h): + for i in range(w): + val = im[j * w + i] + c = val * 255 + f.write(b'%c%c%c' % (c, c, c)) + + +class Chaosgame(object): + + def __init__(self, splines, thickness, subdivs): + self.splines = splines + self.thickness = thickness + self.minx = min([p.x for spl in splines for p in spl.points]) + self.miny = min([p.y for spl in splines for p in spl.points]) + self.maxx = max([p.x for spl in splines for p in spl.points]) + self.maxy = max([p.y for spl in splines for p in spl.points]) + self.height = self.maxy - self.miny + self.width = self.maxx - self.minx + self.num_trafos = [] + maxlength = thickness * self.width / self.height + for spl in splines: + length = 0 + curr = spl(0) + for i in range(1, subdivs + 1): + last = curr + t = 1 / subdivs * i + curr = spl(t) + length += curr.dist(last) + self.num_trafos.append(max(1, int(length / maxlength * 1.5))) + self.num_total = sum(self.num_trafos) + + def get_random_trafo(self): + r = random.randrange(int(self.num_total) + 1) + l = 0 + for i in range(len(self.num_trafos)): + if r >= l and r < l + self.num_trafos[i]: + return i, random.randrange(self.num_trafos[i]) + l += self.num_trafos[i] + return len(self.num_trafos) - 1, random.randrange(self.num_trafos[-1]) + + def transform_point(self, point, trafo=None): + x = (point.x - self.minx) / self.width + y = (point.y - self.miny) / self.height + if trafo is None: + trafo = self.get_random_trafo() + start, end = self.splines[trafo[0]].GetDomain() + length = end - start + seg_length = length / self.num_trafos[trafo[0]] + t = start + seg_length * trafo[1] + seg_length * x + basepoint = self.splines[trafo[0]](t) + if t + 1 / 50000 > end: + neighbour = self.splines[trafo[0]](t - 1 / 50000) + derivative = neighbour - basepoint + else: + neighbour = self.splines[trafo[0]](t + 1 / 50000) + derivative = basepoint - neighbour + if derivative.Mag() != 0: + basepoint.x += derivative.y / derivative.Mag() * (y - 0.5) * \ + self.thickness + basepoint.y += -derivative.x / derivative.Mag() * (y - 0.5) * \ + self.thickness + else: + # can happen, especially with single precision float + pass + self.truncate(basepoint) + return basepoint + + def truncate(self, point): + if point.x >= self.maxx: + point.x = self.maxx + if point.y >= self.maxy: + point.y = self.maxy + if point.x < self.minx: + point.x = self.minx + if point.y < self.miny: + point.y = self.miny + + def create_image_chaos(self, w, h, iterations, rng_seed): + # Always use the same sequence of random numbers + # to get reproductible benchmark + random.seed(rng_seed) + + im = bytearray(w * h) + point = GVector((self.maxx + self.minx) / 2, + (self.maxy + self.miny) / 2, 0) + for _ in range(iterations): + point = self.transform_point(point) + x = (point.x - self.minx) / self.width * w + y = (point.y - self.miny) / self.height * h + x = int(x) + y = int(y) + if x == w: + x -= 1 + if y == h: + y -= 1 + im[(h - y - 1) * w + x] = 1 + + return im + + +########################################################################### +# Benchmark interface + +bm_params = { + (100, 50): (0.25, 100, 50, 50, 50, 1234), + (1000, 1000): (0.25, 200, 400, 400, 1000, 1234), + (5000, 1000): (0.25, 400, 500, 500, 7000, 1234), +} + +def bm_setup(params): + splines = [ + Spline([ + GVector(1.597, 3.304, 0.0), + GVector(1.576, 4.123, 0.0), + GVector(1.313, 5.288, 0.0), + GVector(1.619, 5.330, 0.0), + GVector(2.890, 5.503, 0.0), + GVector(2.373, 4.382, 0.0), + GVector(1.662, 4.360, 0.0)], + 3, [0, 0, 0, 1, 1, 1, 2, 2, 2]), + Spline([ + GVector(2.805, 4.017, 0.0), + GVector(2.551, 3.525, 0.0), + GVector(1.979, 2.620, 0.0), + GVector(1.979, 2.620, 0.0)], + 3, [0, 0, 0, 1, 1, 1]), + Spline([ + GVector(2.002, 4.011, 0.0), + GVector(2.335, 3.313, 0.0), + GVector(2.367, 3.233, 0.0), + GVector(2.367, 3.233, 0.0)], + 3, [0, 0, 0, 1, 1, 1]) + ] + + chaos = Chaosgame(splines, params[0], params[1]) + image = None + + def run(): + nonlocal image + _, _, width, height, iter, rng_seed = params + image = chaos.create_image_chaos(width, height, iter, rng_seed) + + def result(): + norm = params[4] + # Images are not the same when floating point behaviour is different, + # so return percentage of pixels that are set (rounded to int). + #write_ppm(image, params[2], params[3], 'out-.ppm') + pix = int(100 * sum(image) / len(image)) + return norm, pix + + return run, result diff --git a/tests/perf_bench/bm_fannkuch.py b/tests/perf_bench/bm_fannkuch.py new file mode 100644 index 0000000000000..c9782e3e9a732 --- /dev/null +++ b/tests/perf_bench/bm_fannkuch.py @@ -0,0 +1,67 @@ +# Source: https://github.com/python/pyperformance +# License: MIT + +# The Computer Language Benchmarks Game +# http://benchmarksgame.alioth.debian.org/ +# Contributed by Sokolov Yura, modified by Tupteq. + +def fannkuch(n): + count = list(range(1, n + 1)) + max_flips = 0 + m = n - 1 + r = n + check = 0 + perm1 = list(range(n)) + perm = list(range(n)) + perm1_ins = perm1.insert + perm1_pop = perm1.pop + + while 1: + if check < 30: + check += 1 + + while r != 1: + count[r - 1] = r + r -= 1 + + if perm1[0] != 0 and perm1[m] != m: + perm = perm1[:] + flips_count = 0 + k = perm[0] + while k: + perm[:k + 1] = perm[k::-1] + flips_count += 1 + k = perm[0] + + if flips_count > max_flips: + max_flips = flips_count + + while r != n: + perm1_ins(r, perm1_pop(0)) + count[r] -= 1 + if count[r] > 0: + break + r += 1 + else: + return max_flips + + +########################################################################### +# Benchmark interface + +bm_params = { + (50, 10): (5,), + (100, 10): (6,), + (500, 10): (7,), + (1000, 10): (8,), + (5000, 10): (9,), +} + +def bm_setup(params): + state = None + def run(): + nonlocal state + state = fannkuch(params[0]) + def result(): + return params[0], state + return run, result diff --git a/tests/perf_bench/bm_float.py b/tests/perf_bench/bm_float.py new file mode 100644 index 0000000000000..5a66b9bb3e62e --- /dev/null +++ b/tests/perf_bench/bm_float.py @@ -0,0 +1,70 @@ +# Source: https://github.com/python/pyperformance +# License: MIT + +# Artificial, floating point-heavy benchmark originally used by Factor. + +from math import sin, cos, sqrt + + +class Point(object): + __slots__ = ('x', 'y', 'z') + + def __init__(self, i): + self.x = x = sin(i) + self.y = cos(i) * 3 + self.z = (x * x) / 2 + + def __repr__(self): + return "" % (self.x, self.y, self.z) + + def normalize(self): + x = self.x + y = self.y + z = self.z + norm = sqrt(x * x + y * y + z * z) + self.x /= norm + self.y /= norm + self.z /= norm + + def maximize(self, other): + self.x = self.x if self.x > other.x else other.x + self.y = self.y if self.y > other.y else other.y + self.z = self.z if self.z > other.z else other.z + return self + + +def maximize(points): + next = points[0] + for p in points[1:]: + next = next.maximize(p) + return next + + +def benchmark(n): + points = [None] * n + for i in range(n): + points[i] = Point(i) + for p in points: + p.normalize() + return maximize(points) + + +########################################################################### +# Benchmark interface + +bm_params = { + (50, 25): (1, 150), + (100, 100): (1, 250), + (1000, 1000): (10, 1500), + (5000, 1000): (20, 3000), +} + +def bm_setup(params): + state = None + def run(): + nonlocal state + for _ in range(params[0]): + state = benchmark(params[1]) + def result(): + return params[0] * params[1], 'Point(%.4f, %.4f, %.4f)' % (state.x, state.y, state.z) + return run, result diff --git a/tests/perf_bench/bm_hexiom.py b/tests/perf_bench/bm_hexiom.py new file mode 100644 index 0000000000000..3a6f1f6c4bb1d --- /dev/null +++ b/tests/perf_bench/bm_hexiom.py @@ -0,0 +1,647 @@ +# Source: https://github.com/python/pyperformance +# License: MIT + +# Solver of Hexiom board game. +# Benchmark from Laurent Vaucher. +# Source: https://github.com/slowfrog/hexiom : hexiom2.py, level36.txt +# (Main function tweaked by Armin Rigo.) + + +################################## +class Dir(object): + + def __init__(self, x, y): + self.x = x + self.y = y + + +DIRS = [Dir(1, 0), + Dir(-1, 0), + Dir(0, 1), + Dir(0, -1), + Dir(1, 1), + Dir(-1, -1)] + +EMPTY = 7 + +################################## + + +class Done(object): + MIN_CHOICE_STRATEGY = 0 + MAX_CHOICE_STRATEGY = 1 + HIGHEST_VALUE_STRATEGY = 2 + FIRST_STRATEGY = 3 + MAX_NEIGHBORS_STRATEGY = 4 + MIN_NEIGHBORS_STRATEGY = 5 + + def __init__(self, count, empty=False): + self.count = count + self.cells = None if empty else [ + [0, 1, 2, 3, 4, 5, 6, EMPTY] for i in range(count)] + + def clone(self): + ret = Done(self.count, True) + ret.cells = [self.cells[i][:] for i in range(self.count)] + return ret + + def __getitem__(self, i): + return self.cells[i] + + def set_done(self, i, v): + self.cells[i] = [v] + + def already_done(self, i): + return len(self.cells[i]) == 1 + + def remove(self, i, v): + if v in self.cells[i]: + self.cells[i].remove(v) + return True + else: + return False + + def remove_all(self, v): + for i in range(self.count): + self.remove(i, v) + + def remove_unfixed(self, v): + changed = False + for i in range(self.count): + if not self.already_done(i): + if self.remove(i, v): + changed = True + return changed + + def filter_tiles(self, tiles): + for v in range(8): + if tiles[v] == 0: + self.remove_all(v) + + def next_cell_min_choice(self): + minlen = 10 + mini = -1 + for i in range(self.count): + if 1 < len(self.cells[i]) < minlen: + minlen = len(self.cells[i]) + mini = i + return mini + + def next_cell_max_choice(self): + maxlen = 1 + maxi = -1 + for i in range(self.count): + if maxlen < len(self.cells[i]): + maxlen = len(self.cells[i]) + maxi = i + return maxi + + def next_cell_highest_value(self): + maxval = -1 + maxi = -1 + for i in range(self.count): + if (not self.already_done(i)): + maxvali = max(k for k in self.cells[i] if k != EMPTY) + if maxval < maxvali: + maxval = maxvali + maxi = i + return maxi + + def next_cell_first(self): + for i in range(self.count): + if (not self.already_done(i)): + return i + return -1 + + def next_cell_max_neighbors(self, pos): + maxn = -1 + maxi = -1 + for i in range(self.count): + if not self.already_done(i): + cells_around = pos.hex.get_by_id(i).links + n = sum(1 if (self.already_done(nid) and (self[nid][0] != EMPTY)) else 0 + for nid in cells_around) + if n > maxn: + maxn = n + maxi = i + return maxi + + def next_cell_min_neighbors(self, pos): + minn = 7 + mini = -1 + for i in range(self.count): + if not self.already_done(i): + cells_around = pos.hex.get_by_id(i).links + n = sum(1 if (self.already_done(nid) and (self[nid][0] != EMPTY)) else 0 + for nid in cells_around) + if n < minn: + minn = n + mini = i + return mini + + def next_cell(self, pos, strategy=HIGHEST_VALUE_STRATEGY): + if strategy == Done.HIGHEST_VALUE_STRATEGY: + return self.next_cell_highest_value() + elif strategy == Done.MIN_CHOICE_STRATEGY: + return self.next_cell_min_choice() + elif strategy == Done.MAX_CHOICE_STRATEGY: + return self.next_cell_max_choice() + elif strategy == Done.FIRST_STRATEGY: + return self.next_cell_first() + elif strategy == Done.MAX_NEIGHBORS_STRATEGY: + return self.next_cell_max_neighbors(pos) + elif strategy == Done.MIN_NEIGHBORS_STRATEGY: + return self.next_cell_min_neighbors(pos) + else: + raise Exception("Wrong strategy: %d" % strategy) + +################################## + + +class Node(object): + + def __init__(self, pos, id, links): + self.pos = pos + self.id = id + self.links = links + +################################## + + +class Hex(object): + + def __init__(self, size): + self.size = size + self.count = 3 * size * (size - 1) + 1 + self.nodes_by_id = self.count * [None] + self.nodes_by_pos = {} + id = 0 + for y in range(size): + for x in range(size + y): + pos = (x, y) + node = Node(pos, id, []) + self.nodes_by_pos[pos] = node + self.nodes_by_id[node.id] = node + id += 1 + for y in range(1, size): + for x in range(y, size * 2 - 1): + ry = size + y - 1 + pos = (x, ry) + node = Node(pos, id, []) + self.nodes_by_pos[pos] = node + self.nodes_by_id[node.id] = node + id += 1 + + def link_nodes(self): + for node in self.nodes_by_id: + (x, y) = node.pos + for dir in DIRS: + nx = x + dir.x + ny = y + dir.y + if self.contains_pos((nx, ny)): + node.links.append(self.nodes_by_pos[(nx, ny)].id) + + def contains_pos(self, pos): + return pos in self.nodes_by_pos + + def get_by_pos(self, pos): + return self.nodes_by_pos[pos] + + def get_by_id(self, id): + return self.nodes_by_id[id] + + +################################## +class Pos(object): + + def __init__(self, hex, tiles, done=None): + self.hex = hex + self.tiles = tiles + self.done = Done(hex.count) if done is None else done + + def clone(self): + return Pos(self.hex, self.tiles, self.done.clone()) + +################################## + + +def constraint_pass(pos, last_move=None): + changed = False + left = pos.tiles[:] + done = pos.done + + # Remove impossible values from free cells + free_cells = (range(done.count) if last_move is None + else pos.hex.get_by_id(last_move).links) + for i in free_cells: + if not done.already_done(i): + vmax = 0 + vmin = 0 + cells_around = pos.hex.get_by_id(i).links + for nid in cells_around: + if done.already_done(nid): + if done[nid][0] != EMPTY: + vmin += 1 + vmax += 1 + else: + vmax += 1 + + for num in range(7): + if (num < vmin) or (num > vmax): + if done.remove(i, num): + changed = True + + # Computes how many of each value is still free + for cell in done.cells: + if len(cell) == 1: + left[cell[0]] -= 1 + + for v in range(8): + # If there is none, remove the possibility from all tiles + if (pos.tiles[v] > 0) and (left[v] == 0): + if done.remove_unfixed(v): + changed = True + else: + possible = sum((1 if v in cell else 0) for cell in done.cells) + # If the number of possible cells for a value is exactly the number of available tiles + # put a tile in each cell + if pos.tiles[v] == possible: + for i in range(done.count): + cell = done.cells[i] + if (not done.already_done(i)) and (v in cell): + done.set_done(i, v) + changed = True + + # Force empty or non-empty around filled cells + filled_cells = (range(done.count) if last_move is None + else [last_move]) + for i in filled_cells: + if done.already_done(i): + num = done[i][0] + empties = 0 + filled = 0 + unknown = [] + cells_around = pos.hex.get_by_id(i).links + for nid in cells_around: + if done.already_done(nid): + if done[nid][0] == EMPTY: + empties += 1 + else: + filled += 1 + else: + unknown.append(nid) + if len(unknown) > 0: + if num == filled: + for u in unknown: + if EMPTY in done[u]: + done.set_done(u, EMPTY) + changed = True + # else: + # raise Exception("Houston, we've got a problem") + elif num == filled + len(unknown): + for u in unknown: + if done.remove(u, EMPTY): + changed = True + + return changed + + +ASCENDING = 1 +DESCENDING = -1 + + +def find_moves(pos, strategy, order): + done = pos.done + cell_id = done.next_cell(pos, strategy) + if cell_id < 0: + return [] + + if order == ASCENDING: + return [(cell_id, v) for v in done[cell_id]] + else: + # Try higher values first and EMPTY last + moves = list(reversed([(cell_id, v) + for v in done[cell_id] if v != EMPTY])) + if EMPTY in done[cell_id]: + moves.append((cell_id, EMPTY)) + return moves + + +def play_move(pos, move): + (cell_id, i) = move + pos.done.set_done(cell_id, i) + + +def print_pos(pos, output): + hex = pos.hex + done = pos.done + size = hex.size + for y in range(size): + print(" " * (size - y - 1), end="", file=output) + for x in range(size + y): + pos2 = (x, y) + id = hex.get_by_pos(pos2).id + if done.already_done(id): + c = done[id][0] if done[id][0] != EMPTY else "." + else: + c = "?" + print("%s " % c, end="", file=output) + print(end="\n", file=output) + for y in range(1, size): + print(" " * y, end="", file=output) + for x in range(y, size * 2 - 1): + ry = size + y - 1 + pos2 = (x, ry) + id = hex.get_by_pos(pos2).id + if done.already_done(id): + c = done[id][0] if done[id][0] != EMPTY else "." + else: + c = "?" + print("%s " % c, end="", file=output) + print(end="\n", file=output) + + +OPEN = 0 +SOLVED = 1 +IMPOSSIBLE = -1 + + +def solved(pos, output, verbose=False): + hex = pos.hex + tiles = pos.tiles[:] + done = pos.done + exact = True + all_done = True + for i in range(hex.count): + if len(done[i]) == 0: + return IMPOSSIBLE + elif done.already_done(i): + num = done[i][0] + tiles[num] -= 1 + if (tiles[num] < 0): + return IMPOSSIBLE + vmax = 0 + vmin = 0 + if num != EMPTY: + cells_around = hex.get_by_id(i).links + for nid in cells_around: + if done.already_done(nid): + if done[nid][0] != EMPTY: + vmin += 1 + vmax += 1 + else: + vmax += 1 + + if (num < vmin) or (num > vmax): + return IMPOSSIBLE + if num != vmin: + exact = False + else: + all_done = False + + if (not all_done) or (not exact): + return OPEN + + print_pos(pos, output) + return SOLVED + + +def solve_step(prev, strategy, order, output, first=False): + if first: + pos = prev.clone() + while constraint_pass(pos): + pass + else: + pos = prev + + moves = find_moves(pos, strategy, order) + if len(moves) == 0: + return solved(pos, output) + else: + for move in moves: + # print("Trying (%d, %d)" % (move[0], move[1])) + ret = OPEN + new_pos = pos.clone() + play_move(new_pos, move) + # print_pos(new_pos) + while constraint_pass(new_pos, move[0]): + pass + cur_status = solved(new_pos, output) + if cur_status != OPEN: + ret = cur_status + else: + ret = solve_step(new_pos, strategy, order, output) + if ret == SOLVED: + return SOLVED + return IMPOSSIBLE + + +def check_valid(pos): + hex = pos.hex + tiles = pos.tiles + # fill missing entries in tiles + tot = 0 + for i in range(8): + if tiles[i] > 0: + tot += tiles[i] + else: + tiles[i] = 0 + # check total + if tot != hex.count: + raise Exception( + "Invalid input. Expected %d tiles, got %d." % (hex.count, tot)) + + +def solve(pos, strategy, order, output): + check_valid(pos) + return solve_step(pos, strategy, order, output, first=True) + + +# TODO Write an 'iterator' to go over all x,y positions + +def read_file(file): + lines = [line.strip("\r\n") for line in file.splitlines()] + size = int(lines[0]) + hex = Hex(size) + linei = 1 + tiles = 8 * [0] + done = Done(hex.count) + for y in range(size): + line = lines[linei][size - y - 1:] + p = 0 + for x in range(size + y): + tile = line[p:p + 2] + p += 2 + if tile[1] == ".": + inctile = EMPTY + else: + inctile = int(tile) + tiles[inctile] += 1 + # Look for locked tiles + if tile[0] == "+": + # print("Adding locked tile: %d at pos %d, %d, id=%d" % + # (inctile, x, y, hex.get_by_pos((x, y)).id)) + done.set_done(hex.get_by_pos((x, y)).id, inctile) + + linei += 1 + for y in range(1, size): + ry = size - 1 + y + line = lines[linei][y:] + p = 0 + for x in range(y, size * 2 - 1): + tile = line[p:p + 2] + p += 2 + if tile[1] == ".": + inctile = EMPTY + else: + inctile = int(tile) + tiles[inctile] += 1 + # Look for locked tiles + if tile[0] == "+": + # print("Adding locked tile: %d at pos %d, %d, id=%d" % + # (inctile, x, ry, hex.get_by_pos((x, ry)).id)) + done.set_done(hex.get_by_pos((x, ry)).id, inctile) + linei += 1 + hex.link_nodes() + done.filter_tiles(tiles) + return Pos(hex, tiles, done) + + +def solve_file(file, strategy, order, output): + pos = read_file(file) + solve(pos, strategy, order, output) + + +LEVELS = {} + +LEVELS[2] = (""" +2 + . 1 + . 1 1 + 1 . +""", """\ + 1 1 +. . . + 1 1 +""") + +LEVELS[10] = (""" +3 + +.+. . + +. 0 . 2 + . 1+2 1 . + 2 . 0+. + .+.+. +""", """\ + . . 1 + . 1 . 2 +0 . 2 2 . + . . . . + 0 . . +""") + +LEVELS[20] = (""" +3 + . 5 4 + . 2+.+1 + . 3+2 3 . + +2+. 5 . + . 3 . +""", """\ + 3 3 2 + 4 5 . 1 +3 5 2 . . + 2 . . . + . . . +""") + +LEVELS[25] = (""" +3 + 4 . . + . . 2 . + 4 3 2 . 4 + 2 2 3 . + 4 2 4 +""", """\ + 3 4 2 + 2 4 4 . +. . . 4 2 + . 2 4 3 + . 2 . +""") + +LEVELS[30] = (""" +4 + 5 5 . . + 3 . 2+2 6 + 3 . 2 . 5 . + . 3 3+4 4 . 3 + 4 5 4 . 5 4 + 5+2 . . 3 + 4 . . . +""", """\ + 3 4 3 . + 4 6 5 2 . + 2 5 5 . . 2 +. . 5 4 . 4 3 + . 3 5 4 5 4 + . 2 . 3 3 + . . . . +""") + +LEVELS[36] = (""" +4 + 2 1 1 2 + 3 3 3 . . + 2 3 3 . 4 . + . 2 . 2 4 3 2 + 2 2 . . . 2 + 4 3 4 . . + 3 2 3 3 +""", """\ + 3 4 3 2 + 3 4 4 . 3 + 2 . . 3 4 3 +2 . 1 . 3 . 2 + 3 3 . 2 . 2 + 3 . 2 . 2 + 2 2 . 1 +""") + + +########################################################################### +# Benchmark interface + +bm_params = { + (100, 100): (1, 10, DESCENDING, Done.FIRST_STRATEGY), + (1000, 1000): (1, 25, DESCENDING, Done.FIRST_STRATEGY), + (5000, 1000): (10, 25, DESCENDING, Done.FIRST_STRATEGY), +} + +def bm_setup(params): + try: + import uio as io + except ImportError: + import io + + loops, level, order, strategy = params + + board, solution = LEVELS[level] + board = board.strip() + expected = solution.rstrip() + output = None + + def run(): + nonlocal output + for _ in range(loops): + stream = io.StringIO() + solve_file(board, strategy, order, stream) + output = stream.getvalue() + stream = None + + def result(): + norm = params[0] * params[1] + out = '\n'.join(line.rstrip() for line in output.splitlines()) + return norm, ((out == expected), out) + + return run, result diff --git a/tests/perf_bench/bm_nqueens.py b/tests/perf_bench/bm_nqueens.py new file mode 100644 index 0000000000000..87e7245c04aef --- /dev/null +++ b/tests/perf_bench/bm_nqueens.py @@ -0,0 +1,62 @@ +# Source: https://github.com/python/pyperformance +# License: MIT + +# Simple, brute-force N-Queens solver. +# author: collinwinter@google.com (Collin Winter) +# n_queens function: Copyright 2009 Raymond Hettinger + +# Pure-Python implementation of itertools.permutations(). +def permutations(iterable, r=None): + """permutations(range(3), 2) --> (0,1) (0,2) (1,0) (1,2) (2,0) (2,1)""" + pool = tuple(iterable) + n = len(pool) + if r is None: + r = n + indices = list(range(n)) + cycles = list(range(n - r + 1, n + 1))[::-1] + yield tuple(pool[i] for i in indices[:r]) + while n: + for i in reversed(range(r)): + cycles[i] -= 1 + if cycles[i] == 0: + indices[i:] = indices[i + 1:] + indices[i:i + 1] + cycles[i] = n - i + else: + j = cycles[i] + indices[i], indices[-j] = indices[-j], indices[i] + yield tuple(pool[i] for i in indices[:r]) + break + else: + return + +# From http://code.activestate.com/recipes/576647/ +def n_queens(queen_count): + """N-Queens solver. + Args: queen_count: the number of queens to solve for, same as board size. + Yields: Solutions to the problem, each yielded value is a N-tuple. + """ + cols = range(queen_count) + for vec in permutations(cols): + if (queen_count == len(set(vec[i] + i for i in cols)) + == len(set(vec[i] - i for i in cols))): + yield vec + +########################################################################### +# Benchmark interface + +bm_params = { + (50, 25): (1, 5), + (100, 25): (1, 6), + (1000, 100): (1, 7), + (5000, 100): (1, 8), +} + +def bm_setup(params): + res = None + def run(): + nonlocal res + for _ in range(params[0]): + res = len(list(n_queens(params[1]))) + def result(): + return params[0] * 10 ** (params[1] - 3), res + return run, result diff --git a/tests/perf_bench/bm_pidigits.py b/tests/perf_bench/bm_pidigits.py new file mode 100644 index 0000000000000..ca2e5297d01af --- /dev/null +++ b/tests/perf_bench/bm_pidigits.py @@ -0,0 +1,62 @@ +# Source: https://github.com/python/pyperformance +# License: MIT + +# Calculating some of the digits of π. +# This benchmark stresses big integer arithmetic. +# Adapted from code on: http://benchmarksgame.alioth.debian.org/ + + +def compose(a, b): + aq, ar, as_, at = a + bq, br, bs, bt = b + return (aq * bq, + aq * br + ar * bt, + as_ * bq + at * bs, + as_ * br + at * bt) + + +def extract(z, j): + q, r, s, t = z + return (q * j + r) // (s * j + t) + + +def gen_pi_digits(n): + z = (1, 0, 0, 1) + k = 1 + digs = [] + for _ in range(n): + y = extract(z, 3) + while y != extract(z, 4): + z = compose(z, (k, 4 * k + 2, 0, 2 * k + 1)) + k += 1 + y = extract(z, 3) + z = compose((10, -10 * y, 0, 1), z) + digs.append(y) + return digs + + +########################################################################### +# Benchmark interface + +bm_params = { + (50, 25): (1, 35), + (100, 100): (1, 65), + (1000, 1000): (2, 250), + (5000, 1000): (3, 350), +} + +def bm_setup(params): + state = None + + def run(): + nonlocal state + nloop, ndig = params + ndig = params[1] + for _ in range(nloop): + state = None # free previous result + state = gen_pi_digits(ndig) + + def result(): + return params[0] * params[1], ''.join(str(d) for d in state) + + return run, result From 73c269414f41dac87122963dc1fc456442de8b85 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 26 Jun 2019 14:26:20 +1000 Subject: [PATCH 4/6] tests/perf_bench: Add some miscellaneous performance benchmarks. misc_aes.py and misc_mandel.py are adapted from sources in this repository. misc_pystone.py is the standard Python pystone test. misc_raytrace.py is written from scratch. --- tests/perf_bench/misc_aes.py | 225 +++++++++++++++++++++++++++ tests/perf_bench/misc_mandel.py | 31 ++++ tests/perf_bench/misc_pystone.py | 226 ++++++++++++++++++++++++++++ tests/perf_bench/misc_raytrace.py | 242 ++++++++++++++++++++++++++++++ 4 files changed, 724 insertions(+) create mode 100644 tests/perf_bench/misc_aes.py create mode 100644 tests/perf_bench/misc_mandel.py create mode 100644 tests/perf_bench/misc_pystone.py create mode 100644 tests/perf_bench/misc_raytrace.py diff --git a/tests/perf_bench/misc_aes.py b/tests/perf_bench/misc_aes.py new file mode 100644 index 0000000000000..5413a06b12b19 --- /dev/null +++ b/tests/perf_bench/misc_aes.py @@ -0,0 +1,225 @@ +# Pure Python AES encryption routines. +# +# AES is integer based and inplace so doesn't use the heap. It is therefore +# a good test of raw performance and correctness of the VM/runtime. +# +# The AES code comes first (code originates from a C version authored by D.P.George) +# and then the test harness at the bottom. +# +# MIT license; Copyright (c) 2016 Damien P. George on behalf of Pycom Ltd + +################################################################## +# discrete arithmetic routines, mostly from a precomputed table + +# non-linear, invertible, substitution box +aes_s_box_table = bytes(( + 0x63,0x7c,0x77,0x7b,0xf2,0x6b,0x6f,0xc5,0x30,0x01,0x67,0x2b,0xfe,0xd7,0xab,0x76, + 0xca,0x82,0xc9,0x7d,0xfa,0x59,0x47,0xf0,0xad,0xd4,0xa2,0xaf,0x9c,0xa4,0x72,0xc0, + 0xb7,0xfd,0x93,0x26,0x36,0x3f,0xf7,0xcc,0x34,0xa5,0xe5,0xf1,0x71,0xd8,0x31,0x15, + 0x04,0xc7,0x23,0xc3,0x18,0x96,0x05,0x9a,0x07,0x12,0x80,0xe2,0xeb,0x27,0xb2,0x75, + 0x09,0x83,0x2c,0x1a,0x1b,0x6e,0x5a,0xa0,0x52,0x3b,0xd6,0xb3,0x29,0xe3,0x2f,0x84, + 0x53,0xd1,0x00,0xed,0x20,0xfc,0xb1,0x5b,0x6a,0xcb,0xbe,0x39,0x4a,0x4c,0x58,0xcf, + 0xd0,0xef,0xaa,0xfb,0x43,0x4d,0x33,0x85,0x45,0xf9,0x02,0x7f,0x50,0x3c,0x9f,0xa8, + 0x51,0xa3,0x40,0x8f,0x92,0x9d,0x38,0xf5,0xbc,0xb6,0xda,0x21,0x10,0xff,0xf3,0xd2, + 0xcd,0x0c,0x13,0xec,0x5f,0x97,0x44,0x17,0xc4,0xa7,0x7e,0x3d,0x64,0x5d,0x19,0x73, + 0x60,0x81,0x4f,0xdc,0x22,0x2a,0x90,0x88,0x46,0xee,0xb8,0x14,0xde,0x5e,0x0b,0xdb, + 0xe0,0x32,0x3a,0x0a,0x49,0x06,0x24,0x5c,0xc2,0xd3,0xac,0x62,0x91,0x95,0xe4,0x79, + 0xe7,0xc8,0x37,0x6d,0x8d,0xd5,0x4e,0xa9,0x6c,0x56,0xf4,0xea,0x65,0x7a,0xae,0x08, + 0xba,0x78,0x25,0x2e,0x1c,0xa6,0xb4,0xc6,0xe8,0xdd,0x74,0x1f,0x4b,0xbd,0x8b,0x8a, + 0x70,0x3e,0xb5,0x66,0x48,0x03,0xf6,0x0e,0x61,0x35,0x57,0xb9,0x86,0xc1,0x1d,0x9e, + 0xe1,0xf8,0x98,0x11,0x69,0xd9,0x8e,0x94,0x9b,0x1e,0x87,0xe9,0xce,0x55,0x28,0xdf, + 0x8c,0xa1,0x89,0x0d,0xbf,0xe6,0x42,0x68,0x41,0x99,0x2d,0x0f,0xb0,0x54,0xbb,0x16, +)) + +# multiplication of polynomials modulo x^8 + x^4 + x^3 + x + 1 = 0x11b +def aes_gf8_mul_2(x): + if x & 0x80: + return (x << 1) ^ 0x11b + else: + return x << 1 + +def aes_gf8_mul_3(x): + return x ^ aes_gf8_mul_2(x) + +# non-linear, invertible, substitution box +def aes_s_box(a): + return aes_s_box_table[a & 0xff] + +# return 0x02^(a-1) in GF(2^8) +def aes_r_con(a): + ans = 1 + while a > 1: + ans <<= 1; + if ans & 0x100: + ans ^= 0x11b + a -= 1 + return ans + +################################################################## +# basic AES algorithm; see FIPS-197 + +# all inputs must be size 16 +def aes_add_round_key(state, w): + for i in range(16): + state[i] ^= w[i] + +# combined sub_bytes, shift_rows, mix_columns, add_round_key +# all inputs must be size 16 +def aes_sb_sr_mc_ark(state, w, w_idx, temp): + temp_idx = 0 + for i in range(4): + x0 = aes_s_box_table[state[i * 4]] + x1 = aes_s_box_table[state[1 + ((i + 1) & 3) * 4]] + x2 = aes_s_box_table[state[2 + ((i + 2) & 3) * 4]] + x3 = aes_s_box_table[state[3 + ((i + 3) & 3) * 4]] + temp[temp_idx] = aes_gf8_mul_2(x0) ^ aes_gf8_mul_3(x1) ^ x2 ^ x3 ^ w[w_idx] + temp[temp_idx + 1] = x0 ^ aes_gf8_mul_2(x1) ^ aes_gf8_mul_3(x2) ^ x3 ^ w[w_idx + 1] + temp[temp_idx + 2] = x0 ^ x1 ^ aes_gf8_mul_2(x2) ^ aes_gf8_mul_3(x3) ^ w[w_idx + 2] + temp[temp_idx + 3] = aes_gf8_mul_3(x0) ^ x1 ^ x2 ^ aes_gf8_mul_2(x3) ^ w[w_idx + 3] + w_idx += 4 + temp_idx += 4 + for i in range(16): + state[i] = temp[i] + +# combined sub_bytes, shift_rows, add_round_key +# all inputs must be size 16 +def aes_sb_sr_ark(state, w, w_idx, temp): + temp_idx = 0 + for i in range(4): + x0 = aes_s_box_table[state[i * 4]] + x1 = aes_s_box_table[state[1 + ((i + 1) & 3) * 4]] + x2 = aes_s_box_table[state[2 + ((i + 2) & 3) * 4]] + x3 = aes_s_box_table[state[3 + ((i + 3) & 3) * 4]] + temp[temp_idx] = x0 ^ w[w_idx] + temp[temp_idx + 1] = x1 ^ w[w_idx + 1] + temp[temp_idx + 2] = x2 ^ w[w_idx + 2] + temp[temp_idx + 3] = x3 ^ w[w_idx + 3] + w_idx += 4 + temp_idx += 4 + for i in range(16): + state[i] = temp[i] + +# take state as input and change it to the next state in the sequence +# state and temp have size 16, w has size 16 * (Nr + 1), Nr >= 1 +def aes_state(state, w, temp, nr): + aes_add_round_key(state, w) + w_idx = 16 + for i in range(nr - 1): + aes_sb_sr_mc_ark(state, w, w_idx, temp) + w_idx += 16 + aes_sb_sr_ark(state, w, w_idx, temp) + +# expand 'key' to 'w' for use with aes_state +# key has size 4 * Nk, w has size 16 * (Nr + 1), temp has size 16 +def aes_key_expansion(key, w, temp, nk, nr): + for i in range(4 * nk): + w[i] = key[i] + w_idx = 4 * nk - 4 + for i in range(nk, 4 * (nr + 1)): + t = temp + t_idx = 0 + if i % nk == 0: + t[0] = aes_s_box(w[w_idx + 1]) ^ aes_r_con(i // nk) + for j in range(1, 4): + t[j] = aes_s_box(w[w_idx + (j + 1) % 4]) + elif nk > 6 and i % nk == 4: + for j in range(0, 4): + t[j] = aes_s_box(w[w_idx + j]) + else: + t = w + t_idx = w_idx + w_idx += 4 + for j in range(4): + w[w_idx + j] = w[w_idx + j - 4 * nk] ^ t[t_idx + j] + +################################################################## +# simple use of AES algorithm, using output feedback (OFB) mode + +class AES: + def __init__(self, keysize): + if keysize == 128: + self.nk = 4 + self.nr = 10 + elif keysize == 192: + self.nk = 6 + self.nr = 12 + else: + assert keysize == 256 + self.nk = 8 + self.nr = 14 + + self.state = bytearray(16) + self.w = bytearray(16 * (self.nr + 1)) + self.temp = bytearray(16) + self.state_pos = 16 + + def set_key(self, key): + aes_key_expansion(key, self.w, self.temp, self.nk, self.nr) + self.state_pos = 16 + + def set_iv(self, iv): + for i in range(16): + self.state[i] = iv[i] + self.state_pos = 16; + + def get_some_state(self, n_needed): + if self.state_pos >= 16: + aes_state(self.state, self.w, self.temp, self.nr) + self.state_pos = 0 + n = 16 - self.state_pos + if n > n_needed: + n = n_needed + return n + + def apply_to(self, data): + idx = 0 + n = len(data) + while n > 0: + ln = self.get_some_state(n) + n -= ln + for i in range(ln): + data[idx + i] ^= self.state[self.state_pos + i] + idx += ln + self.state_pos += n + +########################################################################### +# Benchmark interface + +bm_params = { + (50, 25): (1, 16), + (100, 100): (1, 32), + (1000, 1000): (4, 256), + (5000, 1000): (20, 256), +} + +def bm_setup(params): + nloop, datalen = params + + aes = AES(256) + key = bytearray(256 // 8) + iv = bytearray(16) + data = bytearray(datalen) + # from now on we don't use the heap + + def run(): + for loop in range(nloop): + # encrypt + aes.set_key(key) + aes.set_iv(iv) + for i in range(2): + aes.apply_to(data) + + # decrypt + aes.set_key(key) + aes.set_iv(iv) + for i in range(2): + aes.apply_to(data) + + # verify + for i in range(len(data)): + assert data[i] == 0 + + def result(): + return params[0] * params[1], True + + return run, result diff --git a/tests/perf_bench/misc_mandel.py b/tests/perf_bench/misc_mandel.py new file mode 100644 index 0000000000000..a4b789136d9d1 --- /dev/null +++ b/tests/perf_bench/misc_mandel.py @@ -0,0 +1,31 @@ +# Compute the Mandelbrot set, to test complex numbers + +def mandelbrot(w, h): + def in_set(c): + z = 0 + for i in range(32): + z = z * z + c + if abs(z) > 100: + return i + return 0 + + img = bytearray(w * h) + + xscale = ((w - 1) / 2.4) + yscale = ((h - 1) / 3.2) + for v in range(h): + line = memoryview(img)[v * w:v * w + w] + for u in range(w): + c = in_set(complex(v / yscale - 2.3, u / xscale - 1.2)) + line[u] = c + + return img + +bm_params = { + (100, 100): (20, 20), + (1000, 1000): (80, 80), + (5000, 1000): (150, 150), +} + +def bm_setup(ps): + return lambda: mandelbrot(ps[0], ps[1]), lambda: (ps[0] * ps[1], None) diff --git a/tests/perf_bench/misc_pystone.py b/tests/perf_bench/misc_pystone.py new file mode 100644 index 0000000000000..88626774b254a --- /dev/null +++ b/tests/perf_bench/misc_pystone.py @@ -0,0 +1,226 @@ +""" +"PYSTONE" Benchmark Program + +Version: Python/1.2 (corresponds to C/1.1 plus 3 Pystone fixes) + +Author: Reinhold P. Weicker, CACM Vol 27, No 10, 10/84 pg. 1013. + + Translated from ADA to C by Rick Richardson. + Every method to preserve ADA-likeness has been used, + at the expense of C-ness. + + Translated from C to Python by Guido van Rossum. +""" + +__version__ = "1.2" + +[Ident1, Ident2, Ident3, Ident4, Ident5] = range(1, 6) + +class Record: + + def __init__(self, PtrComp = None, Discr = 0, EnumComp = 0, + IntComp = 0, StringComp = 0): + self.PtrComp = PtrComp + self.Discr = Discr + self.EnumComp = EnumComp + self.IntComp = IntComp + self.StringComp = StringComp + + def copy(self): + return Record(self.PtrComp, self.Discr, self.EnumComp, + self.IntComp, self.StringComp) + +TRUE = 1 +FALSE = 0 + +def Setup(): + global IntGlob + global BoolGlob + global Char1Glob + global Char2Glob + global Array1Glob + global Array2Glob + + IntGlob = 0 + BoolGlob = FALSE + Char1Glob = '\0' + Char2Glob = '\0' + Array1Glob = [0]*51 + Array2Glob = [x[:] for x in [Array1Glob]*51] + +def Proc0(loops): + global IntGlob + global BoolGlob + global Char1Glob + global Char2Glob + global Array1Glob + global Array2Glob + global PtrGlb + global PtrGlbNext + + PtrGlbNext = Record() + PtrGlb = Record() + PtrGlb.PtrComp = PtrGlbNext + PtrGlb.Discr = Ident1 + PtrGlb.EnumComp = Ident3 + PtrGlb.IntComp = 40 + PtrGlb.StringComp = "DHRYSTONE PROGRAM, SOME STRING" + String1Loc = "DHRYSTONE PROGRAM, 1'ST STRING" + Array2Glob[8][7] = 10 + + for i in range(loops): + Proc5() + Proc4() + IntLoc1 = 2 + IntLoc2 = 3 + String2Loc = "DHRYSTONE PROGRAM, 2'ND STRING" + EnumLoc = Ident2 + BoolGlob = not Func2(String1Loc, String2Loc) + while IntLoc1 < IntLoc2: + IntLoc3 = 5 * IntLoc1 - IntLoc2 + IntLoc3 = Proc7(IntLoc1, IntLoc2) + IntLoc1 = IntLoc1 + 1 + Proc8(Array1Glob, Array2Glob, IntLoc1, IntLoc3) + PtrGlb = Proc1(PtrGlb) + CharIndex = 'A' + while CharIndex <= Char2Glob: + if EnumLoc == Func1(CharIndex, 'C'): + EnumLoc = Proc6(Ident1) + CharIndex = chr(ord(CharIndex)+1) + IntLoc3 = IntLoc2 * IntLoc1 + IntLoc2 = IntLoc3 // IntLoc1 + IntLoc2 = 7 * (IntLoc3 - IntLoc2) - IntLoc1 + IntLoc1 = Proc2(IntLoc1) + +def Proc1(PtrParIn): + PtrParIn.PtrComp = NextRecord = PtrGlb.copy() + PtrParIn.IntComp = 5 + NextRecord.IntComp = PtrParIn.IntComp + NextRecord.PtrComp = PtrParIn.PtrComp + NextRecord.PtrComp = Proc3(NextRecord.PtrComp) + if NextRecord.Discr == Ident1: + NextRecord.IntComp = 6 + NextRecord.EnumComp = Proc6(PtrParIn.EnumComp) + NextRecord.PtrComp = PtrGlb.PtrComp + NextRecord.IntComp = Proc7(NextRecord.IntComp, 10) + else: + PtrParIn = NextRecord.copy() + NextRecord.PtrComp = None + return PtrParIn + +def Proc2(IntParIO): + IntLoc = IntParIO + 10 + while 1: + if Char1Glob == 'A': + IntLoc = IntLoc - 1 + IntParIO = IntLoc - IntGlob + EnumLoc = Ident1 + if EnumLoc == Ident1: + break + return IntParIO + +def Proc3(PtrParOut): + global IntGlob + + if PtrGlb is not None: + PtrParOut = PtrGlb.PtrComp + else: + IntGlob = 100 + PtrGlb.IntComp = Proc7(10, IntGlob) + return PtrParOut + +def Proc4(): + global Char2Glob + + BoolLoc = Char1Glob == 'A' + BoolLoc = BoolLoc or BoolGlob + Char2Glob = 'B' + +def Proc5(): + global Char1Glob + global BoolGlob + + Char1Glob = 'A' + BoolGlob = FALSE + +def Proc6(EnumParIn): + EnumParOut = EnumParIn + if not Func3(EnumParIn): + EnumParOut = Ident4 + if EnumParIn == Ident1: + EnumParOut = Ident1 + elif EnumParIn == Ident2: + if IntGlob > 100: + EnumParOut = Ident1 + else: + EnumParOut = Ident4 + elif EnumParIn == Ident3: + EnumParOut = Ident2 + elif EnumParIn == Ident4: + pass + elif EnumParIn == Ident5: + EnumParOut = Ident3 + return EnumParOut + +def Proc7(IntParI1, IntParI2): + IntLoc = IntParI1 + 2 + IntParOut = IntParI2 + IntLoc + return IntParOut + +def Proc8(Array1Par, Array2Par, IntParI1, IntParI2): + global IntGlob + + IntLoc = IntParI1 + 5 + Array1Par[IntLoc] = IntParI2 + Array1Par[IntLoc+1] = Array1Par[IntLoc] + Array1Par[IntLoc+30] = IntLoc + for IntIndex in range(IntLoc, IntLoc+2): + Array2Par[IntLoc][IntIndex] = IntLoc + Array2Par[IntLoc][IntLoc-1] = Array2Par[IntLoc][IntLoc-1] + 1 + Array2Par[IntLoc+20][IntLoc] = Array1Par[IntLoc] + IntGlob = 5 + +def Func1(CharPar1, CharPar2): + CharLoc1 = CharPar1 + CharLoc2 = CharLoc1 + if CharLoc2 != CharPar2: + return Ident1 + else: + return Ident2 + +def Func2(StrParI1, StrParI2): + IntLoc = 1 + while IntLoc <= 1: + if Func1(StrParI1[IntLoc], StrParI2[IntLoc+1]) == Ident1: + CharLoc = 'A' + IntLoc = IntLoc + 1 + if CharLoc >= 'W' and CharLoc <= 'Z': + IntLoc = 7 + if CharLoc == 'X': + return TRUE + else: + if StrParI1 > StrParI2: + IntLoc = IntLoc + 7 + return TRUE + else: + return FALSE + +def Func3(EnumParIn): + EnumLoc = EnumParIn + if EnumLoc == Ident3: return TRUE + return FALSE + + +########################################################################### +# Benchmark interface + +bm_params = { + (50, 10): (80,), + (100, 10): (300,), + (1000, 10): (4000,), + (5000, 10): (20000,), +} + +def bm_setup(params): + Setup() + return lambda: Proc0(params[0]), lambda: (params[0], 0) diff --git a/tests/perf_bench/misc_raytrace.py b/tests/perf_bench/misc_raytrace.py new file mode 100644 index 0000000000000..76d4194bceb3b --- /dev/null +++ b/tests/perf_bench/misc_raytrace.py @@ -0,0 +1,242 @@ +# A simple ray tracer +# MIT license; Copyright (c) 2019 Damien P. George + +INF = 1e30 +EPS = 1e-6 + +class Vec: + def __init__(self, x, y, z): + self.x, self.y, self.z = x, y, z + + def __neg__(self): + return Vec(-self.x, -self.y, -self.z) + + def __add__(self, rhs): + return Vec(self.x + rhs.x, self.y + rhs.y, self.z + rhs.z) + + def __sub__(self, rhs): + return Vec(self.x - rhs.x, self.y - rhs.y, self.z - rhs.z) + + def __mul__(self, rhs): + return Vec(self.x * rhs, self.y * rhs, self.z * rhs) + + def length(self): + return (self.x ** 2 + self.y ** 2 + self.z ** 2) ** 0.5 + + def normalise(self): + l = self.length() + return Vec(self.x / l, self.y / l, self.z / l) + + def dot(self, rhs): + return self.x * rhs.x + self.y * rhs.y + self.z * rhs.z + +RGB = Vec + +class Ray: + def __init__(self, p, d): + self.p, self.d = p, d + +class View: + def __init__(self, width, height, depth, pos, xdir, ydir, zdir): + self.width = width + self.height = height + self.depth = depth + self.pos = pos + self.xdir = xdir + self.ydir = ydir + self.zdir = zdir + + def calc_dir(self, dx, dy): + return (self.xdir * dx + self.ydir * dy + self.zdir * self.depth).normalise() + +class Light: + def __init__(self, pos, colour, casts_shadows): + self.pos = pos + self.colour = colour + self.casts_shadows = casts_shadows + +class Surface: + def __init__(self, diffuse, specular, spec_idx, reflect, transp, colour): + self.diffuse = diffuse + self.specular = specular + self.spec_idx = spec_idx + self.reflect = reflect + self.transp = transp + self.colour = colour + + @staticmethod + def dull(colour): + return Surface(0.7, 0.0, 1, 0.0, 0.0, colour * 0.6) + + @staticmethod + def shiny(colour): + return Surface(0.2, 0.9, 32, 0.8, 0.0, colour * 0.3) + + @staticmethod + def transparent(colour): + return Surface(0.2, 0.9, 32, 0.0, 0.8, colour * 0.3) + +class Sphere: + def __init__(self, surface, centre, radius): + self.surface = surface + self.centre = centre + self.radsq = radius ** 2 + + def intersect(self, ray): + v = self.centre - ray.p + b = v.dot(ray.d) + det = b ** 2 - v.dot(v) + self.radsq + if det > 0: + det **= 0.5 + t1 = b - det + if t1 > EPS: + return t1 + t2 = b + det + if t2 > EPS: + return t2 + return INF + + def surface_at(self, v): + return self.surface, (v - self.centre).normalise() + +class Plane: + def __init__(self, surface, centre, normal): + self.surface = surface + self.normal = normal.normalise() + self.cdotn = centre.dot(normal) + + def intersect(self, ray): + ddotn = ray.d.dot(self.normal) + if abs(ddotn) > EPS: + t = (self.cdotn - ray.p.dot(self.normal)) / ddotn + if t > 0: + return t + return INF + + def surface_at(self, p): + return self.surface, self.normal + +class Scene: + def __init__(self, ambient, light, objs): + self.ambient = ambient + self.light = light + self.objs = objs + +def trace_scene(canvas, view, scene, max_depth): + for v in range(canvas.height): + y = (-v + 0.5 * (canvas.height - 1)) * view.height / canvas.height + for u in range(canvas.width): + x = (u - 0.5 * (canvas.width - 1)) * view.width / canvas.width + ray = Ray(view.pos, view.calc_dir(x, y)) + c = trace_ray(scene, ray, max_depth) + canvas.put_pix(u, v, c) + +def trace_ray(scene, ray, depth): + # Find closest intersecting object + hit_t = INF + hit_obj = None + for obj in scene.objs: + t = obj.intersect(ray) + if t < hit_t: + hit_t = t + hit_obj = obj + + # Check if any objects hit + if hit_obj is None: + return RGB(0, 0, 0) + + # Compute location of ray intersection + point = ray.p + ray.d * hit_t + surf, surf_norm = hit_obj.surface_at(point) + if ray.d.dot(surf_norm) > 0: + surf_norm = -surf_norm + + # Compute reflected ray + reflected = ray.d - surf_norm * (surf_norm.dot(ray.d) * 2) + + # Ambient light + col = surf.colour * scene.ambient + + # Diffuse, specular and shadow from light source + light_vec = scene.light.pos - point + light_dist = light_vec.length() + light_vec = light_vec.normalise() + ndotl = surf_norm.dot(light_vec) + ldotv = light_vec.dot(reflected) + if ndotl > 0 or ldotv > 0: + light_ray = Ray(point + light_vec * EPS, light_vec) + light_col = trace_to_light(scene, light_ray, light_dist) + if ndotl > 0: + col += light_col * surf.diffuse * ndotl + if ldotv > 0: + col += light_col * surf.specular * ldotv ** surf.spec_idx + + # Reflections + if depth > 0 and surf.reflect > 0: + col += trace_ray(scene, Ray(point + reflected * EPS, reflected), depth - 1) * surf.reflect + + # Transparency + if depth > 0 and surf.transp > 0: + col += trace_ray(scene, Ray(point + ray.d * EPS, ray.d), depth - 1) * surf.transp + + return col + +def trace_to_light(scene, ray, light_dist): + col = scene.light.colour + for obj in scene.objs: + t = obj.intersect(ray) + if t < light_dist: + col *= obj.surface.transp + return col + +class Canvas: + def __init__(self, width, height): + self.width = width + self.height = height + self.data = bytearray(3 * width * height) + + def put_pix(self, x, y, c): + off = 3 * (y * self.width + x) + self.data[off] = min(255, max(0, int(255 * c.x))) + self.data[off + 1] = min(255, max(0, int(255 * c.y))) + self.data[off + 2] = min(255, max(0, int(255 * c.z))) + + def write_ppm(self, filename): + with open(filename, 'wb') as f: + f.write(bytes('P6 %d %d 255\n' % (self.width, self.height), 'ascii')) + f.write(self.data) + +def main(w, h, d): + canvas = Canvas(w, h) + view = View(32, 32, 64, Vec(0, 0, 50), Vec(1, 0, 0), Vec(0, 1, 0), Vec(0, 0, -1)) + scene = Scene( + 0.5, + Light(Vec(0, 8, 0), RGB(1, 1, 1), True), + [ + Plane(Surface.dull(RGB(1, 0, 0)), Vec(-10, 0, 0), Vec(1, 0, 0)), + Plane(Surface.dull(RGB(0, 1, 0)), Vec(10, 0, 0), Vec(-1, 0, 0)), + Plane(Surface.dull(RGB(1, 1, 1)), Vec(0, 0, -10), Vec(0, 0, 1)), + Plane(Surface.dull(RGB(1, 1, 1)), Vec(0, -10, 0), Vec(0, 1, 0)), + Plane(Surface.dull(RGB(1, 1, 1)), Vec(0, 10, 0), Vec(0, -1, 0)), + Sphere(Surface.shiny(RGB(1, 1, 1)), Vec(-5, -4, 3), 4), + Sphere(Surface.dull(RGB(0, 0, 1)), Vec(4, -5, 0), 4), + Sphere(Surface.transparent(RGB(0.2, 0.2, 0.2)), Vec(6, -1, 8), 4), + ] + ) + trace_scene(canvas, view, scene, d) + return canvas + +# For testing +#main(256, 256, 4).write_ppm('rt.ppm') + +########################################################################### +# Benchmark interface + +bm_params = { + (100, 100): (5, 5, 2), + (1000, 100): (18, 18, 3), + (5000, 100): (40, 40, 3), +} + +def bm_setup(params): + return lambda: main(*params), lambda: (params[0] * params[1] * params[2], None) From 73fccf59672c7087d6290bb036fe5c72899086b6 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 26 Jun 2019 14:27:32 +1000 Subject: [PATCH 5/6] tests/perf_bench: Add some viper performance benchmarks. To test raw viper function call overhead: function entry, exit and conversion of arguments to/from objects. --- tests/perf_bench/viper_call0.py | 19 +++++++++++++++++++ tests/perf_bench/viper_call1a.py | 19 +++++++++++++++++++ tests/perf_bench/viper_call1b.py | 19 +++++++++++++++++++ tests/perf_bench/viper_call1c.py | 19 +++++++++++++++++++ tests/perf_bench/viper_call2a.py | 19 +++++++++++++++++++ tests/perf_bench/viper_call2b.py | 19 +++++++++++++++++++ 6 files changed, 114 insertions(+) create mode 100644 tests/perf_bench/viper_call0.py create mode 100644 tests/perf_bench/viper_call1a.py create mode 100644 tests/perf_bench/viper_call1b.py create mode 100644 tests/perf_bench/viper_call1c.py create mode 100644 tests/perf_bench/viper_call2a.py create mode 100644 tests/perf_bench/viper_call2b.py diff --git a/tests/perf_bench/viper_call0.py b/tests/perf_bench/viper_call0.py new file mode 100644 index 0000000000000..0f476b127bce5 --- /dev/null +++ b/tests/perf_bench/viper_call0.py @@ -0,0 +1,19 @@ +@micropython.viper +def f0(): + pass + +@micropython.native +def call(r): + f = f0 + for _ in r: + f() + +bm_params = { + (50, 10): (15000,), + (100, 10): (30000,), + (1000, 10): (300000,), + (5000, 10): (1500000,), +} + +def bm_setup(params): + return lambda: call(range(params[0])), lambda: (params[0] // 1000, None) diff --git a/tests/perf_bench/viper_call1a.py b/tests/perf_bench/viper_call1a.py new file mode 100644 index 0000000000000..2bb4a28fd30b9 --- /dev/null +++ b/tests/perf_bench/viper_call1a.py @@ -0,0 +1,19 @@ +@micropython.viper +def f1a(x): + return x + +@micropython.native +def call(r): + f = f1a + for _ in r: + f(1) + +bm_params = { + (50, 10): (15000,), + (100, 10): (30000,), + (1000, 10): (300000,), + (5000, 10): (1500000,), +} + +def bm_setup(params): + return lambda: call(range(params[0])), lambda: (params[0] // 1000, None) diff --git a/tests/perf_bench/viper_call1b.py b/tests/perf_bench/viper_call1b.py new file mode 100644 index 0000000000000..cda64007fedde --- /dev/null +++ b/tests/perf_bench/viper_call1b.py @@ -0,0 +1,19 @@ +@micropython.viper +def f1b(x) -> int: + return int(x) + +@micropython.native +def call(r): + f = f1b + for _ in r: + f(1) + +bm_params = { + (50, 10): (15000,), + (100, 10): (30000,), + (1000, 10): (300000,), + (5000, 10): (1500000,), +} + +def bm_setup(params): + return lambda: call(range(params[0])), lambda: (params[0] // 1000, None) diff --git a/tests/perf_bench/viper_call1c.py b/tests/perf_bench/viper_call1c.py new file mode 100644 index 0000000000000..c653eb8d3ee18 --- /dev/null +++ b/tests/perf_bench/viper_call1c.py @@ -0,0 +1,19 @@ +@micropython.viper +def f1c(x:int) -> int: + return x + +@micropython.native +def call(r): + f = f1c + for _ in r: + f(1) + +bm_params = { + (50, 10): (15000,), + (100, 10): (30000,), + (1000, 10): (300000,), + (5000, 10): (1500000,), +} + +def bm_setup(params): + return lambda: call(range(params[0])), lambda: (params[0] // 1000, None) diff --git a/tests/perf_bench/viper_call2a.py b/tests/perf_bench/viper_call2a.py new file mode 100644 index 0000000000000..6204f985f9f20 --- /dev/null +++ b/tests/perf_bench/viper_call2a.py @@ -0,0 +1,19 @@ +@micropython.viper +def f2a(x, y): + return x + +@micropython.native +def call(r): + f = f2a + for _ in r: + f(1, 2) + +bm_params = { + (50, 10): (15000,), + (100, 10): (30000,), + (1000, 10): (300000,), + (5000, 10): (1500000,), +} + +def bm_setup(params): + return lambda: call(range(params[0])), lambda: (params[0] // 1000, None) diff --git a/tests/perf_bench/viper_call2b.py b/tests/perf_bench/viper_call2b.py new file mode 100644 index 0000000000000..087cf8ab0cd30 --- /dev/null +++ b/tests/perf_bench/viper_call2b.py @@ -0,0 +1,19 @@ +@micropython.viper +def f2b(x:int, y:int) -> int: + return x + y + +@micropython.native +def call(r): + f = f2b + for _ in r: + f(1, 2) + +bm_params = { + (50, 10): (15000,), + (100, 10): (30000,), + (1000, 10): (300000,), + (5000, 10): (1500000,), +} + +def bm_setup(params): + return lambda: call(range(params[0])), lambda: (params[0] // 1000, None) From 9cebead27600f86a73f3326b455dcfcb065afe5f Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 26 Jun 2019 14:29:06 +1000 Subject: [PATCH 6/6] travis: Enable performance benchmark tests on standard unix build. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b71cabcc808bd..9fbdb1698eb88 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,6 +89,7 @@ jobs: - make ${MAKEOPTS} -C ports/unix deplibs - make ${MAKEOPTS} -C ports/unix - make ${MAKEOPTS} -C ports/unix test + - (cd tests && MICROPY_CPYTHON3=python3 MICROPY_MICROPYTHON=../ports/unix/micropython ./run-perfbench.py 1000 1000) # unix nanbox - stage: test