From 01f69689ed9f87c460ba0e9afc0b670f148f1f95 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Tue, 22 May 2018 16:20:39 -0700 Subject: [PATCH] Initial Support for Airspeed Velocity (#2113) * asv init * move asv * init benchmarks * small changes * running basics * ok done * readme * ok * update * linting * linting * linting * linting * Update and rename README-benchmarks.md to README-benchmarks.rst * Update benchmarks.py * linting --- python/README-benchmarks.rst | 7 ++ python/asv.conf.json | 141 ++++++++++++++++++++++++++++++++ python/benchmarks/__init__.py | 0 python/benchmarks/benchmarks.py | 26 ++++++ 4 files changed, 174 insertions(+) create mode 100644 python/README-benchmarks.rst create mode 100644 python/asv.conf.json create mode 100644 python/benchmarks/__init__.py create mode 100644 python/benchmarks/benchmarks.py diff --git a/python/README-benchmarks.rst b/python/README-benchmarks.rst new file mode 100644 index 000000000..c992151b5 --- /dev/null +++ b/python/README-benchmarks.rst @@ -0,0 +1,7 @@ +Running the benchmarks +====================== + +You can run the benchmark suite by doing the following: + +1. Install https://github.com/airspeed-velocity/asv: ``pip install asv`` +2. Run ``asv dev`` in this directory. diff --git a/python/asv.conf.json b/python/asv.conf.json new file mode 100644 index 000000000..0378ba0da --- /dev/null +++ b/python/asv.conf.json @@ -0,0 +1,141 @@ +{ + // The version of the config file format. Do not change, unless + // you know what you are doing. + "version": 1, + + // The name of the project being benchmarked + "project": "ray", + + // The project's homepage + "project_url": "http://ray.readthedocs.io/en/latest/index.html", + + // The URL or local path of the source code repository for the + // project being benchmarked + "repo": ".", + + // List of branches to benchmark. If not provided, defaults to "master" + // (for git) or "default" (for mercurial). + "branches": ["master"], // for git + // "branches": ["default"], // for mercurial + + // The DVCS being used. If not set, it will be automatically + // determined from "repo" by looking at the protocol in the URL + // (if remote), or by looking for special directories, such as + // ".git" (if local). + "dvcs": "git", + + // The tool to use to create environments. May be "conda", + // "virtualenv" or other value depending on the plugins in use. + // If missing or the empty string, the tool will be automatically + // determined by looking for tools on the PATH environment + // variable. + "environment_type": "conda", + + // timeout in seconds for installing any dependencies in environment + // defaults to 10 min + //"install_timeout": 600, + + // the base URL to show a commit for the project. + "show_commit_url": "http://github.com/ray-project/ray/commit/", + + // The Pythons you'd like to test against. If not provided, defaults + // to the current version of Python used to run `asv`. + "pythons": ["3.6"], + + // The matrix of dependencies to test. Each key is the name of a + // package (in PyPI) and the values are version numbers. An empty + // list or empty string indicates to just test against the default + // (latest) version. null indicates that the package is to not be + // installed. If the package to be tested is only available from + // PyPi, and the 'environment_type' is conda, then you can preface + // the package name by 'pip+', and the package will be installed via + // pip (with all the conda available packages installed first, + // followed by the pip installed packages). + // + // "matrix": { + // "numpy": ["1.6", "1.7"], + // "six": ["", null], // test with and without six installed + // "pip+emcee": [""], // emcee is only available for install with pip. + // }, + + // Combinations of libraries/python versions can be excluded/included + // from the set to test. Each entry is a dictionary containing additional + // key-value pairs to include/exclude. + // + // An exclude entry excludes entries where all values match. The + // values are regexps that should match the whole string. + // + // An include entry adds an environment. Only the packages listed + // are installed. The 'python' key is required. The exclude rules + // do not apply to includes. + // + // In addition to package names, the following keys are available: + // + // - python + // Python version, as in the *pythons* variable above. + // - environment_type + // Environment type, as above. + // - sys_platform + // Platform, as in sys.platform. Possible values for the common + // cases: 'linux2', 'win32', 'cygwin', 'darwin'. + // + // "exclude": [ + // {"python": "3.2", "sys_platform": "win32"}, // skip py3.2 on windows + // {"environment_type": "conda", "six": null}, // don't run without six on conda + // ], + // + // "include": [ + // // additional env for python2.7 + // {"python": "2.7", "numpy": "1.8"}, + // // additional env if run on windows+conda + // {"platform": "win32", "environment_type": "conda", "python": "2.7", "libpython": ""}, + // ], + + // The directory (relative to the current directory) that benchmarks are + // stored in. If not provided, defaults to "benchmarks" + // "benchmark_dir": "benchmarks", + + // The directory (relative to the current directory) to cache the Python + // environments in. If not provided, defaults to "env" + // "env_dir": "env", + + // The directory (relative to the current directory) that raw benchmark + // results are stored in. If not provided, defaults to "results". + // "results_dir": "results", + + // The directory (relative to the current directory) that the html tree + // should be written to. If not provided, defaults to "html". + // "html_dir": "html", + + // The number of characters to retain in the commit hashes. + // "hash_length": 8, + + // `asv` will cache wheels of the recent builds in each + // environment, making them faster to install next time. This is + // number of builds to keep, per environment. + // "wheel_cache_size": 0 + + // The commits after which the regression search in `asv publish` + // should start looking for regressions. Dictionary whose keys are + // regexps matching to benchmark names, and values corresponding to + // the commit (exclusive) after which to start looking for + // regressions. The default is to start from the first commit + // with results. If the commit is `null`, regression detection is + // skipped for the matching benchmark. + // + // "regressions_first_commits": { + // "some_benchmark": "352cdf", // Consider regressions only after this commit + // "another_benchmark": null, // Skip regression detection altogether + // } + + // The thresholds for relative change in results, after which `asv + // publish` starts reporting regressions. Dictionary of the same + // form as in ``regressions_first_commits``, with values + // indicating the thresholds. If multiple entries match, the + // maximum is taken. If no entry matches, the default is 5%. + // + // "regressions_thresholds": { + // "some_benchmark": 0.01, // Threshold of 1% + // "another_benchmark": 0.5, // Threshold of 50% + // } +} diff --git a/python/benchmarks/__init__.py b/python/benchmarks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/benchmarks/benchmarks.py b/python/benchmarks/benchmarks.py new file mode 100644 index 000000000..e374d19a2 --- /dev/null +++ b/python/benchmarks/benchmarks.py @@ -0,0 +1,26 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +class TimeSuite(object): + """An example benchmark.""" + + def setup(self): + self.d = {} + for x in range(500): + self.d[x] = None + + def time_keys(self): + for key in self.d.keys(): + pass + + def time_range(self): + d = self.d + for key in range(500): + d[key] + + +class MemSuite(object): + def mem_list(self): + return [0] * 256