From 228a159435fc6c1aab86adeb9db41e1a7155da61 Mon Sep 17 00:00:00 2001 From: ijl Date: Fri, 19 Apr 2019 13:30:57 +0000 Subject: [PATCH] Benchmarks test for correctness --- bench/benchmark_dumps.py | 30 +++++++++++++++++++++++++----- bench/benchmark_loads.py | 30 +++++++++++++++++++++++++----- graph | 15 +++++++++++---- 3 files changed, 61 insertions(+), 14 deletions(-) diff --git a/bench/benchmark_dumps.py b/bench/benchmark_dumps.py index cd12f786..f37a6508 100644 --- a/bench/benchmark_dumps.py +++ b/bench/benchmark_dumps.py @@ -2,11 +2,11 @@ import unittest -from json import dumps as _json_dumps -from orjson import dumps as _orjson_dumps -from rapidjson import dumps as _rapidjson_dumps -from simplejson import dumps as _simplejson_dumps -from ujson import dumps as _ujson_dumps +from json import dumps as _json_dumps, loads as json_loads +from orjson import dumps as _orjson_dumps, loads as orjson_loads +from rapidjson import dumps as _rapidjson_dumps, loads as rapidjson_loads +from simplejson import dumps as _simplejson_dumps, loads as simplejson_loads +from ujson import dumps as _ujson_dumps, loads as ujson_loads from .util import read_fixture_obj, read_fixture_str @@ -29,118 +29,138 @@ def test_dumps_canada_orjson(benchmark): benchmark.group = 'canada.json serialization' benchmark.extra_info['lib'] = 'orjson' data = read_fixture_obj("canada.json.xz") + benchmark.extra_info['correct'] = json_loads(orjson_dumps(data)) == data benchmark(orjson_dumps, data) def test_dumps_canada_ujson(benchmark): benchmark.group = 'canada.json serialization' benchmark.extra_info['lib'] = 'ujson' data = read_fixture_obj("canada.json.xz") + benchmark.extra_info['correct'] = json_loads(ujson_dumps(data)) == data benchmark(ujson_dumps, data) def test_dumps_canada_json(benchmark): benchmark.group = 'canada.json serialization' benchmark.extra_info['lib'] = 'json' data = read_fixture_obj("canada.json.xz") + benchmark.extra_info['correct'] = json_loads(json_dumps(data)) == data benchmark(json_dumps, data) def test_dumps_canada_rapidjson(benchmark): benchmark.group = 'canada.json serialization' benchmark.extra_info['lib'] = 'rapidjson' data = read_fixture_obj("canada.json.xz") + benchmark.extra_info['correct'] = json_loads(rapidjson_dumps(data)) == data benchmark(rapidjson_dumps, data) def test_dumps_canada_simplejson(benchmark): benchmark.group = 'canada.json serialization' benchmark.extra_info['lib'] = 'simplejson' data = read_fixture_obj("canada.json.xz") + benchmark.extra_info['correct'] = json_loads(simplejson_dumps(data)) == data benchmark(simplejson_dumps, data) def test_dumps_citm_catalog_orjson(benchmark): benchmark.group = 'citm_catalog.json serialization' benchmark.extra_info['lib'] = 'orjson' data = read_fixture_obj("citm_catalog.json.xz") + benchmark.extra_info['correct'] = json_loads(orjson_dumps(data)) == data benchmark(orjson_dumps, data) def test_dumps_citm_catalog_ujson(benchmark): benchmark.group = 'citm_catalog.json serialization' benchmark.extra_info['lib'] = 'ujson' data = read_fixture_obj("citm_catalog.json.xz") + benchmark.extra_info['correct'] = json_loads(ujson_dumps(data)) == data benchmark(ujson_dumps, data) def test_dumps_citm_catalog_json(benchmark): benchmark.group = 'citm_catalog.json serialization' benchmark.extra_info['lib'] = 'json' data = read_fixture_obj("citm_catalog.json.xz") + benchmark.extra_info['correct'] = json_loads(json_dumps(data)) == data benchmark(json_dumps, data) def test_dumps_citm_catalog_rapidjson(benchmark): benchmark.group = 'citm_catalog.json serialization' benchmark.extra_info['lib'] = 'rapidjson' data = read_fixture_obj("citm_catalog.json.xz") + benchmark.extra_info['correct'] = json_loads(rapidjson_dumps(data)) == data benchmark(rapidjson_dumps, data) def test_dumps_citm_catalog_simplejson(benchmark): benchmark.group = 'citm_catalog.json serialization' benchmark.extra_info['lib'] = 'simplejson' data = read_fixture_obj("citm_catalog.json.xz") + benchmark.extra_info['correct'] = json_loads(simplejson_dumps(data)) == data benchmark(simplejson_dumps, data) def test_dumps_github_orjson(benchmark): benchmark.group = 'github.json serialization' benchmark.extra_info['lib'] = 'orjson' data = read_fixture_obj("github.json.xz") + benchmark.extra_info['correct'] = json_loads(orjson_dumps(data)) == data benchmark(orjson_dumps, data) def test_dumps_github_ujson(benchmark): benchmark.group = 'github.json serialization' benchmark.extra_info['lib'] = 'ujson' data = read_fixture_obj("github.json.xz") + benchmark.extra_info['correct'] = json_loads(ujson_dumps(data)) == data benchmark(ujson_dumps, data) def test_dumps_github_json(benchmark): benchmark.group = 'github.json serialization' benchmark.extra_info['lib'] = 'json' data = read_fixture_obj("github.json.xz") + benchmark.extra_info['correct'] = json_loads(json_dumps(data)) == data benchmark(json_dumps, data) def test_dumps_github_rapidjson(benchmark): benchmark.group = 'github.json serialization' benchmark.extra_info['lib'] = 'rapidjson' data = read_fixture_obj("github.json.xz") + benchmark.extra_info['correct'] = json_loads(rapidjson_dumps(data)) == data benchmark(rapidjson_dumps, data) def test_dumps_github_simplejson(benchmark): benchmark.group = 'github.json serialization' benchmark.extra_info['lib'] = 'simplejson' data = read_fixture_obj("github.json.xz") + benchmark.extra_info['correct'] = json_loads(simplejson_dumps(data)) == data benchmark(simplejson_dumps, data) def test_dumps_twitter_orjson(benchmark): benchmark.group = 'twitter.json serialization' benchmark.extra_info['lib'] = 'orjson' data = read_fixture_obj("twitter.json.xz") + benchmark.extra_info['correct'] = json_loads(orjson_dumps(data)) == data benchmark(orjson_dumps, data) def test_dumps_twitter_ujson(benchmark): benchmark.group = 'twitter.json serialization' benchmark.extra_info['lib'] = 'ujson' data = read_fixture_obj("twitter.json.xz") + benchmark.extra_info['correct'] = json_loads(ujson_dumps(data)) == data benchmark(ujson_dumps, data) def test_dumps_twitter_json(benchmark): benchmark.group = 'twitter.json serialization' benchmark.extra_info['lib'] = 'json' data = read_fixture_obj("twitter.json.xz") + benchmark.extra_info['correct'] = json_loads(json_dumps(data)) == data benchmark(json_dumps, data) def test_dumps_twitter_rapidjson(benchmark): benchmark.group = 'twitter.json serialization' benchmark.extra_info['lib'] = 'rapidjson' data = read_fixture_obj("twitter.json.xz") + benchmark.extra_info['correct'] = json_loads(rapidjson_dumps(data)) == data benchmark(rapidjson_dumps, data) def test_dumps_twitter_simplejson(benchmark): benchmark.group = 'twitter.json serialization' benchmark.extra_info['lib'] = 'simplejson' data = read_fixture_obj("twitter.json.xz") + benchmark.extra_info['correct'] = json_loads(simplejson_dumps(data)) == data benchmark(simplejson_dumps, data) diff --git a/bench/benchmark_loads.py b/bench/benchmark_loads.py index 158dee45..ba263b78 100644 --- a/bench/benchmark_loads.py +++ b/bench/benchmark_loads.py @@ -2,11 +2,11 @@ import unittest -from json import loads as json_loads -from orjson import loads as orjson_loads -from rapidjson import loads as rapidjson_loads -from simplejson import loads as simplejson_loads -from ujson import loads as ujson_loads +from json import loads as json_loads, dumps as json_dumps +from orjson import loads as orjson_loads, dumps as orjson_dumps +from rapidjson import loads as rapidjson_loads, dumps as rapidjson_dumps +from simplejson import loads as simplejson_loads, dumps as simplejson_dumps +from ujson import loads as ujson_loads, dumps as ujson_dumps from .util import read_fixture_obj, read_fixture_str @@ -15,118 +15,138 @@ def test_loads_canada_orjson(benchmark): benchmark.group = 'canada.json deserialization' benchmark.extra_info['lib'] = 'orjson' data = read_fixture_str("canada.json.xz") + benchmark.extra_info['correct'] = json_loads(orjson_dumps(orjson_loads(data))) == json_loads(data) benchmark(orjson_loads, data) def test_loads_canada_ujson(benchmark): benchmark.group = 'canada.json deserialization' benchmark.extra_info['lib'] = 'ujson' data = read_fixture_str("canada.json.xz") + benchmark.extra_info['correct'] = json_loads(ujson_dumps(ujson_loads(data))) == json_loads(data) benchmark(ujson_loads, data) def test_loads_canada_json(benchmark): benchmark.group = 'canada.json deserialization' benchmark.extra_info['lib'] = 'json' data = read_fixture_str("canada.json.xz") + benchmark.extra_info['correct'] = json_loads(json_dumps(json_loads(data))) == json_loads(data) benchmark(json_loads, data) def test_loads_canada_rapidjson(benchmark): benchmark.group = 'canada.json deserialization' benchmark.extra_info['lib'] = 'rapidjson' data = read_fixture_str("canada.json.xz") + benchmark.extra_info['correct'] = json_loads(rapidjson_dumps(rapidjson_loads(data))) == json_loads(data) benchmark(rapidjson_loads, data) def test_loads_canada_simplejson(benchmark): benchmark.group = 'canada.json deserialization' benchmark.extra_info['lib'] = 'simplejson' data = read_fixture_str("canada.json.xz") + benchmark.extra_info['correct'] = json_loads(simplejson_dumps(simplejson_loads(data))) == json_loads(data) benchmark(simplejson_loads, data) def test_loads_citm_catalog_orjson(benchmark): benchmark.group = 'citm_catalog.json deserialization' benchmark.extra_info['lib'] = 'orjson' data = read_fixture_str("citm_catalog.json.xz") + benchmark.extra_info['correct'] = json_loads(orjson_dumps(orjson_loads(data))) == json_loads(data) benchmark(orjson_loads, data) def test_loads_citm_catalog_ujson(benchmark): benchmark.group = 'citm_catalog.json deserialization' benchmark.extra_info['lib'] = 'ujson' data = read_fixture_str("citm_catalog.json.xz") + benchmark.extra_info['correct'] = json_loads(ujson_dumps(ujson_loads(data))) == json_loads(data) benchmark(ujson_loads, data) def test_loads_citm_catalog_json(benchmark): benchmark.group = 'citm_catalog.json deserialization' benchmark.extra_info['lib'] = 'json' data = read_fixture_str("citm_catalog.json.xz") + benchmark.extra_info['correct'] = json_loads(json_dumps(json_loads(data))) == json_loads(data) benchmark(json_loads, data) def test_loads_citm_catalog_rapidjson(benchmark): benchmark.group = 'citm_catalog.json deserialization' benchmark.extra_info['lib'] = 'rapidjson' data = read_fixture_str("citm_catalog.json.xz") + benchmark.extra_info['correct'] = json_loads(rapidjson_dumps(rapidjson_loads(data))) == json_loads(data) benchmark(rapidjson_loads, data) def test_loads_citm_catalog_simplejson(benchmark): benchmark.group = 'citm_catalog.json deserialization' benchmark.extra_info['lib'] = 'simplejson' data = read_fixture_str("citm_catalog.json.xz") + benchmark.extra_info['correct'] = json_loads(simplejson_dumps(simplejson_loads(data))) == json_loads(data) benchmark(simplejson_loads, data) def test_loads_github_orjson(benchmark): benchmark.group = 'github.json deserialization' benchmark.extra_info['lib'] = 'orjson' data = read_fixture_str("github.json.xz") + benchmark.extra_info['correct'] = json_loads(orjson_dumps(orjson_loads(data))) == json_loads(data) benchmark(orjson_loads, data) def test_loads_github_ujson(benchmark): benchmark.group = 'github.json deserialization' benchmark.extra_info['lib'] = 'ujson' data = read_fixture_str("github.json.xz") + benchmark.extra_info['correct'] = json_loads(ujson_dumps(ujson_loads(data))) == json_loads(data) benchmark(ujson_loads, data) def test_loads_github_json(benchmark): benchmark.group = 'github.json deserialization' benchmark.extra_info['lib'] = 'json' data = read_fixture_str("github.json.xz") + benchmark.extra_info['correct'] = json_loads(json_dumps(json_loads(data))) == json_loads(data) benchmark(json_loads, data) def test_loads_github_rapidjson(benchmark): benchmark.group = 'github.json deserialization' benchmark.extra_info['lib'] = 'rapidjson' data = read_fixture_str("github.json.xz") + benchmark.extra_info['correct'] = json_loads(rapidjson_dumps(rapidjson_loads(data))) == json_loads(data) benchmark(rapidjson_loads, data) def test_loads_github_simplejson(benchmark): benchmark.group = 'github.json deserialization' benchmark.extra_info['lib'] = 'simplejson' data = read_fixture_str("github.json.xz") + benchmark.extra_info['correct'] = json_loads(simplejson_dumps(simplejson_loads(data))) == json_loads(data) benchmark(simplejson_loads, data) def test_loads_twitter_orjson(benchmark): benchmark.group = 'twitter.json deserialization' benchmark.extra_info['lib'] = 'orjson' data = read_fixture_str("twitter.json.xz") + benchmark.extra_info['correct'] = json_loads(orjson_dumps(orjson_loads(data))) == json_loads(data) benchmark(orjson_loads, data) def test_loads_twitter_ujson(benchmark): benchmark.group = 'twitter.json deserialization' benchmark.extra_info['lib'] = 'ujson' data = read_fixture_str("twitter.json.xz") + benchmark.extra_info['correct'] = json_loads(ujson_dumps(ujson_loads(data))) == json_loads(data) benchmark(ujson_loads, data) def test_loads_twitter_json(benchmark): benchmark.group = 'twitter.json deserialization' benchmark.extra_info['lib'] = 'json' data = read_fixture_str("twitter.json.xz") + benchmark.extra_info['correct'] = json_loads(json_dumps(json_loads(data))) == json_loads(data) benchmark(json_loads, data) def test_loads_twitter_rapidjson(benchmark): benchmark.group = 'twitter.json deserialization' benchmark.extra_info['lib'] = 'rapidjson' data = read_fixture_str("twitter.json.xz") + benchmark.extra_info['correct'] = json_loads(rapidjson_dumps(rapidjson_loads(data))) == json_loads(data) benchmark(rapidjson_loads, data) def test_loads_twitter_simplejson(benchmark): benchmark.group = 'twitter.json deserialization' benchmark.extra_info['lib'] = 'simplejson' data = read_fixture_str("twitter.json.xz") + benchmark.extra_info['correct'] = json_loads(simplejson_dumps(simplejson_loads(data))) == json_loads(data) benchmark(simplejson_loads, data) diff --git a/graph b/graph index 53738eb8..7a86a5da 100755 --- a/graph +++ b/graph @@ -26,6 +26,7 @@ def aggregate(): ], 'median': each['stats']['median'] * 1000, 'ops': each['stats']['ops'], + 'correct': each['extra_info']['correct'] } return res @@ -33,7 +34,7 @@ def box(obj): for group, val in sorted(obj.items()): data = [] for lib in LIBRARIES: - data.append(val[lib]['data']) + data.append(val[lib]['data'] if val[lib]['correct'] else -1) fig = plt.figure(1, figsize=(9, 6)) ax = fig.add_subplot(111) bp = ax.boxplot(data, vert=False, labels=LIBRARIES) @@ -50,13 +51,19 @@ def tab(obj): buf.write('\n' + '#### ' + group + '\n\n') table = [] for lib in LIBRARIES: + correct = val[lib]['correct'] table.append( - [lib, val[lib]['median'], '%.1f' % val[lib]['ops'], 0] + [ + lib, + val[lib]['median'] if correct else None, + '%.1f' % val[lib]['ops'] if correct else None, + 0, + ] ) baseline = table[0][1] for each in table: - each[3] = '%.2f' % (each[1] / baseline) - each[1] = '%.2f' % each[1] + each[3] = '%.2f' % (each[1] / baseline) if isinstance(each[1], float) else None + each[1] = '%.2f' % each[1] if isinstance(each[1], float) else None buf.write(tabulate(table, headers, tablefmt='grid') + '\n')