Skip to content

Commit

Permalink
Benchmarks test for correctness
Browse files Browse the repository at this point in the history
  • Loading branch information
@ijl
ijl committed Apr 19, 2019
1 parent b9208dc commit 228a159
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 14 deletions.
30 changes: 25 additions & 5 deletions bench/benchmark_dumps.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

import unittest

from json import dumps as _json_dumps
from orjson import dumps as _orjson_dumps
from rapidjson import dumps as _rapidjson_dumps
from simplejson import dumps as _simplejson_dumps
from ujson import dumps as _ujson_dumps
from json import dumps as _json_dumps, loads as json_loads
from orjson import dumps as _orjson_dumps, loads as orjson_loads
from rapidjson import dumps as _rapidjson_dumps, loads as rapidjson_loads
from simplejson import dumps as _simplejson_dumps, loads as simplejson_loads
from ujson import dumps as _ujson_dumps, loads as ujson_loads

from .util import read_fixture_obj, read_fixture_str

Expand All @@ -29,118 +29,138 @@ def test_dumps_canada_orjson(benchmark):
benchmark.group = 'canada.json serialization'
benchmark.extra_info['lib'] = 'orjson'
data = read_fixture_obj("canada.json.xz")
benchmark.extra_info['correct'] = json_loads(orjson_dumps(data)) == data
benchmark(orjson_dumps, data)

def test_dumps_canada_ujson(benchmark):
benchmark.group = 'canada.json serialization'
benchmark.extra_info['lib'] = 'ujson'
data = read_fixture_obj("canada.json.xz")
benchmark.extra_info['correct'] = json_loads(ujson_dumps(data)) == data
benchmark(ujson_dumps, data)

def test_dumps_canada_json(benchmark):
benchmark.group = 'canada.json serialization'
benchmark.extra_info['lib'] = 'json'
data = read_fixture_obj("canada.json.xz")
benchmark.extra_info['correct'] = json_loads(json_dumps(data)) == data
benchmark(json_dumps, data)

def test_dumps_canada_rapidjson(benchmark):
benchmark.group = 'canada.json serialization'
benchmark.extra_info['lib'] = 'rapidjson'
data = read_fixture_obj("canada.json.xz")
benchmark.extra_info['correct'] = json_loads(rapidjson_dumps(data)) == data
benchmark(rapidjson_dumps, data)

def test_dumps_canada_simplejson(benchmark):
benchmark.group = 'canada.json serialization'
benchmark.extra_info['lib'] = 'simplejson'
data = read_fixture_obj("canada.json.xz")
benchmark.extra_info['correct'] = json_loads(simplejson_dumps(data)) == data
benchmark(simplejson_dumps, data)

def test_dumps_citm_catalog_orjson(benchmark):
benchmark.group = 'citm_catalog.json serialization'
benchmark.extra_info['lib'] = 'orjson'
data = read_fixture_obj("citm_catalog.json.xz")
benchmark.extra_info['correct'] = json_loads(orjson_dumps(data)) == data
benchmark(orjson_dumps, data)

def test_dumps_citm_catalog_ujson(benchmark):
benchmark.group = 'citm_catalog.json serialization'
benchmark.extra_info['lib'] = 'ujson'
data = read_fixture_obj("citm_catalog.json.xz")
benchmark.extra_info['correct'] = json_loads(ujson_dumps(data)) == data
benchmark(ujson_dumps, data)

def test_dumps_citm_catalog_json(benchmark):
benchmark.group = 'citm_catalog.json serialization'
benchmark.extra_info['lib'] = 'json'
data = read_fixture_obj("citm_catalog.json.xz")
benchmark.extra_info['correct'] = json_loads(json_dumps(data)) == data
benchmark(json_dumps, data)

def test_dumps_citm_catalog_rapidjson(benchmark):
benchmark.group = 'citm_catalog.json serialization'
benchmark.extra_info['lib'] = 'rapidjson'
data = read_fixture_obj("citm_catalog.json.xz")
benchmark.extra_info['correct'] = json_loads(rapidjson_dumps(data)) == data
benchmark(rapidjson_dumps, data)

def test_dumps_citm_catalog_simplejson(benchmark):
benchmark.group = 'citm_catalog.json serialization'
benchmark.extra_info['lib'] = 'simplejson'
data = read_fixture_obj("citm_catalog.json.xz")
benchmark.extra_info['correct'] = json_loads(simplejson_dumps(data)) == data
benchmark(simplejson_dumps, data)

def test_dumps_github_orjson(benchmark):
benchmark.group = 'github.json serialization'
benchmark.extra_info['lib'] = 'orjson'
data = read_fixture_obj("github.json.xz")
benchmark.extra_info['correct'] = json_loads(orjson_dumps(data)) == data
benchmark(orjson_dumps, data)

def test_dumps_github_ujson(benchmark):
benchmark.group = 'github.json serialization'
benchmark.extra_info['lib'] = 'ujson'
data = read_fixture_obj("github.json.xz")
benchmark.extra_info['correct'] = json_loads(ujson_dumps(data)) == data
benchmark(ujson_dumps, data)

def test_dumps_github_json(benchmark):
benchmark.group = 'github.json serialization'
benchmark.extra_info['lib'] = 'json'
data = read_fixture_obj("github.json.xz")
benchmark.extra_info['correct'] = json_loads(json_dumps(data)) == data
benchmark(json_dumps, data)

def test_dumps_github_rapidjson(benchmark):
benchmark.group = 'github.json serialization'
benchmark.extra_info['lib'] = 'rapidjson'
data = read_fixture_obj("github.json.xz")
benchmark.extra_info['correct'] = json_loads(rapidjson_dumps(data)) == data
benchmark(rapidjson_dumps, data)

def test_dumps_github_simplejson(benchmark):
benchmark.group = 'github.json serialization'
benchmark.extra_info['lib'] = 'simplejson'
data = read_fixture_obj("github.json.xz")
benchmark.extra_info['correct'] = json_loads(simplejson_dumps(data)) == data
benchmark(simplejson_dumps, data)

def test_dumps_twitter_orjson(benchmark):
benchmark.group = 'twitter.json serialization'
benchmark.extra_info['lib'] = 'orjson'
data = read_fixture_obj("twitter.json.xz")
benchmark.extra_info['correct'] = json_loads(orjson_dumps(data)) == data
benchmark(orjson_dumps, data)

def test_dumps_twitter_ujson(benchmark):
benchmark.group = 'twitter.json serialization'
benchmark.extra_info['lib'] = 'ujson'
data = read_fixture_obj("twitter.json.xz")
benchmark.extra_info['correct'] = json_loads(ujson_dumps(data)) == data
benchmark(ujson_dumps, data)

def test_dumps_twitter_json(benchmark):
benchmark.group = 'twitter.json serialization'
benchmark.extra_info['lib'] = 'json'
data = read_fixture_obj("twitter.json.xz")
benchmark.extra_info['correct'] = json_loads(json_dumps(data)) == data
benchmark(json_dumps, data)

def test_dumps_twitter_rapidjson(benchmark):
benchmark.group = 'twitter.json serialization'
benchmark.extra_info['lib'] = 'rapidjson'
data = read_fixture_obj("twitter.json.xz")
benchmark.extra_info['correct'] = json_loads(rapidjson_dumps(data)) == data
benchmark(rapidjson_dumps, data)

def test_dumps_twitter_simplejson(benchmark):
benchmark.group = 'twitter.json serialization'
benchmark.extra_info['lib'] = 'simplejson'
data = read_fixture_obj("twitter.json.xz")
benchmark.extra_info['correct'] = json_loads(simplejson_dumps(data)) == data
benchmark(simplejson_dumps, data)
30 changes: 25 additions & 5 deletions bench/benchmark_loads.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

import unittest

from json import loads as json_loads
from orjson import loads as orjson_loads
from rapidjson import loads as rapidjson_loads
from simplejson import loads as simplejson_loads
from ujson import loads as ujson_loads
from json import loads as json_loads, dumps as json_dumps
from orjson import loads as orjson_loads, dumps as orjson_dumps
from rapidjson import loads as rapidjson_loads, dumps as rapidjson_dumps
from simplejson import loads as simplejson_loads, dumps as simplejson_dumps
from ujson import loads as ujson_loads, dumps as ujson_dumps

from .util import read_fixture_obj, read_fixture_str

Expand All @@ -15,118 +15,138 @@ def test_loads_canada_orjson(benchmark):
benchmark.group = 'canada.json deserialization'
benchmark.extra_info['lib'] = 'orjson'
data = read_fixture_str("canada.json.xz")
benchmark.extra_info['correct'] = json_loads(orjson_dumps(orjson_loads(data))) == json_loads(data)
benchmark(orjson_loads, data)

def test_loads_canada_ujson(benchmark):
benchmark.group = 'canada.json deserialization'
benchmark.extra_info['lib'] = 'ujson'
data = read_fixture_str("canada.json.xz")
benchmark.extra_info['correct'] = json_loads(ujson_dumps(ujson_loads(data))) == json_loads(data)
benchmark(ujson_loads, data)

def test_loads_canada_json(benchmark):
benchmark.group = 'canada.json deserialization'
benchmark.extra_info['lib'] = 'json'
data = read_fixture_str("canada.json.xz")
benchmark.extra_info['correct'] = json_loads(json_dumps(json_loads(data))) == json_loads(data)
benchmark(json_loads, data)

def test_loads_canada_rapidjson(benchmark):
benchmark.group = 'canada.json deserialization'
benchmark.extra_info['lib'] = 'rapidjson'
data = read_fixture_str("canada.json.xz")
benchmark.extra_info['correct'] = json_loads(rapidjson_dumps(rapidjson_loads(data))) == json_loads(data)
benchmark(rapidjson_loads, data)

def test_loads_canada_simplejson(benchmark):
benchmark.group = 'canada.json deserialization'
benchmark.extra_info['lib'] = 'simplejson'
data = read_fixture_str("canada.json.xz")
benchmark.extra_info['correct'] = json_loads(simplejson_dumps(simplejson_loads(data))) == json_loads(data)
benchmark(simplejson_loads, data)

def test_loads_citm_catalog_orjson(benchmark):
benchmark.group = 'citm_catalog.json deserialization'
benchmark.extra_info['lib'] = 'orjson'
data = read_fixture_str("citm_catalog.json.xz")
benchmark.extra_info['correct'] = json_loads(orjson_dumps(orjson_loads(data))) == json_loads(data)
benchmark(orjson_loads, data)

def test_loads_citm_catalog_ujson(benchmark):
benchmark.group = 'citm_catalog.json deserialization'
benchmark.extra_info['lib'] = 'ujson'
data = read_fixture_str("citm_catalog.json.xz")
benchmark.extra_info['correct'] = json_loads(ujson_dumps(ujson_loads(data))) == json_loads(data)
benchmark(ujson_loads, data)

def test_loads_citm_catalog_json(benchmark):
benchmark.group = 'citm_catalog.json deserialization'
benchmark.extra_info['lib'] = 'json'
data = read_fixture_str("citm_catalog.json.xz")
benchmark.extra_info['correct'] = json_loads(json_dumps(json_loads(data))) == json_loads(data)
benchmark(json_loads, data)

def test_loads_citm_catalog_rapidjson(benchmark):
benchmark.group = 'citm_catalog.json deserialization'
benchmark.extra_info['lib'] = 'rapidjson'
data = read_fixture_str("citm_catalog.json.xz")
benchmark.extra_info['correct'] = json_loads(rapidjson_dumps(rapidjson_loads(data))) == json_loads(data)
benchmark(rapidjson_loads, data)

def test_loads_citm_catalog_simplejson(benchmark):
benchmark.group = 'citm_catalog.json deserialization'
benchmark.extra_info['lib'] = 'simplejson'
data = read_fixture_str("citm_catalog.json.xz")
benchmark.extra_info['correct'] = json_loads(simplejson_dumps(simplejson_loads(data))) == json_loads(data)
benchmark(simplejson_loads, data)

def test_loads_github_orjson(benchmark):
benchmark.group = 'github.json deserialization'
benchmark.extra_info['lib'] = 'orjson'
data = read_fixture_str("github.json.xz")
benchmark.extra_info['correct'] = json_loads(orjson_dumps(orjson_loads(data))) == json_loads(data)
benchmark(orjson_loads, data)

def test_loads_github_ujson(benchmark):
benchmark.group = 'github.json deserialization'
benchmark.extra_info['lib'] = 'ujson'
data = read_fixture_str("github.json.xz")
benchmark.extra_info['correct'] = json_loads(ujson_dumps(ujson_loads(data))) == json_loads(data)
benchmark(ujson_loads, data)

def test_loads_github_json(benchmark):
benchmark.group = 'github.json deserialization'
benchmark.extra_info['lib'] = 'json'
data = read_fixture_str("github.json.xz")
benchmark.extra_info['correct'] = json_loads(json_dumps(json_loads(data))) == json_loads(data)
benchmark(json_loads, data)

def test_loads_github_rapidjson(benchmark):
benchmark.group = 'github.json deserialization'
benchmark.extra_info['lib'] = 'rapidjson'
data = read_fixture_str("github.json.xz")
benchmark.extra_info['correct'] = json_loads(rapidjson_dumps(rapidjson_loads(data))) == json_loads(data)
benchmark(rapidjson_loads, data)

def test_loads_github_simplejson(benchmark):
benchmark.group = 'github.json deserialization'
benchmark.extra_info['lib'] = 'simplejson'
data = read_fixture_str("github.json.xz")
benchmark.extra_info['correct'] = json_loads(simplejson_dumps(simplejson_loads(data))) == json_loads(data)
benchmark(simplejson_loads, data)

def test_loads_twitter_orjson(benchmark):
benchmark.group = 'twitter.json deserialization'
benchmark.extra_info['lib'] = 'orjson'
data = read_fixture_str("twitter.json.xz")
benchmark.extra_info['correct'] = json_loads(orjson_dumps(orjson_loads(data))) == json_loads(data)
benchmark(orjson_loads, data)

def test_loads_twitter_ujson(benchmark):
benchmark.group = 'twitter.json deserialization'
benchmark.extra_info['lib'] = 'ujson'
data = read_fixture_str("twitter.json.xz")
benchmark.extra_info['correct'] = json_loads(ujson_dumps(ujson_loads(data))) == json_loads(data)
benchmark(ujson_loads, data)

def test_loads_twitter_json(benchmark):
benchmark.group = 'twitter.json deserialization'
benchmark.extra_info['lib'] = 'json'
data = read_fixture_str("twitter.json.xz")
benchmark.extra_info['correct'] = json_loads(json_dumps(json_loads(data))) == json_loads(data)
benchmark(json_loads, data)

def test_loads_twitter_rapidjson(benchmark):
benchmark.group = 'twitter.json deserialization'
benchmark.extra_info['lib'] = 'rapidjson'
data = read_fixture_str("twitter.json.xz")
benchmark.extra_info['correct'] = json_loads(rapidjson_dumps(rapidjson_loads(data))) == json_loads(data)
benchmark(rapidjson_loads, data)

def test_loads_twitter_simplejson(benchmark):
benchmark.group = 'twitter.json deserialization'
benchmark.extra_info['lib'] = 'simplejson'
data = read_fixture_str("twitter.json.xz")
benchmark.extra_info['correct'] = json_loads(simplejson_dumps(simplejson_loads(data))) == json_loads(data)
benchmark(simplejson_loads, data)
15 changes: 11 additions & 4 deletions graph
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,15 @@ def aggregate():
],
'median': each['stats']['median'] * 1000,
'ops': each['stats']['ops'],
'correct': each['extra_info']['correct']
}
return res

def box(obj):
for group, val in sorted(obj.items()):
data = []
for lib in LIBRARIES:
data.append(val[lib]['data'])
data.append(val[lib]['data'] if val[lib]['correct'] else -1)
fig = plt.figure(1, figsize=(9, 6))
ax = fig.add_subplot(111)
bp = ax.boxplot(data, vert=False, labels=LIBRARIES)
Expand All @@ -50,13 +51,19 @@ def tab(obj):
buf.write('\n' + '#### ' + group + '\n\n')
table = []
for lib in LIBRARIES:
correct = val[lib]['correct']
table.append(
[lib, val[lib]['median'], '%.1f' % val[lib]['ops'], 0]
[
lib,
val[lib]['median'] if correct else None,
'%.1f' % val[lib]['ops'] if correct else None,
0,
]
)
baseline = table[0][1]
for each in table:
each[3] = '%.2f' % (each[1] / baseline)
each[1] = '%.2f' % each[1]
each[3] = '%.2f' % (each[1] / baseline) if isinstance(each[1], float) else None
each[1] = '%.2f' % each[1] if isinstance(each[1], float) else None
buf.write(tabulate(table, headers, tablefmt='grid') + '\n')


Expand Down

0 comments on commit 228a159

Please sign in to comment.