diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 00000000..dd033ed7 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,41 @@ +name: Benchmark + +on: + push: + branches: + - dev + - master + +jobs: + benchmark: + name: Performance check + runs-on: self-hosted + steps: + - uses: actions/checkout@v2 + # - uses: actions/setup-python@v1 # (not needed for self-hosted runner) + # with: + # python-version: '3.8' + # run: pip install pytest pytest-benchmark (already in docker image) + - name: Run benchmark + run: | + pip install -e . + pytest tests --benchmark-only --benchmark-timer='time.process_time' --benchmark-json output.json + - name: Download previous benchmark data + uses: actions/cache@v1 + with: + path: ./cache + key: ${{ runner.os }}-benchmark + - name: Store benchmark result + uses: rhysd/github-action-benchmark@v1 + with: + # What benchmark tool the output.txt came from + tool: 'pytest' + output-file-path: output.json + # Personal access token to deploy GitHub Pages branch + github-token: ${{ secrets.BENCHMARKS_TOKEN }} + auto-push: true + # Workflow will fail when an alert happens + alert-threshold: '150%' + comment-on-alert: true + alert-comment-cc-users: '@yoavnash,@pablo-de-andres,@kysrpex' + fail-on-alert: true \ No newline at end of file diff --git a/tests/benchmark.py b/tests/benchmark.py new file mode 100644 index 00000000..32d00b0a --- /dev/null +++ b/tests/benchmark.py @@ -0,0 +1,146 @@ +"""Contains an abstract class that serves as a base for defining benchmarks.""" +import time +from typing import Union +from abc import ABC, abstractmethod + + +class Benchmark(ABC): + """Abstract class that serves as a base for defining benchmarks.""" + + def __init__(self, size: int = 500, *args, **kwargs): + """Set-up the internal attributes of the benchmark. + + Args: + size (int): the number of iterations to be performed by the + benchmark for it to be considered as finished. + """ + super().__init__(*args, **kwargs) + self._size = size + self._iter_times = [None] * size + self._finished = False + + @property + def started(self) -> bool: + """Whether the benchmark was iterated at least once.""" + return self.iterations > 0 + + @property + def finished(self) -> bool: + """Whether the benchmark finished all its programmed iterations.""" + return self._finished or self.iterations >= self.size + + @property + def executed(self) -> bool: + """True of the benchmark is started and finished.""" + return self.started and self.finished + + @property + def duration(self) -> float: + """The process time of the benchmark. + + The process time is calculated using the time module from the Python + Standard Library. Check its definition on the library's docs + https://docs.python.org/dev/library/time.html#time.process_time . + """ + return sum(float(x) for x in self._iter_times if x is not None) + + @property + def iterations(self) -> int: + """The number of iterations already executed.""" + return len(tuple(None for x in self._iter_times if x is not None)) + + @property + def iteration(self) -> Union[int, None]: + """The current iteration. + + Returns: + Union[int, None]: either the current iteration or None if no + iterations were yet run. + """ + if self.iterations > 0: + return self.iterations - 1 + else: + return None + + @property + def size(self) -> int: + """The number of iterations programmed on initialization. + + When the number of executed iterations reaches the value of this + parameter, the benchmark is finished. + """ + return self._size + + def set_up(self): + """Set up the benchmark. The time spent in the setup is not counted.""" + if not self.started and not self.finished: + self._benchmark_set_up() + elif self.started and not self.finished: + raise RuntimeError('This benchmark has already started.') + else: # Both are true. + raise StopIteration('This benchmark is finished.') + + @abstractmethod + def _benchmark_set_up(self): + """Implementation of the setup for a specific benchmark.""" + pass + + def tear_down(self): + """Clean up after the benchmark. The time spent is not counted.""" + self._benchmark_tear_down() + + @abstractmethod + def _benchmark_tear_down(self): + """Implementation of the teardown for a specific benchmark.""" + pass + + def iterate(self): + """Perform one iteration of the benchmark. + + Raises: + StopIteration: when all the iterations of the benchmark were + already executed. + """ + if self.finished: + raise StopIteration('This benchmark is finished.') + iteration = self.iterations + start = time.process_time() + self._benchmark_iterate(iteration=iteration) + end = time.process_time() + self._iter_times[iteration] = end - start + + @abstractmethod + def _benchmark_iterate(self, iteration: int = None): + """Implementation of a benchmark iteration for a specific benchmark. + + The time taken to execute any code inside this method is registered. + + Args: + iteration (int): the iteration number to be performed. + """ + + def run(self): + """Run a benchmark from start to finish. + + This method will only work on a benchmark that has not been started + already. It runs all of its programmed iterations. + """ + self.set_up() + for i in range(self.size): + self.iterate() + self.tear_down() + + @classmethod + def iterate_pytest_benchmark(cls, benchmark, size: int = 500, + *args, **kwargs): + """Template wrapper function for pytest-benchmark. + + Can be overridden on a benchmark basis if desired. + """ + kwargs['iterations'] = kwargs.get('rounds', 1) + kwargs['rounds'] = kwargs.get('rounds', size) + kwargs['warmup_rounds'] = kwargs.get('warmup_rounds', 0) + benchmark_instance = cls(size=size) + benchmark_instance.set_up() + benchmark.pedantic(benchmark_instance.iterate, *args, **kwargs) + benchmark_instance.tear_down() diff --git a/tests/benchmark_cuds_api.py b/tests/benchmark_cuds_api.py new file mode 100644 index 00000000..00160676 --- /dev/null +++ b/tests/benchmark_cuds_api.py @@ -0,0 +1,424 @@ +"""Test the performance of basic API calls.""" + +import random +import itertools +import rdflib +from .benchmark import Benchmark + +try: + from osp.core.namespaces import city +except ImportError: # When the city ontology is not installed. + from osp.core.ontology import Parser + from osp.core.ontology.namespace_registry import namespace_registry + Parser().parse("city") + city = namespace_registry.city + + +DEFAULT_SIZE = 500 + + +class CudsCreate(Benchmark): + """Benchmark the creation of CUDS objects.""" + + def _benchmark_set_up(self): + pass + + def _benchmark_iterate(self, iteration=None): + city.Citizen(name='citizen ' + str(iteration), + age=random.randint(0, 80)) + + def _benchmark_tear_down(self): + pass + + +def benchmark_cuds_create(benchmark): + """Wrapper function for the CudsCreate benchmark.""" + return CudsCreate.iterate_pytest_benchmark(benchmark, size=DEFAULT_SIZE) + + +# `add` method. + +class Cuds_add_Default(Benchmark): + """Benchmark the `add` method using the default relationship.""" + + def _benchmark_set_up(self): + self.city = city.City(name='Freiburg') + self.neighborhoods = tuple(city.Neighborhood(name=f'neighborhood {i}') + for i in range(self.size)) + + def _benchmark_iterate(self, iteration=None): + self.city.add(self.neighborhoods[iteration]) + + def _benchmark_tear_down(self): + pass + + +def benchmark_add_default(benchmark): + """Wrapper function for the Cuds_add_Default benchmark.""" + return Cuds_add_Default.iterate_pytest_benchmark(benchmark, + size=DEFAULT_SIZE) + + +class Cuds_add_Rel(Benchmark): + """Benchmark the `add` method, selecting the relationship.""" + + def _benchmark_set_up(self): + self.city = city.City(name='Freiburg') + self.citizens = tuple(city.Citizen(name=f'citizen {i}') + for i in range(self.size)) + + def _benchmark_iterate(self, iteration=None): + self.city.add(self.citizens[iteration], rel=city.hasInhabitant) + + def _benchmark_tear_down(self): + pass + + +def benchmark_cuds_add_rel(benchmark): + """Wrapper function for the Cuds_add_Rel benchmark.""" + return Cuds_add_Rel.iterate_pytest_benchmark(benchmark, size=DEFAULT_SIZE) + + +# `get` method + +class Cuds_get_ByuidUUID(Benchmark): + """Benchmark getting CUDS objects by uid (of type UUID).""" + + def _benchmark_set_up(self): + self.city = city.City(name='Freiburg') + self.citizens = tuple(city.Citizen(name=f'citizen {i}') + for i in range(self.size)) + self.uids = tuple(citizen.uid for citizen in self.citizens) + for citizen in self.citizens: + self.city.add(citizen, rel=city.hasInhabitant) + + def _benchmark_iterate(self, iteration: int = None): + self.city.get(self.uids[iteration]) + + def _benchmark_tear_down(self): + pass + + +def benchmark_cuds_get_byuiduuid(benchmark): + """Wrapper function for the Cuds_get_ByuidUUID benchmark.""" + return Cuds_get_ByuidUUID.iterate_pytest_benchmark(benchmark, + size=DEFAULT_SIZE) + + +class Cuds_get_ByuidURIRef(Benchmark): + """Benchmark getting CUDS objects by uid (of type IRI).""" + + def _benchmark_set_up(self): + self.city = city.City(name='Freiburg') + self.iris = tuple(rdflib.URIRef(f'http://example.org/city#Citizen_{i}') + for i in range(self.size)) + self.citizens = tuple(city.Citizen(name=f'citizen {i}', + uid=self.iris[i]) + for i in range(self.size)) + for citizen in self.citizens: + self.city.add(citizen, rel=city.hasInhabitant) + + def _benchmark_iterate(self, iteration: int = None): + self.city.get(self.iris[iteration]) + + def _benchmark_tear_down(self): + pass + + +def benchmark_get_byuiduriref(benchmark): + """Wrapper function for the Cuds_get_ByuidURIRef benchmark.""" + return Cuds_get_ByuidURIRef.iterate_pytest_benchmark(benchmark, + size=DEFAULT_SIZE) + + +class Cuds_get_ByRel(Benchmark): + """Benchmark getting CUDS objects by relationship.""" + + def _benchmark_set_up(self): + self.city = city.City(name='Freiburg') + citizen = city.Citizen(name='Citizen') + streets = tuple(city.Street(name=f'street {i}') + for i in range(self.size - 1)) + position = random.randint(0, (self.size - 1) - 1) + things = list(streets) + things.insert(position, citizen) + things = tuple(things) + rel = {position: city.hasInhabitant} + for i, thing in enumerate(things): + self.city.add(thing, rel=rel.get(i, city.hasPart)) + + def _benchmark_iterate(self, iteration: int = None): + self.city.get(rel=city.hasInhabitant) + + def _benchmark_tear_down(self): + pass + + +def benchmark_get_byrel(benchmark): + """Wrapper function for the Cuds_get_ByRel benchmark.""" + return Cuds_get_ByRel.iterate_pytest_benchmark(benchmark, + size=DEFAULT_SIZE) + + +class Cuds_get_Byoclass(Benchmark): + """Benchmark getting CUDS objects by oclass.""" + + def _benchmark_set_up(self): + self.city = city.City(name='Freiburg') + citizen = city.Citizen(name='Citizen') + streets = tuple(city.Street(name=f'street {i}') + for i in range(self.size - 1)) + position = random.randint(0, (self.size - 1) - 1) + things = list(streets) + things.insert(position, citizen) + things = tuple(things) + rel = {position: city.hasInhabitant} + for i, thing in enumerate(things): + self.city.add(thing, rel=rel.get(i, city.hasPart)) + + def _benchmark_iterate(self, iteration: int = None): + self.city.get(oclass=city.Citizen) + + def _benchmark_tear_down(self): + pass + + +def benchmark_get_byoclass(benchmark): + """Wrapper function for the Cuds_get_Byoclass benchmark.""" + return Cuds_get_Byoclass.iterate_pytest_benchmark(benchmark, + DEFAULT_SIZE) + + +# `iter` method + +class Cuds_iter_ByuidUUID(Benchmark): + """Benchmark getting CUDS objects by uid (of type UUID).""" + + def _benchmark_set_up(self): + self.city = city.City(name='Freiburg') + self.citizens = tuple(city.Citizen(name=f'citizen {i}') + for i in range(self.size)) + self.uids = tuple(citizen.uid for citizen in self.citizens) + for citizen in self.citizens: + self.city.add(citizen, rel=city.hasInhabitant) + self.iterator = self.city.iter(*self.uids) + + def _benchmark_iterate(self, iteration: int = None): + next(self.iterator) + + def _benchmark_tear_down(self): + pass + + +def benchmark_cuds_iter_byuiduuid(benchmark): + """Wrapper function for the Cuds_iter_ByuidUUID benchmark.""" + return Cuds_iter_ByuidUUID.iterate_pytest_benchmark(benchmark, + size=DEFAULT_SIZE) + + +class Cuds_iter_ByuidURIRef(Benchmark): + """Benchmark getting CUDS objects by uid (of type UUID).""" + + def _benchmark_set_up(self): + self.city = city.City(name='Freiburg') + self.iris = tuple(rdflib.URIRef(f'http://example.org/city#Citizen_{i}') + for i in range(self.size)) + self.citizens = tuple(city.Citizen(name=f'citizen {i}', + uid=self.iris[i]) + for i in range(self.size)) + for citizen in self.citizens: + self.city.add(citizen, rel=city.hasInhabitant) + self.iterator = self.city.iter(*self.iris) + + def _benchmark_iterate(self, iteration: int = None): + next(self.iterator) + + def _benchmark_tear_down(self): + pass + + +def benchmark_cuds_iter_byuiduriref(benchmark): + """Wrapper function for the Cuds_iter_ByuidURIRef benchmark.""" + return Cuds_iter_ByuidURIRef.iterate_pytest_benchmark(benchmark, + size=DEFAULT_SIZE) + + +class Cuds_iter_ByRel(Benchmark): + """Benchmark getting CUDS objects by relationship.""" + + def _benchmark_set_up(self): + self.city = city.City(name='Freiburg') + citizen = city.Citizen(name='Citizen') + streets = tuple(city.Street(name=f'street {i}') + for i in range(self.size - 1)) + position = random.randint(0, (self.size - 1) - 1) + things = list(streets) + things.insert(position, citizen) + things = tuple(things) + rel = {position: city.hasInhabitant} + for i, thing in enumerate(things): + self.city.add(thing, rel=rel.get(i, city.hasPart)) + + def _benchmark_iterate(self, iteration: int = None): + next(self.city.iter(rel=city.hasInhabitant)) + + def _benchmark_tear_down(self): + pass + + +def benchmark_cuds_iter_byrel(benchmark): + """Wrapper function for the Cuds_iter_ByRel benchmark.""" + return Cuds_iter_ByRel.iterate_pytest_benchmark(benchmark, + size=DEFAULT_SIZE) + + +class Cuds_iter_Byoclass(Benchmark): + """Benchmark getting CUDS objects by oclass.""" + + def _benchmark_set_up(self): + self.city = city.City(name='Freiburg') + citizen = city.Citizen(name='Citizen') + streets = tuple(city.Street(name=f'street {i}') + for i in range(self.size - 1)) + position = random.randint(0, (self.size - 1) - 1) + things = list(streets) + things.insert(position, citizen) + things = tuple(things) + rel = {position: city.hasInhabitant} + for i, thing in enumerate(things): + self.city.add(thing, rel=rel.get(i, city.hasPart)) + + def _benchmark_iterate(self, iteration: int = None): + next(self.city.iter(oclass=city.Citizen)) + + def _benchmark_tear_down(self): + pass + + +def benchmark_iter_byoclass(benchmark): + """Wrapper function for the Cuds_iter_Byoclass benchmark.""" + return Cuds_iter_Byoclass.iterate_pytest_benchmark(benchmark, + size=DEFAULT_SIZE) + + +# `is_a` method + +class Cuds_is_a(Benchmark): + """Benchmark checking the oclass of CUDS objects.""" + + def _benchmark_set_up(self): + oclasses_noname = (city.LivingBeing, city.ArchitecturalComponent, + city.Citizen, city.Person, city.Floor) + oclasses_name = (city.City, city.ArchitecturalStructure, city.Street, + city.Neighborhood, city.PopulatedPlace, city.Building) + unnamed_stuff = tuple(oclass() for oclass in oclasses_noname) + named_stuff = tuple(oclass(name='name') for oclass in oclasses_name) + self.iterator_stuff = itertools.cycle(unnamed_stuff + named_stuff) + self.oclasses = oclasses_noname + oclasses_name + + def _benchmark_iterate(self, iteration: int = None): + cuds = next(self.iterator_stuff) + oclass = random.choice(self.oclasses) + cuds.is_a(oclass) + + def _benchmark_tear_down(self): + pass + + +def benchmark_cuds_is_a(benchmark): + """Wrapper function for the Cuds_is_a benchmark.""" + return Cuds_is_a.iterate_pytest_benchmark(benchmark, size=DEFAULT_SIZE) + + +# `oclass` property + +class Cuds_oclass(Benchmark): + """Benchmark getting the oclass of CUDS objects.""" + + def _benchmark_set_up(self): + self.citizen = city.Citizen(name='someone') + + def _benchmark_iterate(self, iteration: int = None): + self.citizen.oclass + + def _benchmark_tear_down(self): + pass + + +def benchmark_cuds_oclass(benchmark): + """Wrapper function for the Cuds_oclass benchmark.""" + return Cuds_oclass.iterate_pytest_benchmark(benchmark, size=DEFAULT_SIZE) + + +# `uid` property + +class Cuds_uid(Benchmark): + """Benchmark getting the uid of CUDS objects.""" + + def _benchmark_set_up(self): + self.citizen = city.Citizen(name='someone') + + def _benchmark_iterate(self, iteration: int = None): + self.citizen.uid + + def _benchmark_tear_down(self): + pass + + +def benchmark_cuds_uid(benchmark): + """Wrapper function for the Cuds_uid benchmark.""" + return Cuds_uid.iterate_pytest_benchmark(benchmark, size=DEFAULT_SIZE) + + +# `iri` property + +class Cuds_iri(Benchmark): + """Benchmark getting the iri of CUDS objects.""" + + def _benchmark_set_up(self): + self.citizen = city.Citizen(name='someone') + + def _benchmark_iterate(self, iteration: int = None): + self.citizen.iri + + def _benchmark_tear_down(self): + pass + + +def benchmark_cuds_iri(benchmark): + """Wrapper function for the Cuds_iri benchmark.""" + return Cuds_iri.iterate_pytest_benchmark(benchmark, size=DEFAULT_SIZE) + + +# get attributes + +class Cuds_attributes(Benchmark): + """Benchmark fetching attributes of CUDS objects.""" + + def _benchmark_set_up(self): + self.citizen = city.Citizen(name='Lukas', age=93) + self.city = city.City(name='Freiburg', coordinates=[108, 49]) + self.address = city.Address(postalCode=79111) + self.things = itertools.cycle((self.citizen, self.city, self.address)) + self.attributes = itertools.cycle((('age', ), + ('coordinates', ), + ('postalCode', ))) + + def _benchmark_iterate(self, iteration: int = None): + thing = next(self.things) + attributes = next(self.attributes) + for attr in attributes: + getattr(thing, attr) + + def _benchmark_tear_down(self): + pass + + +def benchmark_cuds_attributes(benchmark): + """Wrapper function for the Cuds_attributes benchmark.""" + return Cuds_attributes.iterate_pytest_benchmark(benchmark, + size=DEFAULT_SIZE) + + +if __name__ == '__main__': + pass diff --git a/tests/performance_test.py b/tests/performance_test.py deleted file mode 100644 index 11acb4df..00000000 --- a/tests/performance_test.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Test the performance of basic API calls.""" - -# pip install pympler -import gc -from pympler import asizeof -import time -import uuid -import unittest2 as unittest - -try: - from osp.core.namespaces import city -except ImportError: - from osp.core.ontology import Parser - from osp.core.ontology.namespace_registry import namespace_registry - Parser().parse("city") - city = namespace_registry.city - -RUN_PERFORMANCE_TEST = False - - -class TestPerformance(unittest.TestCase): - """Test the performance of basic API calls.""" - - def setUp(self): - """Start the timer.""" - if not RUN_PERFORMANCE_TEST: - return - self.iterations = 100000 - self.c = city.City(name="A big city") - for i in range(10): - j = i * 9 - self.c.add(city.Citizen(uid=j + 0), rel=city.hasInhabitant) - self.c.add(city.Citizen(uid=j + 1), rel=city.encloses) - self.c.add(city.Citizen(uid=j + 2), rel=city.hasPart) - self.c.add(city.Neighborhood(name="", uid=j + 3), - rel=city.hasInhabitant) - self.c.add(city.Neighborhood(name="", uid=j + 4), - rel=city.encloses) - self.c.add(city.Neighborhood(name="", uid=j + 5), - rel=city.hasPart) - self.c.add(city.Street(name="", uid=j + 6), - rel=city.hasInhabitant) - self.c.add(city.Street(name="", uid=j + 7), rel=city.encloses) - self.c.add(city.Street(name="", uid=j + 8), rel=city.hasPart) - self.start = time.time() - - def tearDown(self): - """Stop the timer and print the results.""" - if not RUN_PERFORMANCE_TEST: - return - self.stop = time.time() - mem_bytes = asizeof.asizeof(self.c) - mem_mb = mem_bytes / (1024 * 1024.0) - print('Memory usage: ' + str(mem_mb) + ' MegaBytes.') - total = self.stop - self.start - if total > 60: - print('Total time: ' + str(total / 60) + ' minutes.') - else: - print('Total time: ' + str(total) + ' seconds.') - self.c._session = None - gc.collect() - - def test_creation(self): - """Test the instantiation and type of the objects.""" - if not RUN_PERFORMANCE_TEST: - return - print("Test cuds object creation") - for i in range(self.iterations): - city.Citizen(name='citizen ' + str(i)) - - def test_add_default(self): - """Test the instantiation and type of the objects.""" - if not RUN_PERFORMANCE_TEST: - return - print("Test adding with the default relationship") - for i in range(self.iterations): - self.c.add(city.Neighborhood( - name='neighborhood ' + str(i))) - - def test_add_rel(self): - """Test the instantiation and type of the objects.""" - if not RUN_PERFORMANCE_TEST: - return - print("Test adding with a general relationship") - for i in range(self.iterations): - self.c.add(city.Citizen(name='citizen ' + str(i)), - rel=city.hasInhabitant) - - def test_get_by_oclass(self): - """Test performance of getting by oclass.""" - if not RUN_PERFORMANCE_TEST: - return - print("Test get by oclass") - for i in range(self.iterations): - self.c.get(oclass=city.Street) - - def test_get_by_uid(self): - """Test the performance of getting by UUID.""" - if not RUN_PERFORMANCE_TEST: - return - print("Test get by uid") - uids = list(map(lambda x: uuid.UUID(int=x), range(10))) - for i in range(self.iterations): - self.c.get(*uids) - - def test_get_by_rel(self): - """Test the performance of getting by relationship.""" - if not RUN_PERFORMANCE_TEST: - return - print("Test get by relationship") - for i in range(self.iterations): - self.c.get(rel=city.hasInhabitant) - - -if __name__ == '__main__': - RUN_PERFORMANCE_TEST = True - unittest.main() diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 00000000..cff4fc42 --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +python_files = benchmark_*.py +python_functions = benchmark_* +testpaths = + tests \ No newline at end of file