From efe837442c13b5f2aff47024b04776e953f62d21 Mon Sep 17 00:00:00 2001 From: Hyaxia Date: Fri, 8 Feb 2019 20:39:09 +0200 Subject: [PATCH 1/2] Defined a standard for adding and using kernel metrics. Any addition of a new metric will not effect the `KernelManager` class at all (see how its designed). --- jupyter_client/kernelmetrics.py | 80 ++++++++++++++++++++++ jupyter_client/manager.py | 32 +++++++++ jupyter_client/tests/test_kernelmanager.py | 12 ++++ 3 files changed, 124 insertions(+) create mode 100644 jupyter_client/kernelmetrics.py diff --git a/jupyter_client/kernelmetrics.py b/jupyter_client/kernelmetrics.py new file mode 100644 index 000000000..b5d815d21 --- /dev/null +++ b/jupyter_client/kernelmetrics.py @@ -0,0 +1,80 @@ +from abc import ABCMeta, abstractmethod +from six import add_metaclass +import re + +METRIC_NAME_PREFIX = "KernelMetric" + + +class KernelMetricMeta(ABCMeta): + """ + Meta class for all of the kernel metric classes. + When defining a class, will check for its name, that it has the + predefined prefix. + If not, throws an assertion error. + """ + def __new__(mcs, name, bases, dct): + assert name.startswith(METRIC_NAME_PREFIX) + return super(KernelMetricMeta, mcs).__new__(mcs, name, bases, dct) + + +@add_metaclass(KernelMetricMeta) +class KernelMetric(object): + """ + An abstract base class for all of the metric classes. + """ + def __init__(self, kernel_manager): + """ + :param kernel_manager: desired kernel to pull the metric from + """ + self.kernel_manager = kernel_manager + + @abstractmethod + def poll(self): + """ + Polls the necessary metric from the kernel and returns it + :return: Metric data + """ + pass + + +class KernelMetricMemoryUsage(KernelMetric): + """ + Returns memory usage of the kernel in MB. + `poll` function should be overwritten if there is a + need for a different kind of implementation. + """ + def poll(self): + from psutil import Process + kernel_pid = self.kernel_manager.kernel.pid + process = Process(kernel_pid) + mem = process.memory_info().vms + return self._bytes_to_mb(mem) + + def _bytes_to_mb(self, num): + return int(num / 1024 ** 2) + + +""" +KERNEL_METRIC_TYPES is a dictionary that maps the names of the metrics +to the metric classes themselves. + +Example: +If the name of the class that exposes a metric is "KernelMetricMemoryUsage", +the dictionary will remove the must have prefix "KernelMetric", +split the rest by camel case letters with a "_" delimiter, and then turn them all to lower case. +the name that maps to the class of the metric will be "memory_usage". + +*ALL KERNEL METRIC CLASSES MUST BE DEFINED ABOVE, INHERITING FROM THE BASE CLASS `KernelMetric` +*ALL KERNEL METRIC CLASSES MUST HAVE THE PREFIX DEFINED AT THE TOP OF THE FILE +*ALL KERNEL METRIC CLASSES MUST RECEIVE ONLY THE KERNEL MANAGER AS ITS FIRST AND ONLY PARAMETER +*ALL KERNEL METRIC CLASSES MUST DEFINE THE `poll` METHOD THAT RETURNS THE DESIRED METRIC +""" +KERNEL_METRIC_TYPES = {} +for cls in KernelMetric.__subclasses__(): + metric_name = cls.__name__[len(METRIC_NAME_PREFIX):] + metric_name = re.sub('(?!^)([A-Z][a-z]+)', r' \1', metric_name).split() + metric_name = "_".join(metric_name).lower() + KERNEL_METRIC_TYPES[metric_name] = cls + + + diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index bf94ad002..5050dc4c0 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -27,6 +27,7 @@ from .managerabc import ( KernelManagerABC ) +from .kernelmetrics import KERNEL_METRIC_TYPES class KernelManager(ConnectionFileMixin): @@ -454,6 +455,37 @@ def is_alive(self): # we don't have a kernel return False + def kernel_metrics(self, *desired_metrics): + """ + Returns a dict containing the results of the gathered + statistics. + + Parameters + ---------- + + *desired_metrics: A list of strings representing the kinds of + metrics they are interested in polling. + + Usage + ----- + >>> ks = km.kernel_metrics("memory_usage", "latency") + The function call above will return a dict with two key-value + pairs (assuming there are function to gather those stats). + The keys are exactly the same as the inputs. + >>> print ks["memory_usage"] + Line above will print the memory usage of the kernel. + + """ + results = {} + for metric_name in desired_metrics: + try: + cls = KERNEL_METRIC_TYPES[metric_name] + except KeyError: + continue + else: + metric = cls(self) + results[metric_name] = metric.poll() + return results KernelManagerABC.register(KernelManager) diff --git a/jupyter_client/tests/test_kernelmanager.py b/jupyter_client/tests/test_kernelmanager.py index a23b33fa6..f1a77988f 100644 --- a/jupyter_client/tests/test_kernelmanager.py +++ b/jupyter_client/tests/test_kernelmanager.py @@ -127,3 +127,15 @@ def test_start_new_kernel(self): self.assertTrue(km.is_alive()) self.assertTrue(kc.is_alive()) + + def test_kernel_stats(self): + c = Config() + km = KernelManager(config=c) + km.start_kernel(stdout=PIPE, stderr=PIPE) + # Call the kernel_stats function requesting the + # memory usage of the kernel. + ks = km.kernel_metrics("memory_usage") + km.shutdown_kernel(now=True) + # An arbitrary check for a value of a certain key in the result + # of `kernel_stats`, in this case its the memory usage one. + assert type(ks["memory_usage"]) is int From 457bee9a42f387bf975a5d595f994b967ee258bf Mon Sep 17 00:00:00 2001 From: Hyaxia Date: Sat, 1 Jun 2019 02:48:53 +0300 Subject: [PATCH 2/2] Refactor - changed the way we save the metric classes. Now the class KernelMetricStore acts as a store of all the metric classes that inherit from `KernelMetric` class, so no matter from where the new metric class is defined, the moment it locates it, the class will be added to the `TYPES` dictionary of the KernelMetricStore class and you will be able to poll it through the `kernel_metrics` function of the `KernelManager` class. --- jupyter_client/kernelmetrics.py | 65 +++++++++++++++++---------------- jupyter_client/manager.py | 4 +- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/jupyter_client/kernelmetrics.py b/jupyter_client/kernelmetrics.py index b5d815d21..719ff26c9 100644 --- a/jupyter_client/kernelmetrics.py +++ b/jupyter_client/kernelmetrics.py @@ -5,19 +5,45 @@ METRIC_NAME_PREFIX = "KernelMetric" -class KernelMetricMeta(ABCMeta): +class KernelMetricStore(ABCMeta): """ - Meta class for all of the kernel metric classes. - When defining a class, will check for its name, that it has the - predefined prefix. - If not, throws an assertion error. + KernelMetricStore Is acting as a store of all the metric classes created using KernelMetric base class. + TYPES is a dictionary that maps the names of the metrics to the metric classes themselves. + + Additionally it defines defines a standard for how kernel metric classes should be named. + Example: + If the name of the class that exposes a metric is "KernelMetricMemoryUsage", + the dictionary will remove the must have prefix "KernelMetric", + split the rest by camel case letters with a "_" delimiter, and then turn them all to lower case. + the name that maps to the class of the metric will be "memory_usage". + + *ALL KERNEL METRIC CLASSES MUST INHERIT FROM THE BASE CLASS `KernelMetric` + *ALL KERNEL METRIC CLASSES MUST HAVE THE PREFIX IN THEIR NAME + + When we define a metric class to inherit from the `KernelMetric`, this class will automatically + check that its name has the need prefix, and will add it to the TYPES dict. """ + TYPES = {} + def __new__(mcs, name, bases, dct): + mcs._check_name_prefix(mcs, name) + new_metric_class = super(KernelMetricStore, mcs).__new__(mcs, name, bases, dct) + if new_metric_class.__name__ == "KernelMetric": + return new_metric_class + mcs._add_new_metric(mcs, new_metric_class) + return new_metric_class + + def _add_new_metric(self, new_metric): + metric_name = new_metric.__name__[len(METRIC_NAME_PREFIX):] + metric_name = re.sub('(?!^)([A-Z][a-z]+)', r' \1', metric_name).split() + metric_name = "_".join(metric_name).lower() + self.TYPES[metric_name] = new_metric + + def _check_name_prefix(self, name): assert name.startswith(METRIC_NAME_PREFIX) - return super(KernelMetricMeta, mcs).__new__(mcs, name, bases, dct) -@add_metaclass(KernelMetricMeta) +@add_metaclass(KernelMetricStore) class KernelMetric(object): """ An abstract base class for all of the metric classes. @@ -53,28 +79,3 @@ def poll(self): def _bytes_to_mb(self, num): return int(num / 1024 ** 2) - -""" -KERNEL_METRIC_TYPES is a dictionary that maps the names of the metrics -to the metric classes themselves. - -Example: -If the name of the class that exposes a metric is "KernelMetricMemoryUsage", -the dictionary will remove the must have prefix "KernelMetric", -split the rest by camel case letters with a "_" delimiter, and then turn them all to lower case. -the name that maps to the class of the metric will be "memory_usage". - -*ALL KERNEL METRIC CLASSES MUST BE DEFINED ABOVE, INHERITING FROM THE BASE CLASS `KernelMetric` -*ALL KERNEL METRIC CLASSES MUST HAVE THE PREFIX DEFINED AT THE TOP OF THE FILE -*ALL KERNEL METRIC CLASSES MUST RECEIVE ONLY THE KERNEL MANAGER AS ITS FIRST AND ONLY PARAMETER -*ALL KERNEL METRIC CLASSES MUST DEFINE THE `poll` METHOD THAT RETURNS THE DESIRED METRIC -""" -KERNEL_METRIC_TYPES = {} -for cls in KernelMetric.__subclasses__(): - metric_name = cls.__name__[len(METRIC_NAME_PREFIX):] - metric_name = re.sub('(?!^)([A-Z][a-z]+)', r' \1', metric_name).split() - metric_name = "_".join(metric_name).lower() - KERNEL_METRIC_TYPES[metric_name] = cls - - - diff --git a/jupyter_client/manager.py b/jupyter_client/manager.py index 5050dc4c0..766d89ffe 100644 --- a/jupyter_client/manager.py +++ b/jupyter_client/manager.py @@ -27,7 +27,7 @@ from .managerabc import ( KernelManagerABC ) -from .kernelmetrics import KERNEL_METRIC_TYPES +from .kernelmetrics import KernelMetricStore class KernelManager(ConnectionFileMixin): @@ -479,7 +479,7 @@ def kernel_metrics(self, *desired_metrics): results = {} for metric_name in desired_metrics: try: - cls = KERNEL_METRIC_TYPES[metric_name] + cls = KernelMetricStore.TYPES[metric_name] except KeyError: continue else: