diff --git a/posttroll/__init__.py b/posttroll/__init__.py index df053e3..da8f4a1 100644 --- a/posttroll/__init__.py +++ b/posttroll/__init__.py @@ -24,9 +24,7 @@ """Posttroll packages.""" -import datetime as dt import logging -import sys from donfig import Config @@ -46,27 +44,3 @@ def get_context(): return get_context() else: raise NotImplementedError(f"No support for backend {backend} implemented (yet?).") - - -def strp_isoformat(strg): - """Decode an ISO formatted string to a datetime object. - - Allow a time-string without microseconds. - - We handle input like: 2011-11-14T12:51:25.123456 - """ - if isinstance(strg, dt.datetime): - return strg - if len(strg) < 19 or len(strg) > 26: - if len(strg) > 30: - strg = strg[:30] + "..." - raise ValueError("Invalid ISO formatted time string '%s'" % strg) - if strg.find(".") == -1: - strg += ".000000" - if sys.version[0:3] >= "2.6": - return dt.datetime.strptime(strg, "%Y-%m-%dT%H:%M:%S.%f") - else: - dat, mis = strg.split(".") - dat = dt.datetime.strptime(dat, "%Y-%m-%dT%H:%M:%S") - mis = int(float("." + mis) * 1000000) - return dat.replace(microsecond=mis) diff --git a/posttroll/address_receiver.py b/posttroll/address_receiver.py index 689410b..5928075 100644 --- a/posttroll/address_receiver.py +++ b/posttroll/address_receiver.py @@ -89,7 +89,7 @@ def __init__(self, max_age=ten_minutes, port=None, self._subject = "/address" self._do_heartbeat = do_heartbeat self._multicast_enabled = multicast_enabled - self._last_age_check = dt.datetime(1900, 1, 1) + self._last_age_check = dt.datetime(1900, 1, 1, tzinfo=dt.timezone.utc) self._do_run = False self._is_running = False self._thread = threading.Thread(target=self._run) @@ -128,11 +128,11 @@ def get(self, name=""): def _check_age(self, pub, min_interval=zero_seconds): """Check the age of the receiver.""" - now = dt.datetime.utcnow() + now = dt.datetime.now(dt.timezone.utc) if (now - self._last_age_check) <= min_interval: return - logger.debug("%s - checking addresses", str(dt.datetime.utcnow())) + LOGGER.debug("%s - checking addresses", str(dt.datetime.now(dt.timezone.utc))) self._last_age_check = now to_del = [] with self._address_lock: @@ -232,7 +232,7 @@ def set_up_address_receiver(self, port): def _add(self, adr, metadata): """Add an address.""" with self._address_lock: - metadata["receive_time"] = dt.datetime.utcnow() + metadata["receive_time"] = dt.datetime.now(dt.timezone.utc) self._addresses[adr] = metadata diff --git a/posttroll/message.py b/posttroll/message.py index ab68484..038bb7a 100644 --- a/posttroll/message.py +++ b/posttroll/message.py @@ -46,8 +46,6 @@ except ImportError: import simplejson as json -from posttroll import strp_isoformat - _MAGICK = "pytroll:/" _VERSION = "v1.01" @@ -132,7 +130,7 @@ def __init__(self, subject="", atype="", data="", binary=False, rawstr=None): self.type = atype self.type = atype self.sender = _getsender() - self.time = dt.datetime.utcnow() + self.time = dt.datetime.now(dt.timezone.utc) self.data = data self.binary = binary self.version = _VERSION @@ -229,7 +227,7 @@ def datetime_decoder(dct): for key, val in pairs: if isinstance(val, str): try: - val = strp_isoformat(val) + val = dt.datetime.fromisoformat(val) except ValueError: pass elif isinstance(val, (dict, list)): @@ -252,7 +250,7 @@ def _decode(rawstr): msg = dict((("subject", raw[0].strip()), ("type", raw[1].strip()), ("sender", raw[2].strip()), - ("time", strp_isoformat(raw[3].strip())), + ("time", dt.datetime.fromisoformat(raw[3].strip())), ("version", version))) # Data part diff --git a/posttroll/ns.py b/posttroll/ns.py index 1bf05b4..a63a6d3 100644 --- a/posttroll/ns.py +++ b/posttroll/ns.py @@ -48,7 +48,7 @@ def get_configured_nameserver_port(): try: port = int(os.environ["NAMESERVER_PORT"]) warnings.warn("NAMESERVER_PORT is pending deprecation, please use POSTTROLL_NAMESERVER_PORT instead.", - PendingDeprecationWarning) + PendingDeprecationWarning, stacklevel=2) except KeyError: port = DEFAULT_NAMESERVER_PORT return config.get("nameserver_port", port) @@ -68,8 +68,8 @@ def get_pub_addresses(names=None, timeout=10, nameserver="localhost"): if names is None: names = ["", ] for name in names: - then = dt.datetime.now() + dt.timedelta(seconds=timeout) - while dt.datetime.now() < then: + then = dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=timeout) + while dt.datetime.now(dt.timezone.utc) < then: addrs += get_pub_address(name, nameserver=nameserver, timeout=timeout) if addrs: break diff --git a/posttroll/publisher.py b/posttroll/publisher.py index dee85cc..4cd1313 100644 --- a/posttroll/publisher.py +++ b/posttroll/publisher.py @@ -140,13 +140,13 @@ class _PublisherHeartbeat: def __init__(self, publisher): self.publisher = publisher self.subject = "/heartbeat/" + publisher.name - self.lastbeat = dt.datetime(1900, 1, 1) + self.lastbeat = dt.datetime(1900, 1, 1, tzinfo=dt.timezone.utc) def __call__(self, min_interval=0): if not min_interval or ( - (dt.datetime.utcnow() - self.lastbeat >= + (dt.datetime.now(dt.timezone.utc) - self.lastbeat >= dt.timedelta(seconds=min_interval))): - self.lastbeat = dt.datetime.utcnow() + self.lastbeat = dt.datetime.now(dt.timezone.utc) LOGGER.debug("Publish heartbeat (min_interval is %.1f sec)", min_interval) self.publisher.send(Message(self.subject, "beat", {"min_interval": min_interval}).encode()) diff --git a/posttroll/subscriber.py b/posttroll/subscriber.py index fc3a8c1..b6a3e52 100644 --- a/posttroll/subscriber.py +++ b/posttroll/subscriber.py @@ -196,8 +196,8 @@ def start(self): """Start the subscriber.""" def _get_addr_loop(service, timeout): """Try to get the address of *service* until for *timeout* seconds.""" - then = dt.datetime.now() + dt.timedelta(seconds=timeout) - while dt.datetime.now() < then: + then = dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=timeout) + while dt.datetime.now(dt.timezone.utc) < then: addrs = get_pub_address(service, self._timeout, nameserver=self._nameserver) if addrs: return [addr["URI"] for addr in addrs] diff --git a/posttroll/tests/data/message_metadata_aware.dumps b/posttroll/tests/data/message_metadata_aware.dumps new file mode 100644 index 0000000..850a6cc --- /dev/null +++ b/posttroll/tests/data/message_metadata_aware.dumps @@ -0,0 +1 @@ +{"satellite": "metop2", "format": "hrpt", "timestamp": "2010-12-03T16:28:39+00:00", "afloat": 1.2344999999999999, "uri": "file://data/my/path/to/hrpt/files/myfile", "orbit": 1222} \ No newline at end of file diff --git a/posttroll/tests/data/message_metadata.dumps b/posttroll/tests/data/message_metadata_unaware.dumps similarity index 100% rename from posttroll/tests/data/message_metadata.dumps rename to posttroll/tests/data/message_metadata_unaware.dumps diff --git a/posttroll/tests/test_message.py b/posttroll/tests/test_message.py index af97236..c677eb2 100644 --- a/posttroll/tests/test_message.py +++ b/posttroll/tests/test_message.py @@ -26,8 +26,9 @@ import copy import os import sys -import unittest -from datetime import datetime +import datetime as dt + +import pytest from posttroll.message import _MAGICK, Message @@ -36,121 +37,128 @@ DATADIR = HOME + "/data" -SOME_METADATA = {"timestamp": datetime(2010, 12, 3, 16, 28, 39), - "satellite": "metop2", - "uri": "file://data/my/path/to/hrpt/files/myfile", - "orbit": 1222, - "format": "hrpt", - "afloat": 1.2345} - - -class Test(unittest.TestCase): - """Test class.""" - - def test_encode_decode(self): - """Test the encoding/decoding of the message class.""" - msg1 = Message("/test/whatup/doc", "info", data="not much to say") - - sender = "%s@%s" % (msg1.user, msg1.host) - assert sender == msg1.sender, "Messaging, decoding user, host from sender failed" - msg2 = Message.decode(msg1.encode()) - assert str(msg2) == str(msg1), "Messaging, encoding, decoding failed" - - def test_decode(self): - """Test the decoding of a message.""" - rawstr = (_MAGICK + - r"/test/1/2/3 info ras@hawaii 2008-04-11T22:13:22.123000 v1.01" + - r' text/ascii "what' + r"'" + r's up doc"') - msg = Message.decode(rawstr) - - assert str(msg) == rawstr, "Messaging, decoding of message failed" - - def test_encode(self): - """Test the encoding of a message.""" - subject = "/test/whatup/doc" - atype = "info" - data = "not much to say" - msg1 = Message(subject, atype, data=data) - sender = "%s@%s" % (msg1.user, msg1.host) - full_message = (_MAGICK + subject + " " + atype + " " + sender + " " + - str(msg1.time.isoformat()) + " " + msg1.version + " " + "text/ascii" + " " + data) - assert full_message == msg1.encode() - - def test_unicode(self): - """Test handling of unicode.""" - try: - msg = ('pytroll://PPS-monitorplot/3/norrköping/utv/polar/direct_readout/ file ' - 'safusr.u@lxserv1096.smhi.se 2018-11-16T12:19:29.934025 v1.01 application/json' - ' {"start_time": "2018-11-16T12:02:43.700000"}') - assert msg == str(Message(rawstr=msg)) - except UnicodeDecodeError: - self.fail("Unexpected unicode decoding error") +TZ_UNAWARE_METADATA = {"timestamp": dt.datetime(2010, 12, 3, 16, 28, 39), + "satellite": "metop2", + "uri": "file://data/my/path/to/hrpt/files/myfile", + "orbit": 1222, + "format": "hrpt", + "afloat": 1.2345} +TZ_AWARE_METADATA = {"timestamp": dt.datetime(2010, 12, 3, 16, 28, 39, tzinfo=dt.timezone.utc), + "satellite": "metop2", + "uri": "file://data/my/path/to/hrpt/files/myfile", + "orbit": 1222, + "format": "hrpt", + "afloat": 1.2345} + + +def test_encode_decode(): + """Test the encoding/decoding of the message class.""" + msg1 = Message("/test/whatup/doc", "info", data="not much to say") + + sender = "%s@%s" % (msg1.user, msg1.host) + assert sender == msg1.sender, "Messaging, decoding user, host from sender failed" + msg2 = Message.decode(msg1.encode()) + assert str(msg2) == str(msg1), "Messaging, encoding, decoding failed" + + +@pytest.mark.parametrize("dstr", (r"2008-04-11T22:13:22.123000", r"2008-04-11T22:13:22.123000+00:00")) +def test_decode(dstr): + """Test the decoding of a message.""" + rawstr = (_MAGICK + + r"/test/1/2/3 info ras@hawaii " + dstr + r" v1.01" + + r' text/ascii "what' + r"'" + r's up doc"') + msg = Message.decode(rawstr) + + assert str(msg) == rawstr, "Messaging, decoding of message failed" + + +def test_encode(): + """Test the encoding of a message.""" + subject = "/test/whatup/doc" + atype = "info" + data = "not much to say" + msg1 = Message(subject, atype, data=data) + sender = "%s@%s" % (msg1.user, msg1.host) + full_message = (_MAGICK + subject + " " + atype + " " + sender + " " + + str(msg1.time.isoformat()) + " " + msg1.version + " " + "text/ascii" + " " + data) + assert full_message == msg1.encode() + + +@pytest.mark.parametrize("dstr", (r"2008-04-11T22:13:22.123000", r"2008-04-11T22:13:22.123000+00:00")) +def test_unicode(dstr): + """Test handling of unicode.""" + msg = ('pytroll://PPS-monitorplot/3/norrköping/utv/polar/direct_readout/ file ' + 'safusr.u@lxserv1096.smhi.se ' + dstr + ' v1.01 application/json' + ' {"start_time": "' + dstr + '"}') + assert msg == str(Message(rawstr=msg)) + + msg = (u'pytroll://oper/polar/direct_readout/norrköping pong sat@MERLIN ' + dstr + + r' v1.01 application/json {"station": "norrk\u00f6ping"}') + try: + assert msg == str(Message(rawstr=msg)).decode("utf-8") + except AttributeError: + assert msg == str(Message(rawstr=msg)) + + +@pytest.mark.parametrize("dstr", (r"2008-04-11T22:13:22.123000", r"2008-04-11T22:13:22.123000+00:00")) +def test_iso(dstr): + """Test handling of iso-8859-1.""" + msg = ('pytroll://oper/polar/direct_readout/norrköping pong sat@MERLIN ' + dstr + + ' v1.01 application/json {"station": "norrköping"}') + try: + iso_msg = msg.decode("utf-8").encode("iso-8859-1") + except AttributeError: + iso_msg = msg.encode("iso-8859-1") + + Message(rawstr=iso_msg) + + +def test_pickle(): + """Test pickling.""" + import pickle + msg1 = Message("/test/whatup/doc", "info", data="not much to say") + try: + fp_ = open("pickle.message", "wb") + pickle.dump(msg1, fp_) + fp_.close() + fp_ = open("pickle.message", "rb") + msg2 = pickle.load(fp_) + fp_.close() + assert str(msg1) == str(msg2), "Messaging, pickle failed" + finally: try: - msg = (u'pytroll://oper/polar/direct_readout/norrköping pong sat@MERLIN 2019-01-07T12:52:19.872171' - r' v1.01 application/json {"station": "norrk\u00f6ping"}') - try: - assert msg == str(Message(rawstr=msg)).decode("utf-8") - except AttributeError: - assert msg == str(Message(rawstr=msg)) - except UnicodeDecodeError: - self.fail("Unexpected unicode decoding error") - - def test_iso(self): - """Test handling of iso-8859-1.""" - msg = ('pytroll://oper/polar/direct_readout/norrköping pong sat@MERLIN ' - '2019-01-07T12:52:19.872171 v1.01 application/json {"station": "norrköping"}') - try: - iso_msg = msg.decode("utf-8").encode("iso-8859-1") - except AttributeError: - iso_msg = msg.encode("iso-8859-1") - try: - Message(rawstr=iso_msg) - except UnicodeDecodeError: - self.fail("Unexpected iso decoding error") - - def test_pickle(self): - """Test pickling.""" - import pickle - msg1 = Message("/test/whatup/doc", "info", data="not much to say") - try: - fp_ = open("pickle.message", "wb") - pickle.dump(msg1, fp_) - fp_.close() - fp_ = open("pickle.message", "rb") - msg2 = pickle.load(fp_) - - fp_.close() - assert str(msg1) == str(msg2), "Messaging, pickle failed" - finally: - try: - os.remove("pickle.message") - except OSError: - pass - - def test_metadata(self): - """Test metadata encoding/decoding.""" - metadata = copy.copy(SOME_METADATA) - msg = Message.decode(Message("/sat/polar/smb/level1", "file", - data=metadata).encode()) - - assert msg.data == metadata, "Messaging, metadata decoding / encoding failed" - - def test_serialization(self): - """Test json serialization.""" - compare_file = "/message_metadata.dumps" - import json - metadata = copy.copy(SOME_METADATA) - metadata["timestamp"] = metadata["timestamp"].isoformat() - fp_ = open(DATADIR + compare_file) + os.remove("pickle.message") + except OSError: + pass + + +@pytest.mark.parametrize("mda", (TZ_UNAWARE_METADATA, TZ_AWARE_METADATA)) +def test_metadata(mda): + """Test metadata encoding/decoding.""" + metadata = copy.copy(mda) + msg = Message.decode(Message("/sat/polar/smb/level1", "file", + data=metadata).encode()) + + assert msg.data == metadata, "Messaging, metadata decoding / encoding failed" + + +@pytest.mark.parametrize(("mda", "compare_file"), + ((TZ_UNAWARE_METADATA, "/message_metadata_unaware.dumps"), + ((TZ_AWARE_METADATA, "/message_metadata_aware.dumps")))) +def test_serialization(mda, compare_file): + """Test json serialization.""" + import json + metadata = copy.copy(mda) + metadata["timestamp"] = metadata["timestamp"].isoformat() + with open(DATADIR + compare_file) as fp_: dump = fp_.read() - fp_.close() - local_dump = json.dumps(metadata) + local_dump = json.dumps(metadata) - msg = json.loads(dump) - for key, val in msg.items(): - assert val == metadata.get(key) + msg = json.loads(dump) + for key, val in msg.items(): + assert val == metadata.get(key) - msg = json.loads(local_dump) - for key, val in msg.items(): - assert val == metadata.get(key) + msg = json.loads(local_dump) + for key, val in msg.items(): + assert val == metadata.get(key) diff --git a/posttroll/tests/test_nameserver.py b/posttroll/tests/test_nameserver.py index f4fe81d..5312db8 100644 --- a/posttroll/tests/test_nameserver.py +++ b/posttroll/tests/test_nameserver.py @@ -1,10 +1,10 @@ """Tests for communication involving the nameserver for service discovery.""" +import datetime as dt import os import time import unittest from contextlib import contextmanager -from datetime import timedelta from threading import Thread from unittest import mock @@ -49,7 +49,7 @@ def create_nameserver_instance(max_age=3, multicast_enabled=True): """Create a nameserver instance.""" config.set(nameserver_port=free_port()) config.set(address_publish_port=free_port()) - ns = NameServer(max_age=timedelta(seconds=max_age), multicast_enabled=multicast_enabled) + ns = NameServer(max_age=dt.timedelta(seconds=max_age), multicast_enabled=multicast_enabled) thr = Thread(target=ns.run) thr.start()