From d14d3cc3fea8de4f03cf36949085484800f45f3a Mon Sep 17 00:00:00 2001 From: Oleksiy Protas Date: Thu, 4 May 2017 23:25:50 +0300 Subject: [PATCH 01/10] projects updated to API scheme --- backend/promis/backend_api/serializer.py | 18 ++++++++++-------- backend/promis/backend_api/views.py | 1 + backend/promis/util/parsers.py | 10 +++++++--- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/backend/promis/backend_api/serializer.py b/backend/promis/backend_api/serializer.py index 1ff6505..e4ae64a 100644 --- a/backend/promis/backend_api/serializer.py +++ b/backend/promis/backend_api/serializer.py @@ -4,7 +4,7 @@ from rest_framework.fields import ReadOnlyField, SerializerMethodField from rest_framework.reverse import reverse from djsw_wrapper.serializers import SwaggerHyperlinkedRelatedField -from hvad.contrib.restframework import TranslatableModelSerializer +from hvad.contrib.restframework import TranslatableModelSerializer, HyperlinkedTranslatableModelSerializer from rest_framework_gis.serializers import GeoModelSerializer from django.contrib.gis.geos import GEOSGeometry, GEOSException import json @@ -14,6 +14,7 @@ from backend_api import helpers import util.parsers +# TODO: can we do the extra_kwargs thing globally? class SessionsSerializer(serializers.ModelSerializer): # TODO: Spike! @lyssdod, correct this @@ -73,19 +74,20 @@ class Meta: model = models.Session fields = ('time',) -class SpaceProjectsSerializer(TranslatableModelSerializer): +class SpaceProjectsSerializer(HyperlinkedTranslatableModelSerializer): timelapse = serializers.SerializerMethodField() def get_timelapse(self, obj): - ret_val = {} - ret_val['begin'] = str(obj.date_start.isoformat()) - ret_val['end'] = str(obj.date_end.isoformat()) - - return ret_val + # TODO: start and end are a DATE not DATETIME, but we convert them implicitly + return { 'start': util.parsers.datetime_to_utc(obj.date_start), + 'end': util.parsers.datetime_to_utc(obj.date_end) } class Meta: model = models.Space_project - fields = ('id', 'name', 'description', 'timelapse') + fields = ('id', 'url', 'name', 'description', 'timelapse') + extra_kwargs = { + 'url': { 'lookup_field': 'id' } + } class ChannelsSerializer(TranslatableModelSerializer): class Meta: diff --git a/backend/promis/backend_api/views.py b/backend/promis/backend_api/views.py index 4658360..d3ca92e 100644 --- a/backend/promis/backend_api/views.py +++ b/backend/promis/backend_api/views.py @@ -51,6 +51,7 @@ class Meta: class ProjectsView(viewsets.ReadOnlyModelViewSet): queryset = models.Space_project.objects.all() serializer_class = serializer.SpaceProjectsSerializer + lookup_field = 'id' permission_classes = (AllowAny,) class DevicesView(viewsets.ReadOnlyModelViewSet): diff --git a/backend/promis/util/parsers.py b/backend/promis/util/parsers.py index 10c67ad..aa101c5 100644 --- a/backend/promis/util/parsers.py +++ b/backend/promis/util/parsers.py @@ -25,11 +25,15 @@ import datetime import pytz +def datetime_to_utc(x): + # Add the 00:00 time if we only got a date + if type(x) is datetime.date: + x = datetime.datetime.combine(x, datetime.datetime.min.time()) + return int(x.replace(tzinfo=pytz.timezone("UTC")).timestamp()) + def str_to_utc(x): time_fmt = "%Y{0}%m{0}%d %H:%M:%S".format(x[4]) - ts = datetime.datetime.strptime(x, time_fmt) - ts = ts.replace(tzinfo=pytz.timezone("UTC")).timestamp() - return int(ts) + return datetime_to_utc(datetime.datetime.strptime(x, time_fmt)) # TODO: cull out those which have standard parsers (CSV?) # TODO: replace ValueErrors with meaningful exception classes when integrating From a7029b78600a80772c1e4f15aeee49979b037ddd Mon Sep 17 00:00:00 2001 From: Oleksiy Protas Date: Fri, 5 May 2017 00:42:31 +0300 Subject: [PATCH 02/10] devices and channels updated to API scheme --- .../backend_api/fixtures/init_data.json | 6 +- backend/promis/backend_api/models.py | 2 +- backend/promis/backend_api/serializer.py | 98 ++++++++++--------- backend/promis/backend_api/views.py | 20 ++-- doc/promis_api.yaml | 17 ++-- test/data/test_set.json | 2 +- 6 files changed, 79 insertions(+), 66 deletions(-) diff --git a/backend/promis/backend_api/fixtures/init_data.json b/backend/promis/backend_api/fixtures/init_data.json index d53fe92..c809b93 100644 --- a/backend/promis/backend_api/fixtures/init_data.json +++ b/backend/promis/backend_api/fixtures/init_data.json @@ -92,21 +92,21 @@ "model": "backend_api.device", "pk": 1, "fields": { - "satellite": 1 + "space_project": 1 } }, { "model": "backend_api.device", "pk": 2, "fields": { - "satellite": 1 + "space_project": 1 } }, { "model": "backend_api.device", "pk": 3, "fields": { - "satellite": 1 + "space_project": 1 } }, { diff --git a/backend/promis/backend_api/models.py b/backend/promis/backend_api/models.py index edba336..afcae88 100644 --- a/backend/promis/backend_api/models.py +++ b/backend/promis/backend_api/models.py @@ -78,7 +78,7 @@ def __str__(self): class Device(TranslatableModel): - satellite = ForeignKey('Space_project') + space_project = ForeignKey('Space_project') translations = TranslatedFields( name = TextField(), diff --git a/backend/promis/backend_api/serializer.py b/backend/promis/backend_api/serializer.py index e4ae64a..b775098 100644 --- a/backend/promis/backend_api/serializer.py +++ b/backend/promis/backend_api/serializer.py @@ -14,7 +14,60 @@ from backend_api import helpers import util.parsers -# TODO: can we do the extra_kwargs thing globally? + +class LookupById: + '''Shortcut to include extra_kwargs to every Meta class''' + extra_kwargs = { 'url': { 'lookup_field': 'id' } } + +# TODO: can we have smth like this? +#class IdAndURLSerializer(serializers.HyperlinkedModelSerializer): + #'''Serializes anything to a id/url pair''' + #class Meta(LookupById): + #model = AbstractModel + #fields = ('id', 'url') + + +class SpaceProjectsSerializer(HyperlinkedTranslatableModelSerializer): + timelapse = serializers.SerializerMethodField() + + def get_timelapse(self, obj): + # TODO: start and end are a DATE not DATETIME, but we convert them implicitly + return { 'start': util.parsers.datetime_to_utc(obj.date_start), + 'end': util.parsers.datetime_to_utc(obj.date_end) } + + class Meta(LookupById): + model = models.Space_project + fields = ('id', 'url', 'name', 'description', 'timelapse') + + # TODO: STUB + def __init__(self, *args, idurl=False, **kwargs): + super().__init__(*args, **kwargs) + if idurl: + for f in ('name', 'description', 'timelapse'): + self.fields.pop(f) + + +class ChannelsSerializer(HyperlinkedTranslatableModelSerializer): + class Meta(LookupById): + fields = ('id', 'url', 'name', 'description') + model = models.Channel + + # TODO: STUB + def __init__(self, *args, idurl=False, **kwargs): + super().__init__(*args, **kwargs) + if idurl: + for f in ('name', 'description'): + self.fields.pop(f) + + +class DevicesSerializer(TranslatableModelSerializer): + space_project = SpaceProjectsSerializer(many = False, idurl = True) + channels = ChannelsSerializer(many = True, idurl = True) + + class Meta: + model = models.Device + fields = ('id', 'name', 'description', 'space_project', 'channels') + class SessionsSerializer(serializers.ModelSerializer): # TODO: Spike! @lyssdod, correct this @@ -74,49 +127,6 @@ class Meta: model = models.Session fields = ('time',) -class SpaceProjectsSerializer(HyperlinkedTranslatableModelSerializer): - timelapse = serializers.SerializerMethodField() - - def get_timelapse(self, obj): - # TODO: start and end are a DATE not DATETIME, but we convert them implicitly - return { 'start': util.parsers.datetime_to_utc(obj.date_start), - 'end': util.parsers.datetime_to_utc(obj.date_end) } - - class Meta: - model = models.Space_project - fields = ('id', 'url', 'name', 'description', 'timelapse') - extra_kwargs = { - 'url': { 'lookup_field': 'id' } - } - -class ChannelsSerializer(TranslatableModelSerializer): - class Meta: - fields = ('id', 'name', 'description',) - model = models.Channel - -class DevicesSerializer(TranslatableModelSerializer): - satellite = SpaceProjectsSerializer(many = False) - channels = ChannelsSerializer(many = True) - - class Meta: - model = models.Device - fields = ('id', 'name', 'description', 'satellite', 'channels') - -class FunctionsSerializer(TranslatableModelSerializer): - class Meta: - fields = ('__all__') - model = models.Function - -class UnitsSerializer(TranslatableModelSerializer): - class Meta: - fields = ('__all__') - model = models.Unit - -class ValuesSerializer(TranslatableModelSerializer): - class Meta: - fields = ('__all__') - model = models.Value - class ParametersSerializer(TranslatableModelSerializer): # TODO: fix the bug # channel = HyperlinkedRelatedField(many = False, view_name = 'channel-detail', read_only = True) diff --git a/backend/promis/backend_api/views.py b/backend/promis/backend_api/views.py index d3ca92e..a210038 100644 --- a/backend/promis/backend_api/views.py +++ b/backend/promis/backend_api/views.py @@ -30,6 +30,13 @@ import datetime from rest_framework.decorators import permission_classes +class PromisViewSet(viewsets.ReadOnlyModelViewSet): + '''Collects most commonly used View stuff''' + lookup_field = 'id' + permission_classes = (AllowAny,) + # TODO: Is it okay to put it here even for classes that don't need it? + filter_backends = (DjangoFilterBackend,) + class SessionFilter(django_filters.rest_framework.FilterSet): time_begin = django_filters.IsoDateTimeFilter(lookup_expr='gte') time_end = django_filters.IsoDateTimeFilter(lookup_expr='lte') @@ -48,23 +55,18 @@ class Meta: model = models.Measurement fields = ['session', 'parameter'] -class ProjectsView(viewsets.ReadOnlyModelViewSet): +class ProjectsView(PromisViewSet): queryset = models.Space_project.objects.all() serializer_class = serializer.SpaceProjectsSerializer - lookup_field = 'id' - permission_classes = (AllowAny,) -class DevicesView(viewsets.ReadOnlyModelViewSet): +class DevicesView(PromisViewSet): queryset = models.Device.objects.all() serializer_class = serializer.DevicesSerializer - filter_backends = (DjangoFilterBackend,) - filter_fields = ('satellite',) - permission_classes = (AllowAny,) + filter_fields = ('space_project',) -class ChannelsView(viewsets.ReadOnlyModelViewSet): +class ChannelsView(PromisViewSet): queryset = models.Channel.objects.all() serializer_class = serializer.ChannelsSerializer - permission_classes = (PromisPermission,) class SessionsView(viewsets.ReadOnlyModelViewSet): queryset = models.Session.objects.all() diff --git a/doc/promis_api.yaml b/doc/promis_api.yaml index 2120211..835942a 100644 --- a/doc/promis_api.yaml +++ b/doc/promis_api.yaml @@ -527,7 +527,7 @@ definitions: name: type: string description: Short name - desc: + description: type: string description: In-depth description #allOf: @@ -550,13 +550,15 @@ definitions: name: type: string description: Short name - desc: + description: type: string description: In-depth description #allOf: #- $ref: '#/definitions/IdURL' #- $ref: '#/definitions/NameDesc' #- properties: + space_project: + $ref: '#/definitions/IdURL' channels: type: array description: Channels the device provides @@ -576,7 +578,7 @@ definitions: name: type: string description: Short name - desc: + description: type: string description: In-depth description #allOf: @@ -596,7 +598,7 @@ definitions: name: type: string description: Short name - desc: + description: type: string description: In-depth description #allOf: @@ -637,9 +639,8 @@ definitions: #$ref: '#/definitions/IdURL' #timelapse: #$ref: '#/definitions/Time' - project: - type: integer - description: Related project ID + space_project: + $ref: '#/definitions/IdURL' orbit: type: number description: Orbit representation @@ -798,7 +799,7 @@ definitions: name: type: string description: Short name - desc: + description: type: string description: In-depth description diff --git a/test/data/test_set.json b/test/data/test_set.json index 32bef1c..7cada68 100644 --- a/test/data/test_set.json +++ b/test/data/test_set.json @@ -108,7 +108,7 @@ "model": "backend_api.device", "pk": 42000, "fields": { - "satellite": 42001 + "space_project": 42001 } }, { From 9837fcf63c85348bf965e5a577c3f8ed18ed84cb Mon Sep 17 00:00:00 2001 From: Oleksiy Protas Date: Fri, 5 May 2017 00:47:29 +0300 Subject: [PATCH 03/10] parameters updated to API scheme --- backend/promis/backend_api/serializer.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/backend/promis/backend_api/serializer.py b/backend/promis/backend_api/serializer.py index b775098..41a97a4 100644 --- a/backend/promis/backend_api/serializer.py +++ b/backend/promis/backend_api/serializer.py @@ -60,6 +60,21 @@ def __init__(self, *args, idurl=False, **kwargs): self.fields.pop(f) +class ParametersSerializer(TranslatableModelSerializer): + channel = ChannelsSerializer(idurl = True) + + class Meta(LookupById): + fields = ('id', 'url', 'name', 'description', 'channel') + model = models.Parameter + + # TODO: STUB + def __init__(self, *args, idurl=False, **kwargs): + super().__init__(*args, **kwargs) + if idurl: + for f in ('name', 'description', 'channel'): + self.fields.pop(f) + + class DevicesSerializer(TranslatableModelSerializer): space_project = SpaceProjectsSerializer(many = False, idurl = True) channels = ChannelsSerializer(many = True, idurl = True) @@ -127,14 +142,6 @@ class Meta: model = models.Session fields = ('time',) -class ParametersSerializer(TranslatableModelSerializer): -# TODO: fix the bug -# channel = HyperlinkedRelatedField(many = False, view_name = 'channel-detail', read_only = True) - channel = ChannelsSerializer() - class Meta: - fields = ('id', 'name', 'description', 'channel') - model = models.Parameter - '''class QuicklookHyperlink(serializers.HyperlinkedRelatedField): view_name = 'document-detail' From 4a848dfbc657acd3d683dc595223488d07930f89 Mon Sep 17 00:00:00 2001 From: Oleksiy Protas Date: Fri, 5 May 2017 13:35:14 +0300 Subject: [PATCH 04/10] Some polishing and stub for #195 --- backend/promis/backend_api/serializer.py | 53 ++++++++++++------------ backend/promis/backend_api/views.py | 19 +++++---- 2 files changed, 37 insertions(+), 35 deletions(-) diff --git a/backend/promis/backend_api/serializer.py b/backend/promis/backend_api/serializer.py index 41a97a4..87ac6f6 100644 --- a/backend/promis/backend_api/serializer.py +++ b/backend/promis/backend_api/serializer.py @@ -19,13 +19,15 @@ class LookupById: '''Shortcut to include extra_kwargs to every Meta class''' extra_kwargs = { 'url': { 'lookup_field': 'id' } } -# TODO: can we have smth like this? -#class IdAndURLSerializer(serializers.HyperlinkedModelSerializer): - #'''Serializes anything to a id/url pair''' - #class Meta(LookupById): - #model = AbstractModel - #fields = ('id', 'url') - +# TODO: STUB, please implement a fitting serializer in Swagger so we can just SwaggerHyperlinkedRelatedField all over +def _remove_extra_fields(obj, inline): + print("in extra fields") + if not inline: + fields = tuple(obj.fields.keys()) + print(fields) + for f in fields: + if f not in ('id', 'url'): + obj.fields.pop(f) class SpaceProjectsSerializer(HyperlinkedTranslatableModelSerializer): timelapse = serializers.SerializerMethodField() @@ -40,11 +42,9 @@ class Meta(LookupById): fields = ('id', 'url', 'name', 'description', 'timelapse') # TODO: STUB - def __init__(self, *args, idurl=False, **kwargs): + def __init__(self, *args, inline = True, **kwargs): super().__init__(*args, **kwargs) - if idurl: - for f in ('name', 'description', 'timelapse'): - self.fields.pop(f) + _remove_extra_fields(self, inline) class ChannelsSerializer(HyperlinkedTranslatableModelSerializer): @@ -53,35 +53,36 @@ class Meta(LookupById): model = models.Channel # TODO: STUB - def __init__(self, *args, idurl=False, **kwargs): + def __init__(self, *args, inline = True, **kwargs): super().__init__(*args, **kwargs) - if idurl: - for f in ('name', 'description'): - self.fields.pop(f) + _remove_extra_fields(self, inline) -class ParametersSerializer(TranslatableModelSerializer): - channel = ChannelsSerializer(idurl = True) +class ParametersSerializer(HyperlinkedTranslatableModelSerializer): + channel = ChannelsSerializer(inline = False) class Meta(LookupById): fields = ('id', 'url', 'name', 'description', 'channel') model = models.Parameter # TODO: STUB - def __init__(self, *args, idurl=False, **kwargs): + def __init__(self, *args, inline = True, **kwargs): super().__init__(*args, **kwargs) - if idurl: - for f in ('name', 'description', 'channel'): - self.fields.pop(f) + _remove_extra_fields(self, inline) -class DevicesSerializer(TranslatableModelSerializer): - space_project = SpaceProjectsSerializer(many = False, idurl = True) - channels = ChannelsSerializer(many = True, idurl = True) +class DevicesSerializer(HyperlinkedTranslatableModelSerializer): + space_project = SpaceProjectsSerializer(many = False, inline = False) + channels = ChannelsSerializer(many = True, inline = False) - class Meta: + class Meta(LookupById): model = models.Device - fields = ('id', 'name', 'description', 'space_project', 'channels') + fields = ('id', 'url', 'name', 'description', 'space_project', 'channels') + + # TODO: STUB + def __init__(self, *args, inline = True, **kwargs): + super().__init__(*args, **kwargs) + _remove_extra_fields(self, inline) class SessionsSerializer(serializers.ModelSerializer): diff --git a/backend/promis/backend_api/views.py b/backend/promis/backend_api/views.py index a210038..c9f08f2 100644 --- a/backend/promis/backend_api/views.py +++ b/backend/promis/backend_api/views.py @@ -59,14 +59,20 @@ class ProjectsView(PromisViewSet): queryset = models.Space_project.objects.all() serializer_class = serializer.SpaceProjectsSerializer +class ChannelsView(PromisViewSet): + queryset = models.Channel.objects.all() + serializer_class = serializer.ChannelsSerializer + +class ParametersView(PromisViewSet): + queryset = models.Parameter.objects.all() + serializer_class = serializer.ParametersSerializer + filter_fields = ('channel',) + class DevicesView(PromisViewSet): queryset = models.Device.objects.all() serializer_class = serializer.DevicesSerializer filter_fields = ('space_project',) -class ChannelsView(PromisViewSet): - queryset = models.Channel.objects.all() - serializer_class = serializer.ChannelsSerializer class SessionsView(viewsets.ReadOnlyModelViewSet): queryset = models.Session.objects.all() @@ -121,12 +127,7 @@ def get_queryset(self): return queryset -class ParametersView(viewsets.ReadOnlyModelViewSet): - queryset = models.Parameter.objects.all() - serializer_class = serializer.ParametersSerializer - filter_backends = (DjangoFilterBackend,) - filter_fields = ('channel',) - permission_classes = (AllowAny,) + class MeasurementsView(viewsets.ReadOnlyModelViewSet): queryset = models.Measurement.objects.all() From 07b2e74d66c47ea4aeb39e970bbde16b350d25d3 Mon Sep 17 00:00:00 2001 From: Oleksiy Protas Date: Fri, 5 May 2017 13:57:48 +0300 Subject: [PATCH 05/10] id/url pairs removed in favour of normal links closes #195 as unnecessary --- backend/promis/backend_api/serializer.py | 35 ++--------------- doc/promis_api.yaml | 48 ++++++++++++++---------- 2 files changed, 31 insertions(+), 52 deletions(-) diff --git a/backend/promis/backend_api/serializer.py b/backend/promis/backend_api/serializer.py index 87ac6f6..efaae3a 100644 --- a/backend/promis/backend_api/serializer.py +++ b/backend/promis/backend_api/serializer.py @@ -19,15 +19,6 @@ class LookupById: '''Shortcut to include extra_kwargs to every Meta class''' extra_kwargs = { 'url': { 'lookup_field': 'id' } } -# TODO: STUB, please implement a fitting serializer in Swagger so we can just SwaggerHyperlinkedRelatedField all over -def _remove_extra_fields(obj, inline): - print("in extra fields") - if not inline: - fields = tuple(obj.fields.keys()) - print(fields) - for f in fields: - if f not in ('id', 'url'): - obj.fields.pop(f) class SpaceProjectsSerializer(HyperlinkedTranslatableModelSerializer): timelapse = serializers.SerializerMethodField() @@ -41,49 +32,29 @@ class Meta(LookupById): model = models.Space_project fields = ('id', 'url', 'name', 'description', 'timelapse') - # TODO: STUB - def __init__(self, *args, inline = True, **kwargs): - super().__init__(*args, **kwargs) - _remove_extra_fields(self, inline) - class ChannelsSerializer(HyperlinkedTranslatableModelSerializer): class Meta(LookupById): fields = ('id', 'url', 'name', 'description') model = models.Channel - # TODO: STUB - def __init__(self, *args, inline = True, **kwargs): - super().__init__(*args, **kwargs) - _remove_extra_fields(self, inline) - class ParametersSerializer(HyperlinkedTranslatableModelSerializer): - channel = ChannelsSerializer(inline = False) + channel = SwaggerHyperlinkedRelatedField(many = False, read_only = True, view_name = 'channel-detail') class Meta(LookupById): fields = ('id', 'url', 'name', 'description', 'channel') model = models.Parameter - # TODO: STUB - def __init__(self, *args, inline = True, **kwargs): - super().__init__(*args, **kwargs) - _remove_extra_fields(self, inline) - class DevicesSerializer(HyperlinkedTranslatableModelSerializer): - space_project = SpaceProjectsSerializer(many = False, inline = False) - channels = ChannelsSerializer(many = True, inline = False) + space_project = SwaggerHyperlinkedRelatedField(many = False, read_only = True, view_name = 'space_project-detail') + channels = SwaggerHyperlinkedRelatedField(many = True, read_only = True, view_name = 'channel-detail') class Meta(LookupById): model = models.Device fields = ('id', 'url', 'name', 'description', 'space_project', 'channels') - # TODO: STUB - def __init__(self, *args, inline = True, **kwargs): - super().__init__(*args, **kwargs) - _remove_extra_fields(self, inline) - class SessionsSerializer(serializers.ModelSerializer): # TODO: Spike! @lyssdod, correct this diff --git a/doc/promis_api.yaml b/doc/promis_api.yaml index 835942a..41f81af 100644 --- a/doc/promis_api.yaml +++ b/doc/promis_api.yaml @@ -558,12 +558,14 @@ definitions: #- $ref: '#/definitions/NameDesc' #- properties: space_project: - $ref: '#/definitions/IdURL' + type: string + description: API Link to the related Space Project channels: type: array description: Channels the device provides items: - $ref: '#/definitions/IdURL' + type: string + description: API Link to the channel provided Channel: type: object @@ -609,7 +611,8 @@ definitions: #allOf: #- description: Channel the parameter depends on #- $ref: '#/definitions/IdURL' - $ref: '#/definitions/IdURL' + type: string + description: API Link to the related channel Session: type: object @@ -640,7 +643,8 @@ definitions: #timelapse: #$ref: '#/definitions/Time' space_project: - $ref: '#/definitions/IdURL' + type: string + description: API Link to the related Space Project orbit: type: number description: Orbit representation @@ -650,7 +654,8 @@ definitions: type: array # api links to measurement instances description: Available measurements for this session items: - $ref: '#/definitions/IdURL' + type: string + description: API Link to the measurement during the session timelapse: $ref: '#/definitions/Time' @@ -694,30 +699,33 @@ definitions: #type: string #description: URL to parameter data download session: - $ref: '#/definitions/IdURL' + type: string + description: API Link to the related session channel: - $ref: '#/definitions/IdURL' + type: string + description: API Link to the measured channel parameter: - $ref: '#/definitions/IdURL' + type: string + description: API Link to the measured parameter freq: - type: number - description: Sampling frequency + type: number + description: Sampling frequency min_freq: - type: number + type: number max_freq: - type: number + type: number channel_quicklook: - type: string - description: URL to channel quicklook + type: string + description: URL to channel quicklook channel_download: - type: string - description: URL to channel data download + type: string + description: URL to channel data download parameter_quicklook: - type: string - description: URL to parameter quicklook + type: string + description: URL to parameter quicklook parameter_download: - type: string - description: URL to parameter data download + type: string + description: URL to parameter data download Quicklook: type: object From 2de091538f54a05401c94ea2351a890a661e28de Mon Sep 17 00:00:00 2001 From: Oleksiy Protas Date: Fri, 5 May 2017 15:05:55 +0300 Subject: [PATCH 06/10] Some session code cleanup. Related to #179 --- backend/promis/backend_api/serializer.py | 44 ++++++++++-------------- doc/promis_api.yaml | 25 ++++++++++++++ 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/backend/promis/backend_api/serializer.py b/backend/promis/backend_api/serializer.py index efaae3a..7aed024 100644 --- a/backend/promis/backend_api/serializer.py +++ b/backend/promis/backend_api/serializer.py @@ -20,7 +20,7 @@ class LookupById: extra_kwargs = { 'url': { 'lookup_field': 'id' } } -class SpaceProjectsSerializer(HyperlinkedTranslatableModelSerializer): +class SpaceProjectsSerializer(TranslatableModelSerializer): timelapse = serializers.SerializerMethodField() def get_timelapse(self, obj): @@ -33,13 +33,13 @@ class Meta(LookupById): fields = ('id', 'url', 'name', 'description', 'timelapse') -class ChannelsSerializer(HyperlinkedTranslatableModelSerializer): +class ChannelsSerializer(TranslatableModelSerializer): class Meta(LookupById): fields = ('id', 'url', 'name', 'description') model = models.Channel -class ParametersSerializer(HyperlinkedTranslatableModelSerializer): +class ParametersSerializer(TranslatableModelSerializer): channel = SwaggerHyperlinkedRelatedField(many = False, read_only = True, view_name = 'channel-detail') class Meta(LookupById): @@ -47,7 +47,7 @@ class Meta(LookupById): model = models.Parameter -class DevicesSerializer(HyperlinkedTranslatableModelSerializer): +class DevicesSerializer(TranslatableModelSerializer): space_project = SwaggerHyperlinkedRelatedField(many = False, read_only = True, view_name = 'space_project-detail') channels = SwaggerHyperlinkedRelatedField(many = True, read_only = True, view_name = 'channel-detail') @@ -57,24 +57,13 @@ class Meta(LookupById): class SessionsSerializer(serializers.ModelSerializer): -# TODO: Spike! @lyssdod, correct this -# measurements = SwaggerHyperlinkedRelatedField(many = True, view_name = 'measurement-detail', read_only = True) - measurements = SerializerMethodField() + # TODO: wtf? /quicklook/ here + measurements = SwaggerHyperlinkedRelatedField(many = True, read_only = True, view_name = 'measurement-detail') + space_project = SwaggerHyperlinkedRelatedField(many = False, read_only = True, view_name = 'space_project-detail') geo_line = serializers.SerializerMethodField() time = serializers.SerializerMethodField() -# TODO: Spike! @lyssdod, correct this - def get_measurements(self, obj): - meas = models.Measurement.objects.filter(session = obj) - #TODO: SPIKE: remove below hard code and replace to related view path. - ret_val = [] - for m in meas: - ret_val.append(self.context['request'].build_absolute_uri('/en/api/measurements/' + str(m.id))) - - return ret_val - - def get_geo_line(self, obj): # Just in case for the future #return obj.geo_line.wkb.hex() @@ -83,17 +72,20 @@ def get_geo_line(self, obj): return util.parsers.wkb(obj.geo_line.wkb) # <- Generator def get_time(self, obj): - ret_val = {} - ret_val['begin'] = str(obj.time_begin.isoformat()) - ret_val['end'] = str(obj.time_end.isoformat()) - - return ret_val + # TODO: change to time_start in model for consistency + return { 'start': util.parsers.datetime_to_utc(obj.time_begin), + 'end': util.parsers.datetime_to_utc(obj.time_end) } - class Meta: + class Meta(LookupById): model = models.Session - fields = ('id', 'space_project', 'orbit_code', 'geo_line', 'time', 'measurements') - geo_field = 'geo_line' + fields = ('id', 'url', 'space_project', 'orbit_code', 'geo_line', 'time', 'measurements') + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # If we are serializing a list of sessions, don't include the geo_line + if type(args[0]) is list: + self.fields.pop('geo_line') # TODO: merge with the class above class CompactSessionsSerializer(serializers.ModelSerializer): diff --git a/doc/promis_api.yaml b/doc/promis_api.yaml index 41f81af..8cf2393 100644 --- a/doc/promis_api.yaml +++ b/doc/promis_api.yaml @@ -659,6 +659,31 @@ definitions: timelapse: $ref: '#/definitions/Time' + # merge with Session using allOf when #193 is resolved + SessionCompact: + type: object + properties: + id: + type: number + description: Unique numberic identifier + url: + type: string + description: URL location within API + space_project: + type: string + description: API Link to the related Space Project + orbit: + type: number + description: Orbit representation + measurements: + type: array # api links to measurement instances + description: Available measurements for this session + items: + type: string + description: API Link to the measurement during the session + timelapse: + $ref: '#/definitions/Time' + Measurement: type: object From 59fe0d8896b59d40e081a77e9449b7deec47d8c5 Mon Sep 17 00:00:00 2001 From: Oleksiy Protas Date: Sat, 6 May 2017 01:09:05 +0300 Subject: [PATCH 07/10] Stub code restored, see #196 for details --- backend/promis/backend_api/serializer.py | 11 +++++++++-- backend/promis/backend_api/urls.py | 17 ----------------- doc/promis_api.yaml | 2 +- 3 files changed, 10 insertions(+), 20 deletions(-) diff --git a/backend/promis/backend_api/serializer.py b/backend/promis/backend_api/serializer.py index 7aed024..c2a8f29 100644 --- a/backend/promis/backend_api/serializer.py +++ b/backend/promis/backend_api/serializer.py @@ -57,8 +57,15 @@ class Meta(LookupById): class SessionsSerializer(serializers.ModelSerializer): - # TODO: wtf? /quicklook/ here - measurements = SwaggerHyperlinkedRelatedField(many = True, read_only = True, view_name = 'measurement-detail') + # TODO: STUB, see #196 + #measurements = SwaggerHyperlinkedRelatedField(many = True, read_only = True, view_name = 'measurement-detail') + # cut from here: + measurements = SerializerMethodField() + def get_measurements(self, obj): + return (self.context['request'].build_absolute_uri('/api/measurements/' + str(m.id)) + for m in models.Measurement.objects.filter(session = obj)) + # to here ^ + space_project = SwaggerHyperlinkedRelatedField(many = False, read_only = True, view_name = 'space_project-detail') geo_line = serializers.SerializerMethodField() diff --git a/backend/promis/backend_api/urls.py b/backend/promis/backend_api/urls.py index baddc12..829d387 100644 --- a/backend/promis/backend_api/urls.py +++ b/backend/promis/backend_api/urls.py @@ -15,25 +15,8 @@ docrout.register(r'api/quicklook', views.QuicklookView) docrout.register(r'api/download', views.DownloadData) -''' -#== db view == -#TODO: This is used only for debugging, and should be removed -#TODO: Remove it - -dbview = SimpleRouter() -dbview.register(r'functions', views.FunctionsView) -dbview.register(r'documents', views.DocumentsView) -dbview.register(r'parameters', views.ParametersView) - -#============= -''' urlpatterns = router.urls + userreg.urls + [ url('^api-auth/', include('rest_framework.urls', namespace = 'rest_framework')), url(r'^user/update/$', views.UserUpdate), ] + docrout.urls - - -# + dbview.urls - -#print(urlpatterns) diff --git a/doc/promis_api.yaml b/doc/promis_api.yaml index 8cf2393..12267b0 100644 --- a/doc/promis_api.yaml +++ b/doc/promis_api.yaml @@ -217,7 +217,7 @@ paths: schema: type: array items: - $ref: '#/definitions/IdURL' + $ref: '#/definitions/SessionCompact' default: description: Unexpected error schema: From 0809e84ed74fdbb58dd8ff789a520db9d05e1ac2 Mon Sep 17 00:00:00 2001 From: Oleksiy Protas Date: Sat, 6 May 2017 01:28:55 +0300 Subject: [PATCH 08/10] measurements sort of updated to the new API, but some blockers persist --- backend/promis/backend_api/serializer.py | 101 +++++------------------ backend/promis/backend_api/views.py | 31 +------ doc/promis_api.yaml | 6 +- test/tests_backend/test_auth.py | 12 +-- 4 files changed, 33 insertions(+), 117 deletions(-) diff --git a/backend/promis/backend_api/serializer.py b/backend/promis/backend_api/serializer.py index c2a8f29..fc194ae 100644 --- a/backend/promis/backend_api/serializer.py +++ b/backend/promis/backend_api/serializer.py @@ -94,34 +94,6 @@ def __init__(self, *args, **kwargs): if type(args[0]) is list: self.fields.pop('geo_line') -# TODO: merge with the class above -class CompactSessionsSerializer(serializers.ModelSerializer): - time = serializers.SerializerMethodField() - - def get_geo_line(self, obj): - return util.parsers.wkb(obj.geo_line.wkb) - def get_time(self, obj): - return { 'begin': obj.time_begin.isoformat(), - 'end': obj.time_end.isoformat() } - - def __init__(self, *args, need_geo_line=True, **kwargs): - super().__init__(*args, **kwargs) - if need_geo_line: - self.fields.update({ "geo_line": serializers.SerializerMethodField() }) - - class Meta: - model = models.Session - fields = ('time',) - - -'''class QuicklookHyperlink(serializers.HyperlinkedRelatedField): - view_name = 'document-detail' - read_only = True - - queryset = models.Document.objects.all() - - def get_object -''' def _context_function_call(self, *args): ''' @@ -189,72 +161,43 @@ class PlainTextRenderer: def render(self, data, media_type=None, renderer_context=None): return "\n".join(data) -#TODO: class below need some refactoring..... -class DownloadViewSerializer(serializers.ModelSerializer): - chn_quicklook = serializers.SerializerMethodField() - par_quicklook = serializers.SerializerMethodField() - - channel_doc = serializers.SerializerMethodField() - parameter_doc = serializers.SerializerMethodField() - - class Meta: - fields = ('chn_quicklook', 'par_quicklook', 'channel_doc', 'parameter_doc') - model = models.Measurement - - def get_chn_quicklook(self, obj): - id = obj.id - #TODO: SPIKE: remove below hard code and replace to related view path. - return self.context['request'].build_absolute_uri('/en/api/quicklook/' + str(id) + '/channel') - - def get_par_quicklook(self, obj): - id = obj.id - #TODO: SPIKE: remove below hard code and replace to related view path. - return self.context['request'].build_absolute_uri('/en/api/quicklook/' + str(id) + '/parameter') - - def get_channel_doc(self, obj): - id = obj.id - #TODO: SPIKE: remove below hard code and replace to related view path. - return self.context['request'].build_absolute_uri('/en/api/download/' + str(id) + '/channel') - - def get_parameter_doc(self, obj): - id = obj.id - #TODO: SPIKE: remove below hard code and replace to related view path. - return self.context['request'].build_absolute_uri('/en/api/download/' + str(id) + '/parameter') - - def __init__(self, *args, **kwargs): - - super().__init__(*args, **kwargs) - - user = self.context['request'].user - if not (helpers.UserInGroup(user, 'level1') or helpers.IsSuperUser(user)): - self.fields.pop('channel_doc') - self.fields.pop('chn_quicklook') class MeasurementsSerializer(serializers.ModelSerializer): session = SwaggerHyperlinkedRelatedField(many = False, view_name = 'session-detail', read_only = True) channel = SwaggerHyperlinkedRelatedField(many = False, view_name = 'channel-detail', read_only = True) parameter = SwaggerHyperlinkedRelatedField(many = False, view_name = 'parameter-detail', read_only = True) - data = serializers.SerializerMethodField() + channel_quicklook = serializers.SerializerMethodField() + channel_download = serializers.SerializerMethodField() + parameter_quicklook = serializers.SerializerMethodField() + parameter_download = serializers.SerializerMethodField() - class Meta: - fields = ('session', 'parameter', 'channel', 'sampling_frequency', 'min_frequency', 'max_frequency', 'data') + class Meta(LookupById): + # TODO: add 'url' here, currently it's broken, see #196 + fields = ('id', 'session', 'parameter', 'channel', 'sampling_frequency', 'min_frequency', 'max_frequency', 'channel_quicklook', 'channel_download', 'parameter_quicklook', 'parameter_download') model = models.Measurement - def get_data(self, obj): - id = obj.channel_doc.id - #TODO: SPIKE: remove below hard code and replace to related view path. - return self.context['request'].build_absolute_uri('/en/api/download/' + str(id)) + #TODO: SPIKE: remove below hard code and replace to related view path. + def construct_data_url(self, obj, source, action): + id = getattr(obj, source + "_doc").id + return self.context['request'].build_absolute_uri('/api/%s/%d/%s' % (action, id, source)) + + def get_channel_quicklook(self, obj): + return self.construct_data_url(obj, "channel", "quicklook") + def get_channel_download(self, obj): + return self.construct_data_url(obj, "channel", "download") + def get_parameter_quicklook(self, obj): + return self.construct_data_url(obj, "parameter", "quicklook") + def get_parameter_download(self, obj): + return self.construct_data_url(obj, "parameter", "download") + # cut here ^ def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - user = self.context['request'].user - if not (helpers.UserInGroup(user, 'level1') or helpers.IsSuperUser(user)): - self.fields.pop('channel') + self.fields.pop('channel_download') class UserSerializer(serializers.ModelSerializer): diff --git a/backend/promis/backend_api/views.py b/backend/promis/backend_api/views.py index c9f08f2..e2a61af 100644 --- a/backend/promis/backend_api/views.py +++ b/backend/promis/backend_api/views.py @@ -74,13 +74,11 @@ class DevicesView(PromisViewSet): filter_fields = ('space_project',) -class SessionsView(viewsets.ReadOnlyModelViewSet): +class SessionsView(PromisViewSet): queryset = models.Session.objects.all() serializer_class = serializer.SessionsSerializer - filter_backends = (DjangoFilterBackend,) filter_class = SessionFilter pagination_class = LimitOffsetPagination - permission_classes = (AllowAny,) def get_queryset(self): @@ -129,32 +127,11 @@ def get_queryset(self): -class MeasurementsView(viewsets.ReadOnlyModelViewSet): +class MeasurementsView(PromisViewSet): queryset = models.Measurement.objects.all() serializer_class = serializer.MeasurementsSerializer - permission_classes = (AllowAny,) filter_class = MeasurementsFilter - filter_backends = (DjangoFilterBackend,) - - -''' -#TODO: This is used only for debugging, and should be removed -#======== Added to view db contents. Remove it: ====== - -class DocumentsView(viewsets.ReadOnlyModelViewSet): - queryset = models.Document.objects.all() - serializer_class = serializer.DocumentsSerializer -class FunctionsView(viewsets.ReadOnlyModelViewSet): - queryset = models.Function.objects.all() - serializer_class = serializer.FunctionsSerializer - -class ParameterssView(viewsets.ReadOnlyModelViewSet): - queryset = models.Parameter.objects.all() - serializer_class = serializer.ParametersSerializer - -#===================================================== -''' class QuicklookView(RetrieveModelMixin, viewsets.GenericViewSet): def _quicklook(self, obj, src_name, src_serializer): @@ -218,10 +195,6 @@ def parameter(self, request, id): permission_classes = (AllowAny,) serializer_class = serializer.MeasurementsSerializer -class DownloadView(viewsets.ReadOnlyModelViewSet): - queryset = models.Measurement.objects.all() - permission_classes = (AllowAny,) - serializer_class = serializer.DownloadViewSerializer # TODO: make a common base class for this and QuicklookView from rest_framework.renderers import JSONRenderer, BrowsableAPIRenderer diff --git a/doc/promis_api.yaml b/doc/promis_api.yaml index 12267b0..c5e3e92 100644 --- a/doc/promis_api.yaml +++ b/doc/promis_api.yaml @@ -732,12 +732,12 @@ definitions: parameter: type: string description: API Link to the measured parameter - freq: + sampling_frequency: type: number description: Sampling frequency - min_freq: + min_frequency: type: number - max_freq: + max_frequency: type: number channel_quicklook: type: string diff --git a/test/tests_backend/test_auth.py b/test/tests_backend/test_auth.py index 3c879fd..1ca3763 100644 --- a/test/tests_backend/test_auth.py +++ b/test/tests_backend/test_auth.py @@ -101,18 +101,18 @@ def test_level1_listing(connie): # TODO: generalise the code def test_level2_access(melanie): '''Check that a level2 user can see only parameter fields''' - r = melanie.get("/en/api/download/1") + r = melanie.get("/en/api/measurements/1") assert r.status_code == 200, "Invalid status code" json_data = r.json() - assert "channel_doc" not in json_data, "Can see channels" - assert "parameter_doc" in json_data, "Can't see parameters" + assert "channel_download" not in json_data, "Can see channels" + assert "parameter_download" in json_data, "Can't see parameters" def test_level1_access(connie): '''Check that a level1 user can see all fields''' - r = connie.get("/en/api/download/1") + r = connie.get("/en/api/measurements/1") assert r.status_code == 200, "Invalid status code" json_data = r.json() - assert "channel_doc" in json_data, "Can't see channels" - assert "parameter_doc" in json_data, "Can't see parameters" + assert "channel_download" in json_data, "Can't see channels" + assert "parameter_download" in json_data, "Can't see parameters" From 7798d32ceddcfaa0a7b7048f6c4294f685dbdb62 Mon Sep 17 00:00:00 2001 From: Oleksiy Protas Date: Sat, 6 May 2017 02:06:49 +0300 Subject: [PATCH 09/10] Sessions are now filtered by UNIX time at UTC fixes #176 --- backend/promis/backend_api/views.py | 30 +++++++++++++---------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/backend/promis/backend_api/views.py b/backend/promis/backend_api/views.py index e2a61af..adaee32 100644 --- a/backend/promis/backend_api/views.py +++ b/backend/promis/backend_api/views.py @@ -26,6 +26,10 @@ from rest_framework.permissions import AllowAny, IsAuthenticated from rest_framework.exceptions import NotAuthenticated, NotFound, MethodNotAllowed +# TODO: also used by orbit.py, maybe make util.time +import datetime, pytz +def maketime(u): + return datetime.datetime.fromtimestamp(u, tz=pytz.utc) import datetime from rest_framework.decorators import permission_classes @@ -38,13 +42,20 @@ class PromisViewSet(viewsets.ReadOnlyModelViewSet): filter_backends = (DjangoFilterBackend,) class SessionFilter(django_filters.rest_framework.FilterSet): - time_begin = django_filters.IsoDateTimeFilter(lookup_expr='gte') - time_end = django_filters.IsoDateTimeFilter(lookup_expr='lte') + time_begin = django_filters.NumberFilter(method='unix_time_filter') + time_end = django_filters.NumberFilter(method='unix_time_filter') project = django_filters.ModelChoiceFilter(name='space_project', queryset = models.Space_project.objects.all()) satellite = django_filters.ModelChoiceFilter(name='space_project', queryset = models.Space_project.objects.all()) + # TODO: make a separate class? + def unix_time_filter(self, queryset, name, value): + # Composition of the queryset.filter argument depending on which field was used + filter_actions = { 'time_begin': 'time_begin__gte', 'time_end': 'time_end__lte' } + return queryset.filter(**{ filter_actions[name]: maketime(value) }) + + class Meta: model = models.Session fields = ['space_project', 'time_begin', 'time_end', 'project', 'satellite'] @@ -81,24 +92,9 @@ class SessionsView(PromisViewSet): pagination_class = LimitOffsetPagination def get_queryset(self): - queryset = models.Session.objects.all() polygon = self.request.query_params.get('polygon', None) -#commented to allow Anonymous access - ''' - user = self.request.user - if not helpers.UserExists(user): - return models.Session.objects.none() - - if helpers.UserGroupsNo(user) <= 0: - now = datetime.datetime.now() - half_year_ago = now - datetime.timedelta(183) - ago = datetime.date(1900, 1, 1) - queryset = models.Session.objects.filter(time_end__range = (ago, half_year_ago)) - else: - queryset = models.Session.objects.all() - ''' if polygon is not None: try: From 3471a1c6d5cd24fec981a2b9f8baea726f7176d7 Mon Sep 17 00:00:00 2001 From: Oleksiy Protas Date: Sat, 6 May 2017 02:14:53 +0300 Subject: [PATCH 10/10] Time functions in a separate module --- backend/promis/backend_api/serializer.py | 10 +++---- backend/promis/backend_api/views.py | 7 ++--- backend/promis/functions/potential.py | 6 ++-- backend/promis/functions/test_set.py | 6 ++-- backend/promis/util/orbit.py | 5 ---- backend/promis/util/parsers.py | 17 ++--------- backend/promis/util/unix_time.py | 37 ++++++++++++++++++++++++ 7 files changed, 53 insertions(+), 35 deletions(-) create mode 100644 backend/promis/util/unix_time.py diff --git a/backend/promis/backend_api/serializer.py b/backend/promis/backend_api/serializer.py index fc194ae..afde00c 100644 --- a/backend/promis/backend_api/serializer.py +++ b/backend/promis/backend_api/serializer.py @@ -13,7 +13,7 @@ from django.contrib.auth.models import Group from backend_api import helpers import util.parsers - +import util.unix_time class LookupById: '''Shortcut to include extra_kwargs to every Meta class''' @@ -25,8 +25,8 @@ class SpaceProjectsSerializer(TranslatableModelSerializer): def get_timelapse(self, obj): # TODO: start and end are a DATE not DATETIME, but we convert them implicitly - return { 'start': util.parsers.datetime_to_utc(obj.date_start), - 'end': util.parsers.datetime_to_utc(obj.date_end) } + return { 'start': util.unix_time.datetime_to_utc(obj.date_start), + 'end': util.unix_time.datetime_to_utc(obj.date_end) } class Meta(LookupById): model = models.Space_project @@ -80,8 +80,8 @@ def get_geo_line(self, obj): def get_time(self, obj): # TODO: change to time_start in model for consistency - return { 'start': util.parsers.datetime_to_utc(obj.time_begin), - 'end': util.parsers.datetime_to_utc(obj.time_end) } + return { 'start': util.unix_time.datetime_to_utc(obj.time_begin), + 'end': util.unix_time.datetime_to_utc(obj.time_end) } class Meta(LookupById): diff --git a/backend/promis/backend_api/views.py b/backend/promis/backend_api/views.py index adaee32..2b51e56 100644 --- a/backend/promis/backend_api/views.py +++ b/backend/promis/backend_api/views.py @@ -26,10 +26,7 @@ from rest_framework.permissions import AllowAny, IsAuthenticated from rest_framework.exceptions import NotAuthenticated, NotFound, MethodNotAllowed -# TODO: also used by orbit.py, maybe make util.time -import datetime, pytz -def maketime(u): - return datetime.datetime.fromtimestamp(u, tz=pytz.utc) +import util.unix_time import datetime from rest_framework.decorators import permission_classes @@ -53,7 +50,7 @@ class SessionFilter(django_filters.rest_framework.FilterSet): def unix_time_filter(self, queryset, name, value): # Composition of the queryset.filter argument depending on which field was used filter_actions = { 'time_begin': 'time_begin__gte', 'time_end': 'time_end__lte' } - return queryset.filter(**{ filter_actions[name]: maketime(value) }) + return queryset.filter(**{ filter_actions[name]: util.unix_time.maketime(value) }) class Meta: diff --git a/backend/promis/functions/potential.py b/backend/promis/functions/potential.py index 57b08c5..ac9b887 100644 --- a/backend/promis/functions/potential.py +++ b/backend/promis/functions/potential.py @@ -22,7 +22,7 @@ # TODO: maintain 1 continuous FTP object from django.contrib.gis.geos import LineString -import util.orbit, util.ftp, util.parsers, util.stats, util.export +import util.orbit, util.ftp, util.parsers, util.stats, util.export, util.unix_time import backend_api.models as model # TODO: integrate into ftp.py somehow @@ -133,8 +133,8 @@ def fetch(daydir): line_gen = ( (y.lon, y.lat, y.alt) for _, y, _ in util.orbit.generate_orbit(orbit, time_start, time_end) ) # Converting time to python objects for convenience # This is the point where onboard time gets converted to the UTC - time_start = util.orbit.maketime(time_start) - time_end = util.orbit.maketime(time_end) + time_start = util.unix_time.maketime(time_start) + time_end = util.unix_time.maketime(time_end) time_dur = time_end - time_start print("\tSession: [ %s, %s ] (%s)." % (time_start.isoformat(), time_end.isoformat(), str(time_dur)) ) diff --git a/backend/promis/functions/test_set.py b/backend/promis/functions/test_set.py index a3a11a3..8c2300c 100644 --- a/backend/promis/functions/test_set.py +++ b/backend/promis/functions/test_set.py @@ -19,7 +19,7 @@ # permissions and limitations under the Licence. # -import util.ftp, util.parsers, util.orbit +import util.ftp, util.parsers, util.unix_time from django.contrib.gis.geos import LineString import backend_api.models as model @@ -34,8 +34,8 @@ def general_fetch(path, satellite_object, add_measurement=False): ftp.cwd(sess_name) with ftp.xopen("orbit.csv") as fp: line_gen = [ pt for pt in util.parsers.csv(fp) ] - time_start = util.orbit.maketime(timemark) - time_end = util.orbit.maketime(timemark + len(line_gen)) # Orbit points are 1 per second + time_start = util.unix_time.maketime(timemark) + time_end = util.unix_time.maketime(timemark + len(line_gen)) # Orbit points are 1 per second time_dur = time_end - time_start # TODO: maybe let the caller print these diagnostics? print("\tSession: [ %s, %s ] (%s)." % (time_start.isoformat(), time_end.isoformat(), str(time_dur)) ) diff --git a/backend/promis/util/orbit.py b/backend/promis/util/orbit.py index cd799b2..e3bb6cd 100644 --- a/backend/promis/util/orbit.py +++ b/backend/promis/util/orbit.py @@ -23,14 +23,9 @@ import collections import operator import util.cubefit -import datetime, pytz _earth_radius = 6371 # km -# TODO: onboard time shift -def maketime(u): - return datetime.datetime.fromtimestamp(u, tz=pytz.utc) - def sign(x): """Returns 1 for non-negative arguments and -1 otherwise.""" return 1 if x>=0 else -1 diff --git a/backend/promis/util/parsers.py b/backend/promis/util/parsers.py index aa101c5..341db0d 100644 --- a/backend/promis/util/parsers.py +++ b/backend/promis/util/parsers.py @@ -22,18 +22,7 @@ import re, struct import util.orbit -import datetime -import pytz - -def datetime_to_utc(x): - # Add the 00:00 time if we only got a date - if type(x) is datetime.date: - x = datetime.datetime.combine(x, datetime.datetime.min.time()) - return int(x.replace(tzinfo=pytz.timezone("UTC")).timestamp()) - -def str_to_utc(x): - time_fmt = "%Y{0}%m{0}%d %H:%M:%S".format(x[4]) - return datetime_to_utc(datetime.datetime.strptime(x, time_fmt)) +import util.unix_time # TODO: cull out those which have standard parsers (CSV?) # TODO: replace ValueErrors with meaningful exception classes when integrating @@ -90,7 +79,7 @@ def scan_sect(sect): m = re.search("^[0-9]* ([0-9.-]*) (2.*)", ln) if m: # Yielding a nested tuple e.g. ( "RX", (1, 432.0) ), will be converted to dict - yield ( sect, (str_to_utc(m.group(2)), float(m.group(1))) ) + yield ( sect, (util.unix_time.str_to_utc(m.group(2)), float(m.group(1))) ) else: raise ValueError("Input inconsistency detected") @@ -150,7 +139,7 @@ def sets(fp, keys=None): keys_found.add(key) # Yield the data - yield key, int(value) if key != "utc" else str_to_utc(value) + yield key, int(value) if key != "utc" else util.unix_time.str_to_utc(value) # Reduce the counter of keys to look for and break if necessary if keys_left > 0: diff --git a/backend/promis/util/unix_time.py b/backend/promis/util/unix_time.py new file mode 100644 index 0000000..1a64f49 --- /dev/null +++ b/backend/promis/util/unix_time.py @@ -0,0 +1,37 @@ +# +# Copyright 2016 Space Research Institute of NASU and SSAU (Ukraine) +# +# Licensed under the EUPL, Version 1.1 or – as soon they +# will be approved by the European Commission - subsequent +# versions of the EUPL (the "Licence"); +# You may not use this work except in compliance with the +# Licence. +# You may obtain a copy of the Licence at: +# +# https://joinup.ec.europa.eu/software/page/eupl +# +# Unless required by applicable law or agreed to in +# writing, software distributed under the Licence is +# distributed on an "AS IS" basis, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. +# See the Licence for the specific language governing +# permissions and limitations under the Licence. +# +"""Useful utilities for converting UNIX timestamps back and forth""" + +import datetime +import pytz + +def datetime_to_utc(x): + # Add the 00:00 time if we only got a date + if type(x) is datetime.date: + x = datetime.datetime.combine(x, datetime.datetime.min.time()) + return int(x.replace(tzinfo=pytz.timezone("UTC")).timestamp()) + +def str_to_utc(x): + time_fmt = "%Y{0}%m{0}%d %H:%M:%S".format(x[4]) + return datetime_to_utc(datetime.datetime.strptime(x, time_fmt)) + +def maketime(u): + return datetime.datetime.fromtimestamp(u, tz=pytz.utc)