diff --git a/README.md b/README.md index 1db9456..07df8c9 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ It allows you to: * [Executing](#executing) * [Discovering Automatically Published Products](#discovering-automatically-published-products) - [Setting Up Your Development Environment](#setting-up-your-development-environment) +- [CLI](#cli) - [TODO List](#todo-list) - [Ideas](#ideas) @@ -341,7 +342,7 @@ Here you can see a metadata example for the [L0](/tests/cosmo/processes/L0.py) p "identifier": "crude", "title": "Crude data", "abstract": "Downloaded from the satellite to the ground station", - "dataType": "string" + "format": "GEOTIFF" } ], "outputs": [ @@ -375,7 +376,7 @@ You can retrieve a list of the published processes, using the _{host}/configurat "metadata": ["Level 0", "Processor"], "inputs": [ { - "dataType": "string", + "format": "GEOTIFF", "format": null, "abstract": "Downloaded from the satellite to the ground station", "identifier": "crude", @@ -485,7 +486,7 @@ _LycheePy_ will calculate and show you which are inputs and the outputs of the c "metadata": ["Cosmo", "Skymed", "Mission", "Chain"], "inputs": [ { - "dataType": "string", + "format": "GEOTIFF", "format": null, "abstract": "Downloaded from the satellite to the ground station", "identifier": "crude", @@ -656,10 +657,19 @@ _{host}/wps?service=WPS&request=describeprocess&version=1.0.0&identifier=**L0**_ crude Crude data - - string - - + + + + + image/tiff; subtype=geotiff + + + + + image/tiff; subtype=geotiff + + + @@ -698,10 +708,19 @@ _{host}/wps?service=WPS&request=describeprocess&version=1.0.0&identifier=**Cosmo crude Crude data - - string - - + + + + + image/tiff; subtype=geotiff + + + + + image/tiff; subtype=geotiff + + + @@ -758,13 +777,12 @@ The following example will execute the _L0_ processor: L0 - - crude - - /cosmo/downlinks/matera/vesubio.sar - - - + + crude + + + + ``` @@ -784,7 +802,8 @@ As a response, you will receive something like this: RAW RAW product - + + @@ -801,13 +820,12 @@ So, this request is identical to the [previous](#executing-a-process), since the Cosmo Skymed - - crude - - /cosmo/downlinks/matera/vesubio.sar - - - + + crude + + + + ``` @@ -827,12 +845,12 @@ As a response, you will receive something like this: GEC GEC Product - + GTC GTC Product - + @@ -1052,6 +1070,11 @@ sudo ./start.sh And now you're able to use your LycheePy instance. +## CLI + +LycheePy also has a web client, which you can find [here](https://github.com/gabrielbazan/lycheepy.cli). + + ## TODO List To be defined. diff --git a/lycheepy/configuration/Dockerfile b/lycheepy/configuration/Dockerfile index d882adb..f449afd 100644 --- a/lycheepy/configuration/Dockerfile +++ b/lycheepy/configuration/Dockerfile @@ -1,17 +1,17 @@ FROM python:2 -ADD requirements.txt /root/ -ADD configuration/ /root/configuration/ -ADD wait-service.sh /root/ - WORKDIR /root/ +ADD requirements.txt /root/ + RUN virtualenv venv RUN /root/venv/bin/pip install -r requirements.txt +ADD configuration/ /root/configuration/ +ADD wait-service.sh /root/ + CMD ./wait-service.sh persistence 5432 && \ ./wait-service.sh processes 21 && \ /root/venv/bin/python /root/configuration/models.py && \ /root/venv/bin/python /root/configuration/parametric.py && \ cd /root/configuration; /root/venv/bin/gunicorn -b 0.0.0.0:80 wsgi - #/root/venv/bin/python /root/configuration/app.py diff --git a/lycheepy/configuration/configuration/models.py b/lycheepy/configuration/configuration/models.py index 86b6501..70e1c00 100644 --- a/lycheepy/configuration/configuration/models.py +++ b/lycheepy/configuration/configuration/models.py @@ -23,6 +23,11 @@ class Process(Describable): version = Column(Text, nullable=False) meta_data = relationship('Metadata', secondary='process_metadata', backref='processes') + @property + def chains(self): + steps = self.steps_before + self.steps_after + return [step.chain for step in steps] + class Chain(Describable): __tablename__ = 'chain' @@ -94,9 +99,9 @@ class Step(Model): __tablename__ = 'step' id = Column(Integer, primary_key=True) after_id = Column(Integer, ForeignKey('process.id'), nullable=False) - after = relationship('Process', foreign_keys=after_id) + after = relationship('Process', foreign_keys=after_id, backref='steps_after') before_id = Column(Integer, ForeignKey('process.id'), nullable=False) - before = relationship('Process', foreign_keys=before_id) + before = relationship('Process', foreign_keys=before_id, backref='steps_before') chain_id = Column(Integer, ForeignKey('chain.id'), nullable=False) chain = relationship('Chain', backref='steps') diff --git a/lycheepy/configuration/configuration/serializers/process.py b/lycheepy/configuration/configuration/serializers/process.py index 834bbb7..b47ef0f 100644 --- a/lycheepy/configuration/configuration/serializers/process.py +++ b/lycheepy/configuration/configuration/serializers/process.py @@ -5,7 +5,7 @@ from simplyrestful.serializers import Serializer from simplyrestful.database import session from simplyrestful.models import get_or_create -from simplyrestful.exceptions import Conflict +from simplyrestful.exceptions import Conflict, NotFound from models import * from validators import ProcessValidator @@ -19,13 +19,11 @@ class ProcessSerializer(Serializer): def create(self, data): self.save_file(data.get('identifier'), creation=True) - serial = super(ProcessSerializer, self).create(data) - return serial + return super(ProcessSerializer, self).create(data) def update(self, identifier, data): self.save_file(data.get('identifier'), creation=False) - serial = super(ProcessSerializer, self).update(identifier, data) - return serial + return super(ProcessSerializer, self).update(identifier, data) def deserialize(self, data, instance): instance.identifier = data.get('identifier', instance.identifier) @@ -112,3 +110,25 @@ def save_file(self, identifier, creation=True): @staticmethod def is_valid_file(filename): return '.' in filename and filename.rsplit('.', 1)[1] in ALLOWED_PROCESSES_EXTENSIONS + + def delete(self, identifier): + try: + instance = self.query.filter_by(id=identifier).one_or_none() + + if not instance: + raise NotFound('The process does not exist') + + if instance.chains: + raise Conflict('The process belongs to a chain, so it cannot be deleted') + + # Delete related objects + session.query(ProcessMetadata).filter_by(process_id=identifier).delete() + session.query(Output).filter_by(process_id=identifier).delete() + session.query(Input).filter_by(process_id=identifier).delete() + + # Delete object + session.delete(instance) + session.commit() + except: + session.rollback() + raise diff --git a/lycheepy/configuration/configuration/validators/chain.py b/lycheepy/configuration/configuration/validators/chain.py index 718cad4..d19388f 100644 --- a/lycheepy/configuration/configuration/validators/chain.py +++ b/lycheepy/configuration/configuration/validators/chain.py @@ -1,44 +1,54 @@ from networkx import DiGraph, is_directed_acyclic_graph from simplyrestful.exceptions import Conflict +from simplyrestful.validators import Validator from models import Process, Output -from validators.executable import ExecutableValidator -class ChainValidator(ExecutableValidator): +class ChainValidator(Validator): def validate(self, data, instance=None): - super(ChainValidator, self).validate(data, instance=instance) steps = data.get('steps', []) - self._validate_at_least_one_step(steps) - graph = self._build_digraph(steps) - self._validate_acyclic(graph) - self._validate_match(steps, graph) + ChainValidator.validate_at_least_one_step(steps) + graph = ChainValidator.build_digraph(steps) + ChainValidator.validate_acyclic(graph) + ChainValidator.validate_match(steps, graph) + ChainValidator.validate_unique(data) - def _validate_at_least_one_step(self, steps): + @staticmethod + def validate_unique(data): + identifier = data.get('identifier') + process = Process.query.filter_by(identifier=identifier).one_or_none() + if process: + raise Conflict('Already exists a process with the given identifier') + + @staticmethod + def validate_at_least_one_step(steps): if not steps: raise Conflict('At least one step is required') - def _validate_acyclic(self, graph): + @staticmethod + def validate_acyclic(graph): if not is_directed_acyclic_graph(graph): raise Conflict('Graph must be directed and acyclic') - def _validate_match(self, steps, graph): - processes = self._get_chain_processes(graph) + @staticmethod + def validate_match(steps, graph): + m = 'The "{}" input of "{}" requires an explicit match, or only one predecessor output with the same identifier' + processes = ChainValidator.get_chain_processes(graph) for process in processes: process_identifier = process.identifier predecessors = graph.predecessors(process_identifier) if predecessors: - predecessors_outputs = self._get_outpus(predecessors) + predecessors_outputs = ChainValidator.get_outputs(predecessors) for process_input in process.inputs: input_identifier = process_input.identifier - has_match = self._has_match(process_identifier, input_identifier, steps) - matchable = self._is_matchable(input_identifier, predecessors_outputs) + has_match = ChainValidator.has_match(process_identifier, input_identifier, steps) + matchable = ChainValidator.is_matchable(input_identifier, predecessors_outputs) if not has_match and not matchable: - raise Conflict( - 'The "{}" input of "{}" requires an explicit match, or only one predecessor output with the same identifier'.format(input_identifier, process_identifier) - ) + raise Conflict(m.format(input_identifier, process_identifier)) - def _has_match(self, process, process_input, steps): + @staticmethod + def has_match(process, process_input, steps): has = False for step in steps: if step.get('after') == process: @@ -47,27 +57,32 @@ def _has_match(self, process, process_input, steps): has = True return has - def _is_matchable(self, process_input, predecessors_outputs): + @staticmethod + def is_matchable(process_input, predecessors_outputs): count = 0 for output in predecessors_outputs: if output.identifier == process_input: count += 1 return count == 1 - def _get_outpus(self, processes): + @staticmethod + def get_outputs(processes): return Output.query.filter(Output.process.has(Process.identifier.in_(processes))).all() - def _get_chain_processes(self, graph): + @staticmethod + def get_chain_processes(graph): return Process.query.filter(Process.identifier.in_(graph.nodes())).all() - def _build_digraph(self, steps): + @staticmethod + def build_digraph(steps): graph = DiGraph() for step in steps: - self._validate_repeated(graph, step['before'], step['after']) + ChainValidator.validate_repeated(graph, step['before'], step['after']) graph.add_edge(step['before'], step['after']) return graph - def _validate_repeated(self, graph, before, after): + @staticmethod + def validate_repeated(graph, before, after): nodes = graph.nodes() if before in nodes and after in nodes and graph.number_of_edges(before, after): raise Conflict('{} to {} step is repeated'.format(before, after)) diff --git a/lycheepy/configuration/configuration/validators/executable.py b/lycheepy/configuration/configuration/validators/executable.py deleted file mode 100644 index a1c9a2f..0000000 --- a/lycheepy/configuration/configuration/validators/executable.py +++ /dev/null @@ -1,18 +0,0 @@ -from simplyrestful.validators import Validator -from models import Chain, Process -from simplyrestful.exceptions import Conflict - - -class ExecutableValidator(Validator): - - def validate(self, data, instance=None): - self._validate_identifier(data, instance) - - @staticmethod - def _validate_identifier(data, instance): - identifier = data.get('identifier') - process = Process.query.filter_by(identifier=identifier).one_or_none() - chain = Chain.query.filter_by(identifier=identifier).one_or_none() - executable = process if process else chain - if executable and not instance or executable and instance and executable.id != instance.id: - raise Conflict('A process or chain with the given identifier already exists') diff --git a/lycheepy/configuration/configuration/validators/process.py b/lycheepy/configuration/configuration/validators/process.py index 0c807b1..fd4cc07 100644 --- a/lycheepy/configuration/configuration/validators/process.py +++ b/lycheepy/configuration/configuration/validators/process.py @@ -1,17 +1,24 @@ -from validators.executable import ExecutableValidator +from simplyrestful.validators import Validator from simplyrestful.exceptions import Conflict -from models import Format, DataType +from models import Format, DataType, Chain -class ProcessValidator(ExecutableValidator): +class ProcessValidator(Validator): def validate(self, data, instance=None): - super(ProcessValidator, self).validate(data, instance=instance) - self._validate_parameters(data.get('inputs')) - self._validate_parameters(data.get('outputs')) + ProcessValidator.validate_parameters(data.get('inputs')) + ProcessValidator.validate_parameters(data.get('outputs')) + ProcessValidator.validate_unique(data) @staticmethod - def _validate_parameters(parameters): + def validate_unique(data): + identifier = data.get('identifier') + chain = Chain.query.filter_by(identifier=identifier).one_or_none() + if chain: + raise Conflict('Already exists a chain with the given identifier') + + @staticmethod + def validate_parameters(parameters): for parameter in parameters: data_format = parameter.get('format') data_type = parameter.get('dataType') diff --git a/lycheepy/configuration/wait-service.sh b/lycheepy/configuration/wait-service.sh index 4f1ec3b..ad1e01c 100755 --- a/lycheepy/configuration/wait-service.sh +++ b/lycheepy/configuration/wait-service.sh @@ -1,8 +1,8 @@ #!/bin/bash -HOST=$1 -PORT=$2 +HOST=${1} +PORT=${2} TIME=5 while ! timeout 1 bash -c "echo > /dev/tcp/${HOST}/${PORT}"; do sleep ${TIME}; done diff --git a/lycheepy/executions/Dockerfile b/lycheepy/executions/Dockerfile index 2c31adf..5198bea 100644 --- a/lycheepy/executions/Dockerfile +++ b/lycheepy/executions/Dockerfile @@ -1,14 +1,15 @@ FROM python:2 -ADD requirements.txt /root/ -ADD executions/ /root/executions/ -ADD wait-service.sh /root/ - WORKDIR /root/ +ADD requirements.txt /root/ + RUN virtualenv venv RUN /root/venv/bin/pip install -r requirements.txt +ADD executions/ /root/executions/ +ADD wait-service.sh /root/ + CMD ./wait-service.sh executions-persistence 5432 && \ /root/venv/bin/python /root/executions/models.py && \ /root/venv/bin/python /root/executions/parametric.py && \ diff --git a/lycheepy/executions/wait-service.sh b/lycheepy/executions/wait-service.sh index 4f1ec3b..ad1e01c 100755 --- a/lycheepy/executions/wait-service.sh +++ b/lycheepy/executions/wait-service.sh @@ -1,8 +1,8 @@ #!/bin/bash -HOST=$1 -PORT=$2 +HOST=${1} +PORT=${2} TIME=5 while ! timeout 1 bash -c "echo > /dev/tcp/${HOST}/${PORT}"; do sleep ${TIME}; done diff --git a/lycheepy/static_ip.txt b/lycheepy/static_ip.txt deleted file mode 100644 index 2511bf9..0000000 --- a/lycheepy/static_ip.txt +++ /dev/null @@ -1,11 +0,0 @@ -sudo vim /etc/netplan/01-netcfg.yaml - - enp0s8: - addresses: - - 192.168.56.12/24 - dhcp4: no - - -sudo netplan generate -sudo netplan apply - diff --git a/lycheepy/worker/Dockerfile b/lycheepy/worker/Dockerfile index 5695020..790a3b4 100644 --- a/lycheepy/worker/Dockerfile +++ b/lycheepy/worker/Dockerfile @@ -2,7 +2,6 @@ FROM python:2 ADD requirements.txt /root/ ADD worker/ /root/worker/ -ADD workdir/ /root/workdir/ ADD wait-service.sh /root/ WORKDIR /root/ @@ -10,7 +9,11 @@ WORKDIR /root/ RUN virtualenv venv RUN /root/venv/bin/pip install -r requirements.txt +RUN /root/venv/bin/pip install -e git+https://github.com/geopython/pywps.git@master#egg=pywps-dev +RUN /root/venv/bin/pip install flufl.enum + RUN mkdir /root/worker/processes/ +RUN mkdir /root/workdir/ CMD ./wait-service.sh broker 5672 && \ cd /root/worker; /root/venv/bin/celery -A distribution.worker worker --loglevel=info diff --git a/lycheepy/worker/requirements.txt b/lycheepy/worker/requirements.txt index d2d67fa..32c4946 100644 --- a/lycheepy/worker/requirements.txt +++ b/lycheepy/worker/requirements.txt @@ -1,3 +1,3 @@ celery -pywps +#pywps gsconfig diff --git a/lycheepy/worker/wait-service.sh b/lycheepy/worker/wait-service.sh index 4f1ec3b..ad1e01c 100755 --- a/lycheepy/worker/wait-service.sh +++ b/lycheepy/worker/wait-service.sh @@ -1,8 +1,8 @@ #!/bin/bash -HOST=$1 -PORT=$2 +HOST=${1} +PORT=${2} TIME=5 while ! timeout 1 bash -c "echo > /dev/tcp/${HOST}/${PORT}"; do sleep ${TIME}; done diff --git a/lycheepy/worker/workdir/CSKS2_GEC_B_HI_16_HH_RA_SF_20130301045754_20130301045801.S01.QLK.tif b/lycheepy/worker/workdir/CSKS2_GEC_B_HI_16_HH_RA_SF_20130301045754_20130301045801.S01.QLK.tif deleted file mode 100644 index b5ad82f..0000000 Binary files a/lycheepy/worker/workdir/CSKS2_GEC_B_HI_16_HH_RA_SF_20130301045754_20130301045801.S01.QLK.tif and /dev/null differ diff --git a/lycheepy/worker/workdir/GEC.tif b/lycheepy/worker/workdir/GEC.tif deleted file mode 100644 index 2f78407..0000000 Binary files a/lycheepy/worker/workdir/GEC.tif and /dev/null differ diff --git a/lycheepy/worker/workdir/MDG.tiff b/lycheepy/worker/workdir/MDG.tiff deleted file mode 100644 index 99bff31..0000000 Binary files a/lycheepy/worker/workdir/MDG.tiff and /dev/null differ diff --git a/lycheepy/worker/workdir/SCS_QLK.tif b/lycheepy/worker/workdir/SCS_QLK.tif deleted file mode 100644 index 9802691..0000000 Binary files a/lycheepy/worker/workdir/SCS_QLK.tif and /dev/null differ diff --git a/lycheepy/worker/workdir/SCS_QLK.tif.aux.xml b/lycheepy/worker/workdir/SCS_QLK.tif.aux.xml deleted file mode 100644 index 9049c8e..0000000 --- a/lycheepy/worker/workdir/SCS_QLK.tif.aux.xml +++ /dev/null @@ -1,238 +0,0 @@ - - - HIMAGE - 1100/ICACQ/Matera - -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan - - -1530 -1518 -1506 -1494 -1482 -1470 -1458 -1446 -1434 -1422 -1410 -1398 -1386 -1374 -1362 -1350 -1338 -1326 -1314 -1302 -1290 -1278 -1266 -1254 -1242 -1230 -1218 -1206 -1194 -1182 -1170 -1158 -1146 -1134 -1122 -1110 -1098 -1086 -1074 -1062 -1050 -1038 -1026 -1014 -1002 -990 -978 -966 -954 -942 -930 -918 -906 -894 -882 -870 -858 -846 -834 -822 -810 -798 -786 -774 -762 -750 -738 -726 -714 -702 -690 -678 -666 -654 -642 -630 -618 -606 -594 -582 -570 -558 -546 -534 -522 -510 -498 -486 -474 -462 -450 -438 -426 -414 -402 -390 -378 -366 -354 -342 -330 -318 -306 -294 -282 -270 -258 -246 -234 -222 -210 -198 -186 -174 -162 -150 -138 -126 -114 -102 -90 -78 -66 -54 -42 -30 -18 -6 6 18 30 42 54 66 78 90 102 114 126 138 150 162 174 186 198 210 222 234 246 258 270 282 294 306 318 330 342 354 366 378 390 402 414 426 438 450 462 474 486 498 510 522 534 546 558 570 582 594 606 618 630 642 654 666 678 690 702 714 726 738 750 762 774 786 798 810 822 834 846 858 870 882 894 906 918 930 942 954 966 978 990 1002 1014 1026 1038 1050 1062 1074 1086 1098 1110 1122 1134 1146 1158 1170 1182 1194 1206 1218 1230 1242 1254 1266 1278 1290 1302 1314 1326 1338 1350 1362 1374 1386 1398 1410 1422 1434 1446 1458 1470 1482 1494 1506 1518 1530 - -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -90 -78 -66 -54 -42 -30 -18 -6 6 18 30 42 54 66 78 90 -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan - 5.6 - ELLIPSOID - - RESTITUTED - 0.955781962876973 -0.292833898284025 0.00797272650971892 -0.02579889700786 0.955780520691639 -0.292849681020622 0.00783151165824841 -0.0257162986777919 0.955789828296638 -0.29284397635948 0.00780331434230514 -0.0254424432774682 0.955811933012235 -0.292786221033683 0.00772289524215787 -0.0253008769276781 0.955808681781914 -0.292821560600223 0.00770208971846652 -0.0250194982351463 0.955807314928453 -0.292843317700203 0.00755893616578169 -0.0248602190585759 0.955824307052977 -0.292809377810286 0.00755176888021656 -0.0246075821613872 0.955823834496207 -0.29282704021078 0.00742718139154428 -0.0244531982967248 0.955822165271058 -0.292855836974937 0.00734689419546945 -0.0241964929677063 0.955842171040717 -0.292812114412473 0.00741530210031588 -0.0239128210010186 0.955850172984326 -0.292805120892412 0.00727700437247343 -0.0237203118102737 0.9558515212459 -0.2928192280492 0.00720633226298038 -0.0235125028843164 0.955844686369918 -0.29286251234882 0.00710399910906831 -0.0232812713525123 0.955856363220515 -0.292844577066352 0.0069983545657389 -0.0230583955871155 0.955870726536444 -0.292820862649123 0.00702097792713597 -0.0227552722915327 - 17800 17810 17820 17830 17840 17850 17860 17870 17880 17890 17900 17910 17920 17930 17940 - ELLIPSOID - 0.203847201669185 - 44697.0941753227 - 0.8 - HAMMING - 3 - 17877.5163772106 - - - - - 6369509.07391284 - -511.532733607015 -0.229717477658737 0.735067212659961 0 0 0 - -511.532733607015 -309855.374643785 1839580305.77403 0 0 0 - NEAR-FAR - 0 0 0 - 1 - 0 0 0 - STANDARD - 0 - 0.81 - GEOMETRY - 3d - 0.177123583698024 - 0.83 - -nan - 100 - ACCC - 1 - 0.86 - 0.04 - 0.2 - GEOMETRY - -3720.14502734044 0.056037513140689 0.00050724738040424 0 0 0 - -3720.14502734044 640574.284058705 -102047129.461859 0 0 0 - 2013-03-01 05:10:01.483266000 - 2013-03-01 05:10:23.783437000 - -0.522401063786425 -0.298367835405598 0.798793923639061 -0.531576292712369 -0.299035041496746 0.792466141693473 -0.540502157743188 -0.299338277287724 0.786289920702796 -0.549444343478333 -0.299829930975474 0.779878633797786 -0.558231713659458 -0.300038768306377 0.773532017695589 -0.567149126986165 -0.300390909223091 0.766879909883447 -0.575776823898631 -0.300574887038427 0.760350808032 -0.584525027913211 -0.300857994623507 0.75353332008134 -0.59313648859955 -0.300923927543967 0.746747175963355 -0.601438395981892 -0.30095479609414 0.740064498465196 -0.609995730001839 -0.301034861204767 0.732994247090728 -0.618373913382505 -0.301070809640315 0.725925180477714 -0.626730095047951 -0.3010086128821 0.718749229700679 -0.63501288733142 -0.300918360264144 0.711480136301836 -0.643028284785229 -0.300690498222659 0.704341552290378 - -6.90437277999999 -0.37743852499998 -4.66262836999999 -6.858339615 -0.341028959999994 -4.73334362 -6.81145715 -0.304647815000021 -4.80350184999998 -6.76373101499999 -0.268300620000014 -4.87309454000001 -6.715167015 -0.231992919999993 -4.94211330000003 -6.66577127 -0.195730304999984 -5.01054999500002 -6.61555003000003 -0.159518364999985 -5.07839660999998 -6.56450964999999 -0.123362655000005 -5.14564517999997 -6.51265639499998 -0.0872686400000021 -5.21228791500002 -6.45999664000001 -0.0512417550000009 -5.27831714500003 -6.406536955 -0.015287410000019 -5.34372536999999 -6.35228397000001 0.0205890250000039 -5.40850502499998 -6.29724436000001 0.0563822150000078 -5.472648635 -6.24142478500003 0.092086879999988 -5.5361488 -6.18483203000001 0.12769773 -5.59899819500001 - 5681100.697 827643.676 4001118.767 5640978.781 804863.271 4061886.691 5600171.024 782048.764 4122181.276 5558682.114 759203.793 4181995.506 5516516.824 736331.991 4241322.422 5473680.011 713436.991 4300155.122 5430176.613 690522.418 4358486.762 5386011.654 667591.894 4416310.558 5341190.237 644649.033 4473619.783 5295717.547 621697.447 4530407.775 5249598.852 598740.737 4586667.93 5202839.496 575782.499 4642393.707 5155444.905 552826.321 4697578.629 5107420.583 529875.781 4752216.28 5058772.112 506934.451 4806300.311 - -3977.7453793 -2276.214015 6100.2241037 -4046.5610715 -2279.8062883 6053.2428621 -4114.9121716 -2283.0345942 6005.5572313 -4182.7902145 -2285.8992446 5957.1728251 -4250.1867919 -2288.4006066 5908.0953405 -4317.0935548 -2290.539103 5858.3305591 -4383.5022173 -2292.3152127 5807.8843406 -4449.4045554 -2293.7294703 5756.7626269 -4514.7924103 -2294.7824658 5704.971437 -4579.6576833 -2295.4748431 5652.5168686 -4643.9923431 -2295.8073009 5599.4050941 -4707.7884224 -2295.7805913 5545.6423612 -4771.0380225 -2295.3955204 5491.2349936 -4833.7333096 -2294.652947 5436.1893885 -4895.8665182 -2293.5537828 5380.5120176 - WGS84 - 6378137 - 6356752.314 - 41.0199169783229 14.0059431436185 0 - 41.0718281916016 14.4765521433817 0 - 40.6013020310757 14.0897469156048 0 - 40.6543909989738 14.5571246590853 0 - OMEGA-KEY - 39971.2039346965 - 3 - - 10 - LINEAR - 10 - ZERO DOPPLER - ELLIPSOID - J2000 - 3.182634435 5.86119362499999 -4.66662650999997 3.171469625 5.810197285 -4.73732871500001 3.15993466 5.75852342000003 -4.80747343500002 3.14803091 5.706178215 -4.87705214500002 3.135759795 5.65316799999996 -4.94605646 3.12312287999999 5.59949938 -5.01447825999999 3.11012177499999 5.54517908000003 -5.08230951999999 3.09675812 5.490213885 -5.14954226999998 3.08303356 5.43461045499998 -5.21616873500002 3.06894981 5.37837552999999 -5.28218125000003 3.054508665 5.32151597500001 -5.34757230499999 3.03971194999999 5.26403870500003 -5.41233433999996 3.02456153000001 5.20595063999999 -5.47645988999998 3.00905928000001 5.14725866999997 -5.53994154000002 2.993207075 5.08796977500001 -5.60277198500003 - -2738479.671 -5043123.46 4004558.081 -2728932.318 -4999351.323 4065314.838 -2719067.816 -4954998.16 4125597.858 -2708887.316 -4910069.14 4185400.126 -2698392.011 -4864569.496 4244714.683 -2687583.126 -4818504.529 4303534.63 -2676461.926 -4771879.608 4361853.125 -2665029.711 -4724700.163 4419663.383 -2653287.818 -4676971.691 4476958.682 -2641237.617 -4628699.753 4533732.359 -2628880.519 -4579889.973 4589977.813 -2616217.967 -4530548.035 4645688.505 -2603251.441 -4480679.689 4700857.957 -2589982.456 -4430290.743 4755479.759 -2576412.562 -4379387.065 4809547.56 - 938.8402453 4347.991899 6099.1273804 970.6116927 4406.3505551 6052.1062215 1002.2696378 4464.1958447 6004.3808061 1033.8103859 4521.5210235 5955.9567528 1065.230256 4578.319409 5906.8397632 1096.5255818 4634.5843835 5857.0356236 1127.6927136 4690.3093966 5806.550198 1158.7280173 4745.4879651 5755.3894332 1189.627876 4800.1136743 5703.5593526 1220.3886885 4854.1801742 5651.0660585 1251.0068722 4907.6811849 5597.9157276 1281.4788618 4960.6104937 5544.1146124 1311.8011112 5012.961959 5489.6690408 1341.9700924 5064.7295065 5434.5854146 1371.9822968 5115.9071324 5378.87021 - nan - 0 - 1.75 - 1.39 - -12 -12 - 10.0.8 (2013.315.14) - 01.04.22 - 0 - NULL - 299792458 - EARLY-LATE - 0.5 - RIGHT - CSK - H4-16 - - 15d - ASCENDING - 28279d - - 0.15 7.5 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 1.38644099235535 - -999999 - FILTERED - PASS - -40 - ICUGS - 4194537 - - CSKS2_SCS_B_HI_16_HH_RA_SF_20130301045754_20130301045801.h5 - 2014-10-30 17:43:37.000000000 - CSM-UGS-STD-SPE-0048-3.1 - SCS_B - 35535d - SLANT RANGE/AZIMUTH - -22 -22 - 9600000000 - 0.0312283810416667 - ZERO DOPPLER - - 0.68 - HAMMING - 0.006006367 - - ZERO DOPPLER - 0.6 - 0.1 - 1 - 2 - 0.001 - 1d - 3 - 1 - 2 - - 40 - 780000 - 1.5 - 2013-03-01 00:00:00.000000000 - REPLICA - 0.168833762407303 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - / - -9.885 - -30.72172977 -31.77598977 -32.89878977 -34.16547977 -35.60823977 -37.18121977 -38.91104977 -40.83251977 -42.99352977 -45.46249977 -48.34222977 -40.07600977 -34.24577977 -29.88830977 -26.40761977 -23.50927977 -21.02605977 -18.85383177 -16.87323077 -15.04984977 -13.40059877 -11.89443977 -10.50851377 -9.22503577 -8.02989677 -6.91171477 -6.10334877 -5.33589477 -4.60091677 -3.89577577 -3.21814577 -2.56595877 -1.93718887 -1.51289297 -1.28849757 -1.06700087 -0.84829297 -0.63230457 -0.41896877 -0.20822127 0 -0.20503877 -0.41252657 -0.62252247 -0.83508817 -1.05028717 -1.26818547 -1.48899947 -1.91010167 -2.53690577 -3.18717577 -3.86273677 -4.56563577 -5.29817977 -6.06297977 -6.87061077 -7.99376077 -9.19198877 -10.47903277 -11.86912077 -13.38019877 -15.03535777 -16.86499777 -18.85825577 -21.05815977 -23.57767977 -26.52579977 -30.07893977 -34.55155977 -40.59322977 -49.31065977 -45.93230977 -43.11584977 -40.69857977 -38.58021977 -36.69434977 -34.99463977 -33.44579977 -32.13148977 -31.00104977 -29.93992977 - -0.4 - 0.01 - 2817.5313059034 - 2817.5313059034 - 2.43483066703037 - 17873.8527658861 - 17881.179392687 - - 0d - 0 - - 0d - 0 - 63083481.182359 - 63083473.8557129 - 0.006006367 - 0d - 0d - -7.94705046770501 -24.6501555213507 -7.86782481984438 -24.5396803950099 -7.79731718588112 -24.5293150414388 -7.66712000325879 -24.4311643517411 -7.6282724435372 -24.3906106523492 -7.65830691604742 -24.4339868762327 -7.74474237778078 -24.5709728088246 -7.79316052591359 -24.6492526646297 -7.85408710468233 -24.5358427262427 -8.02125433904506 -24.5337598589314 -8.28667020858767 -24.687700208746 -8.41964668561672 -24.7819430069262 -8.35186855040221 -24.5668181441707 -8.39108504215975 -24.6161471681837 -8.47264018779787 -24.7388430392429 -8.46838596336057 -24.7915087659965 -8.50676076064359 -24.7528521539159 -8.5217416961162 -24.7637639303372 -8.52779141413503 -24.7362246107154 -8.52877777919435 -24.6759730696095 -8.52345005589063 -24.6674625866514 -8.53978378470909 -24.7273636414919 -8.50025908540499 -24.6481777545233 -8.5129459254493 -24.6776522996812 - 1.00406625073919 1.00641225622943 1.00358836031722 1.00112521004928 1.00115458185268 1.00193371512421 1.00172406838886 1.00106374275097 1.00422999575024 1.00412413917688 1.00180563732905 0.997184213595029 0.99603184321138 0.996174981805033 0.997260741345036 0.996578049753112 0.996510505631821 0.997126706658105 0.998960294403067 1.00064748215925 1.00061628781437 1.00026758379256 1.00130838232753 1.00282244206103 - 0.685775079165286 0.725207353278276 0.00519939547565119 0.00587143620787944 0.598553813105038 0.605713286561744 -nan -nan 2.53010366517109 -nan 0.0166284127521504 0.017918955230371 0.0192828329958325 0.0210746120638662 0.455902992670965 0.484180558453326 0.325787979821115 0.3609547106396 0.00775432932314519 0.00930028167066412 0.0246647693538132 0.0388664863657592 0.0231878934791713 0.0343276093693347 0.216398791681972 0.320529432021092 0.0245829302748725 0.0316162090679376 0.0327782153475887 0.0435054857198372 0.714535742958782 0.746650304097879 0.189411102277668 0.205355907183807 1.05454106189949 1.23654524696912 0.0141656676973894 0.0162530726815597 0.0158449240866829 0.0205758626149444 2.24737178618795 2.14085507216889 0.00383877980897591 0.00422034207909934 0.00296358459523274 0.0032847480499381 0.00626615402465082 0.006622736503843 - 90.171123453859 90.2137721515558 90.1862944216404 90.1315459227202 90.0136966549618 89.9757136240612 90.0611571802374 90.1709365772127 90.1735340363324 90.1444597080415 90.1547148569987 90.1353079998876 90.1349991928975 90.1527815313421 90.1160646538354 90.1146016803026 90.1348940563635 90.1176097859074 90.1490099484234 90.1493549089447 90.1230829731555 90.1423231374508 90.143409207247 90.1499087553832 - 0d - 18d 1022d 1042d 2046d 2066d 3070d 3090d 4094d 4114d 5118d 5138d 6142d 6162d 7166d 7186d 8190d 8210d 9214d 9234d 10238d 10258d 11262d 11282d 12286d 12306d 13310d 13330d 14334d 14354d 15358d 15378d 16382d 16402d 17406d 17426d 18430d 18450d 19454d 19474d 20478d 20498d 21502d 21522d 22526d 22546d 23550d 23570d 24574d - 0.390625 - 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d 2d - 2.95627384194913 2.23106935228358 - 0.0321781381420388 0.0329132883140636 0.033964843119793 0.0339746974848005 0.0340835704520513 0.033779517611737 0.0333081083768846 0.0330576872805034 0.0329559958982708 0.0316612817045526 0.0298661100197103 0.0291803974882757 0.0282591327901586 0.0284407019547975 0.0291498476902048 0.0289770845617214 0.0292657792941613 0.0293260949661849 0.0293508019230541 0.0291316273647311 0.0289875183491499 0.0290411044252206 0.0291424041818121 0.0290348062677609 - 107.425641208842 106.990590640564 93.6419656367422 93.0453351070827 101.623010767945 101.259654641494 121.871277265387 121.734300607002 131.451851575149 131.300254683838 130.204591286913 129.953298627915 131.389246993942 131.16311281735 126.530524605164 126.396071700142 107.255945827167 106.804164664528 111.208059493256 110.751305694551 147.818554463618 147.552128831818 227.791974059225 228.435198786385 319.161321230995 320.432849016115 302.210215984163 303.370614102925 249.592951642232 250.278529269686 266.832950846466 267.749175202655 262.647111244321 263.566826199985 237.68506049485 238.369967334901 206.798434912925 207.013668182376 185.318930418045 185.199017358395 195.226580005027 195.106338346197 197.310246730837 197.257463830555 177.645877970772 177.413752951748 166.538811561308 166.070088358845 - 4.26417113148612 6.05022929681271 - - 0d - H4-16 - 43.95 - 1d - 3.97569656353014e+22 - 2000-01-01 00:00:08.000000000 - 90000000 - 709.619754079693 -0.0793740159919506 2.21961306536976e-06 - 40.8504543871434 14.2814577028433 0 - 75000000 - -nan - -nan - 0 - -0.767663356553972 0.57236701882341 -0.288249871472589 -0.763185585217151 0.574913779966597 -0.294993042043026 -0.758659060371094 0.577427416450993 -0.301683545473779 -0.754110810417896 0.579852370381537 -0.308361855867484 -0.749401091953107 0.582438716002248 -0.314900036925028 -0.744567129505495 0.585034172169345 -0.321486646360123 -0.739785807028948 0.58750276271104 -0.327957757408469 -0.734822998589421 0.590087424357928 -0.334411110539858 -0.729767361197138 0.592697876445773 -0.340804338130421 -0.724831338300239 0.595129721765372 -0.347043067539615 -0.719626525778962 0.597699054694157 -0.353400940072591 -0.714396281580057 0.600273148425503 -0.359595325494643 -0.709024988060858 0.60291553929968 -0.365752778600375 -0.703624386429301 0.605462367820245 -0.371922976543423 -0.698252972913542 0.607958205602278 -0.377926242045988 - 20992d - 2.77365846773076 - 24576d - 90000000 - HH - 3354.20393559928 - -75.18884 -71.89775 -69.27212 -66.69033 -64.21155 -62.10831 -60.27333 -58.65979 -57.21439 -55.89217 -54.67248 -53.58771 -52.59937 -51.66786 -50.7867 -50.01551 -49.30862 -48.63078 -47.97961 -47.43093 -46.92715 -46.43823 -45.96327 -45.59025 -45.24836 -44.9134 -44.58506 -44.36136 -44.15806 -43.95727 -43.75893 -43.66937 -43.59152 -43.51408 -43.43704 -43.47318 -43.51406 -43.55504 -43.59795 -43.75371 -43.91088 -44.0695 -44.23714 -44.50659 -44.78043 -45.05883 -45.35358 -45.73666 -46.12918 -46.53163 -46.95723 -47.45495 -47.9704 -48.50506 -49.0696 -49.69128 -50.34664 -51.04006 -51.77837 -52.57876 -53.45348 -54.41842 -55.49221 -56.72193 -58.18037 -59.96531 -61.95439 -64.01841 -66.85728 -71.31194 -70.74734 -64.77719 -61.13881 -58.48423 -56.36031 -54.57165 -53.02917 -51.66847 -50.47135 -49.39898 -48.41115 -47.49419 -46.68066 -45.94258 -45.24354 -44.57919 -44.00468 -43.48539 -42.98576 -42.50419 -42.11174 -41.76254 -41.42309 -41.09278 -40.85606 -40.65404 -40.45649 -40.26319 -40.17095 -40.10689 -40.04498 -39.98515 -40.03628 -40.11085 -40.18777 -40.26709 -40.46981 -40.69344 -40.92252 -41.15736 -41.53105 -41.9266 -42.3351 -42.75758 -43.33686 -43.94536 -44.58243 -45.25386 -46.08611 -46.97037 -47.91451 -48.92733 -49.99982 -51.15625 -52.41271 -53.70103 -54.33654 -55.00048 -55.6958 -56.22702 -55.78138 -55.50825 -55.39222 -55.3712 -55.32521 -55.52073 -55.9748 -56.70749 -57.74797 -59.32188 -61.7604 -63.64133 -60.93483 -58.68968 -56.75973 -54.62775 -52.06192 -49.9428 -48.12645 -46.54764 -45.16449 -43.90998 -42.76003 -41.73142 -40.82216 -39.96858 -39.16358 -38.44558 -37.81706 -37.21613 -36.64024 -36.14014 -35.71506 -35.30405 -34.90611 -34.58243 -34.32599 -34.07665 -33.83405 -33.66989 -33.56927 -33.47278 -33.38032 -33.37548 -33.43414 -33.49756 -33.56584 -33.736981 -33.976502 -34.22645 -34.487541 -34.875778 -35.346202 -35.842999 -36.369329 -37.057897 -37.859865 -38.731128 -39.68585 -40.74822 -41.93061 -43.26173 -44.7891 -44.99315 -44.74586 -44.52459 -44.32757 -42.94657 -41.58486 -40.49518 -39.60586 -38.69887 -37.9419 -37.34075 -36.86861 -36.47869 -36.20942 -36.05712 -36.0158 -36.10796 -36.34236 -36.72907 -37.28924 -38.13658 -39.31459 -40.95686 -43.3495 -43.63687 -43.80449 -44.04606 -44.36785 -39.93383 -36.69024 -34.16742 -32.08405 -30.23486 -28.61558 -27.17097 -25.863265 -24.640198 -23.512777 -22.465844 -21.487275 -20.558435 -19.68264 -18.853713 -18.066509 -17.316348 -16.599891 -15.914064 -15.256749 -14.63054 -14.027413 -13.44565 -12.884896 -12.351904 -11.835439 -11.334474 -10.849893 -10.390874 -9.944018 -9.50868 -9.086724 -8.688694 -8.29976 -7.919509 -7.550621 -7.204327 -6.864895 -6.532055 -6.20919 -5.907772 -5.611543 -5.320325 -5.038113 -4.77633 -4.518458 -4.264379 -4.018631 -3.792395 -3.569078 -3.348605 -3.135995 -2.942069 -2.750287 -2.560605 -2.378467 -2.214272 -2.051616 -1.8904704 -1.7366671 -1.6001507 -1.4646994 -1.3302966 -1.2031238 -1.0926697 -0.9829136 -0.8738467 -0.7719713 -0.6863357 -0.6011203 -0.5163209 -0.438737 -0.3770058 -0.3154933 -0.2541978 -0.20019765 -0.16175574 -0.12339873 -0.08512624 -0.05428023 -0.0387914 -0.02331642 -0.007855258 0 -0.007394532 -0.014792285 -0.022193263 -0.03742636 -0.06789496 -0.09841719 -0.12899324 -0.16767367 -0.2216677 -0.2758301 -0.3301621 -0.392917 -0.4711551 -0.5497472 -0.6286965 -0.7164349 -0.8199173 -0.92402 -1.0287502 -1.1426863 -1.2727188 -1.4037323 -1.5357414 -1.6774298 -1.8356604 -1.9953466 -2.156516 -2.327903 -2.516376 -2.706921 -2.899582 -3.103083 -3.324321 -3.548423 -3.775463 -4.014071 -4.271191 -4.532194 -4.797203 -5.074655 -5.37154 -5.673634 -5.981124 -6.302146 -6.643716 -6.992229 -7.34798 -7.71867 -8.111291 -8.513173 -8.924773 -9.353212 -9.805371 -10.269966 -10.74772 -11.245015 -11.76848 -12.308893 -12.86743 -13.449586 -14.061546 -14.697203 -15.358573 -16.050199 -16.777588 -17.539583 -18.339914 -19.182456 -20.071957 -21.015898 -22.022128 -23.095627 -24.241661 -25.485984 -26.8496 -28.3487 -30.00104 -31.89299 -34.12043 -36.68531 -39.41964 -43.02499 -48.51729 -52.47053 -45.63069 -42.22932 -40.08758 -38.63267 -37.62077 -36.89498 -36.39562 -36.09187 -35.95328 -35.94816 -36.07628 -36.32848 -36.65388 -37.10336 -37.6966 -38.39421 -38.96792 -39.66845 -40.5256 -41.34234 -41.19626 -41.05809 -40.92748 -40.64568 -39.61481 -38.666316 -37.786793 -36.965061 -36.193183 -35.472015 -34.79477 -34.174063 -33.715903 -33.278254 -32.859275 -32.472606 -32.244601 -32.024265 -31.811188 -31.615452 -31.567626 -31.52292 -31.481276 -31.448581 -31.556704 -31.66774 -31.78181 -31.90073 -32.15734 -32.41985 -32.68859 -32.96387 -33.37325 -33.79705 -34.23404 -34.68515 -35.26837 -35.88127 -36.52038 -37.18836 -37.99461 -38.85587 -39.76944 -40.74287 -41.87863 -43.12146 -44.47994 -45.9811 -47.7158 -49.70872 -52.04959 -54.91451 -56.74689 -58.38022 -60.19104 -62.22488 -60.49739 -58.76338 -57.79251 -57.3714 -57.37708 -57.84149 -58.86498 -60.67731 -61.30666 -61.19759 -61.19415 -61.29623 -58.52487 -55.24233 -52.66747 -50.52935 -48.804 -47.33921 -46.01559 -44.80591 -43.78993 -42.91145 -42.08321 -41.29926 -40.63913 -40.08083 -39.54309 -39.02432 -38.5957 -38.25669 -37.92571 -37.60235 -37.34866 -37.18133 -37.01648 -36.85401 -36.74698 -36.72742 -36.70846 -36.69008 -36.71659 -36.83413 -36.95293 -37.07302 -37.23014 -37.48351 -37.74107 -38.003 -38.29676 -38.69314 -39.09936 -39.51595 -39.96252 -40.51898 -41.09484 -41.69158 -42.32189 -43.06983 -43.85365 -44.6772 -45.54888 -46.54592 -47.61039 -48.75294 -49.98718 -51.35814 -52.87491 -54.57717 -56.52513 -58.6684 -61.28875 -64.73954 -69.95877 -68.1793 -65.89132 -64.13417 -62.71763 -60.48242 -58.45642 -56.76245 -55.30136 -53.92128 -52.63992 -51.46482 -50.37817 -49.37416 -48.4338 -47.54531 -46.70297 -45.95035 -45.261 -44.59893 -43.96197 -43.40688 -42.91967 -42.44612 -41.98546 -41.59357 -41.27583 -40.964 -40.65785 -40.40669 -40.23718 -40.06946 -39.90347 -39.77947 -39.74549 -39.7117 -39.67809 -39.67476 -39.77021 -39.8663 -39.96307 -40.07994 -40.3041 -40.5313 -40.76163 -41.00411 -41.36052 -41.7245 -42.09637 -42.47649 -42.97055 -43.47972 -44.00429 -44.54525 -45.17427 -45.8341 -46.52002 -47.23418 -47.99341 -48.79168 -49.62993 -50.51255 -51.37617 -52.27041 - -6 - 0.0199999999999996 - 3e-05 - -2475585937500 - 2700d - 72000000 - 72000000 - -8.33e-07 - - 1024d 20992d - -nan - 90000000 - 0.467913736444654 - 1024d - - INT - 628005.339820165 - CSKS2 - 40.8504543871434 14.2814577028433 0 - 171.517907821783 - 2013-03-01 04:57:53.852774526 - 2013-03-01 04:58:01.178805059 - OFF - -22 -22 - 17800 17810 17820 17830 17840 17850 17860 17870 17880 17890 17900 17910 17920 17930 17940 - - 0 0 0 0 0 0 - 0 0 - 1d - 1d - 24d - 2d - 2d - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - - - - NEAR-FAR - 49.9654096666667 - EARLY-LATE - 61.3577754763284 - SLANT RANGE/AZIMUTH - - - diff --git a/lycheepy/worker/worker/distribution/serialization.py b/lycheepy/worker/worker/distribution/serialization.py index 9adb226..3f281b8 100644 --- a/lycheepy/worker/worker/distribution/serialization.py +++ b/lycheepy/worker/worker/distribution/serialization.py @@ -21,12 +21,12 @@ def to_json(outputs): if isinstance(output, ComplexOutput): additional_properties = dict( type='complex', - file=output.file, # TODO + file=output.file, source=output.source, data_format=output.data_format.json, supported_formats=[f.json for f in output.supported_formats], - data = output.data if not output.as_reference else None, - as_reference = output.as_reference + data=output.data if not output.as_reference else None, + as_reference=output.as_reference ) else: additional_properties = dict( @@ -40,7 +40,9 @@ def to_json(outputs): output_json.update(additional_properties) - # TODO: Support multiple occurrences - outputs_json[identifier] = [output_json] + if identifier not in outputs_json: + outputs_json[identifier] = [] + + outputs_json[identifier].append(output_json) return outputs_json diff --git a/lycheepy/worker/worker/distribution/worker.py b/lycheepy/worker/worker/distribution/worker.py index 749c132..84f3413 100644 --- a/lycheepy/worker/worker/distribution/worker.py +++ b/lycheepy/worker/worker/distribution/worker.py @@ -14,7 +14,7 @@ app.config_from_object(configuration) -@app.task(name=BROKER_TASK_NAME) +@app.task(name=BROKER_PROCESS_EXECUTION_TASK_NAME) def run_process(identifier, wps_request_json): processes = ProcessesGateway( @@ -41,7 +41,7 @@ def run_process(identifier, wps_request_json): return dict(process=identifier, outputs=outputs) -@app.task(name='run_chain_process') +@app.task(name=BROKER_CHAIN_PROCESS_EXECUTION_TASK_NAME) def run_chain_process(identifier, wps_request_json, products, chain_identifier, execution_id): outputs = run_process(identifier, wps_request_json).get('outputs') publish(products, identifier, outputs, chain_identifier, execution_id) diff --git a/lycheepy/worker/worker/settings.py b/lycheepy/worker/worker/settings.py index 6b9e496..82c5c22 100644 --- a/lycheepy/worker/worker/settings.py +++ b/lycheepy/worker/worker/settings.py @@ -2,7 +2,8 @@ BROKER_APPLICATION_NAME = 'lycheepy' -BROKER_TASK_NAME = 'run_process' +BROKER_PROCESS_EXECUTION_TASK_NAME = 'run_process' +BROKER_CHAIN_PROCESS_EXECUTION_TASK_NAME = 'run_chain_process' REPOSITORIES = { Repositories.GEO_SERVER: [ diff --git a/lycheepy/wps/Dockerfile b/lycheepy/wps/Dockerfile index 217bc39..d21a932 100644 --- a/lycheepy/wps/Dockerfile +++ b/lycheepy/wps/Dockerfile @@ -1,17 +1,21 @@ FROM python:2 -ADD requirements.txt /root/ -ADD wps/ /root/wps/ -ADD wait-service.sh /root/ - WORKDIR /root/ +ADD requirements.txt /root/ + RUN virtualenv venv RUN /root/venv/bin/pip install -r requirements.txt -RUN mkdir /root/wps/outputs -RUN mkdir /root/wps/workdir -RUN mkdir /root/wps/logs +RUN /root/venv/bin/pip install -e git+https://github.com/geopython/pywps.git@master#egg=pywps-dev +RUN /root/venv/bin/pip install flufl.enum + +ADD wps/ /root/wps/ +ADD wait-service.sh /root/ + +RUN mkdir /root/outputs +RUN mkdir /root/workdir +RUN mkdir /root/logs CMD ./wait-service.sh configuration 80 && \ cd /root/wps; /root/venv/bin/gunicorn -b 0.0.0.0:80 wsgi diff --git a/lycheepy/wps/requirements.txt b/lycheepy/wps/requirements.txt index 3575ef5..82f1e45 100644 --- a/lycheepy/wps/requirements.txt +++ b/lycheepy/wps/requirements.txt @@ -4,5 +4,5 @@ simplyrestful networkx==1.11 gunicorn celery -pywps +#pywps requests diff --git a/lycheepy/wps/wps/adapter/executable_adapter.py b/lycheepy/wps/wps/adapter/executable_adapter.py index b30ee61..6e82b76 100644 --- a/lycheepy/wps/wps/adapter/executable_adapter.py +++ b/lycheepy/wps/wps/adapter/executable_adapter.py @@ -31,9 +31,10 @@ def _handle(self, request, response): for output in self.outputs: output_identifier = output.identifier - OutputsDeserializer.add_data( - outputs.get(output_identifier)[0], # TODO: Handle outputs with multiple occurrences - response.outputs.get(output_identifier) - ) + for result_output in outputs.get(output_identifier): + OutputsDeserializer.add_data(result_output, response.outputs.get(output_identifier)) return response + + def clean(self): + pass diff --git a/lycheepy/wps/wps/app.py b/lycheepy/wps/wps/app.py index 48d09a2..25c7db2 100644 --- a/lycheepy/wps/wps/app.py +++ b/lycheepy/wps/wps/app.py @@ -1,4 +1,4 @@ -import os +from os.path import join, isfile from flask import abort, send_from_directory from flask_cors import CORS @@ -23,11 +23,12 @@ def wps(): ).add_executables().build() -@application.route('/outputs/') +@application.route('/outputs/', methods=['GET']) def output_file(filename): outputs_dir = get_config_value('server', 'outputpath') - target_file = os.path.join(outputs_dir, filename) - if os.path.isfile(target_file): + target_file = join(outputs_dir, filename) + + if isfile(target_file): return send_from_directory(outputs_dir, filename) else: abort(404) diff --git a/lycheepy/wps/wps/executor/executor.py b/lycheepy/wps/wps/executor/executor.py index 24ac039..c413d7d 100644 --- a/lycheepy/wps/wps/executor/executor.py +++ b/lycheepy/wps/wps/executor/executor.py @@ -13,7 +13,8 @@ def __init__(self): BROKER_PROTOCOL, BROKER_USERNAME, BROKER_APPLICATION_NAME, - BROKER_TASK_NAME + BROKER_PROCESS_EXECUTION_TASK_NAME, + BROKER_CHAIN_PROCESS_EXECUTION_TASK_NAME ) self.executions = ExecutionsGateway(EXECUTIONS_URL) diff --git a/lycheepy/wps/wps/gateways/broker/gateway.py b/lycheepy/wps/wps/gateways/broker/gateway.py index 567abcc..491350a 100644 --- a/lycheepy/wps/wps/gateways/broker/gateway.py +++ b/lycheepy/wps/wps/gateways/broker/gateway.py @@ -3,7 +3,7 @@ class BrokerGateway(object): - def __init__(self, host, protocol, username, application_name, task_name): + def __init__(self, host, protocol, username, application_name, process_task_name, chain_process_task_name): self.app = Celery(application_name) url = '{}://{}@{}//'.format(protocol, username, host) self.app.conf.update( @@ -11,11 +11,11 @@ def __init__(self, host, protocol, username, application_name, task_name): result_backend=url ) - @self.app.task(name=task_name) + @self.app.task(name=process_task_name) def run_process(identifier, wps_request_json): pass - @self.app.task(name='run_chain_process') + @self.app.task(name=chain_process_task_name) def run_chain_process(identifier, wps_request_json, products, chain_identifier, execution_id): pass diff --git a/lycheepy/wps/wps/pywps.cfg b/lycheepy/wps/wps/pywps.cfg index 07b8eb3..aacc7d1 100644 --- a/lycheepy/wps/wps/pywps.cfg +++ b/lycheepy/wps/wps/pywps.cfg @@ -1,6 +1,6 @@ [metadata:main] identification_title=LycheePy Server -identification_abstract=LycheePy is a geospatial data processing server that allows you to expose predeterminated chains, and executes them in a distributed way. Also, it can be directly integrated with geospatial repositories, so you can configure these chains to automaticly publish outputs into them +identification_abstract=LycheePy is a distributed processing server which allows the publication of pre-defined processing chains through a WPS interface, implemented with PyWPS. In addition, it provides direct integration with repositories (e.g. GeoServer, FTP servers, or you own integration), allowing the automatic publication of processing results. identification_keywords=LycheePy,WPS,PyWPS,Processes,Geospatial,Distribution,Automatic,Products,Publication,Repositories identification_keywords_type=theme identification_fees=None @@ -9,33 +9,30 @@ provider_name=LycheePy Development Team provider_url=http://lycheepy.org/ contact_name=Gabriel Jose Bazan contact_position=Developer +contact_stateorprovince=Somewhere +contact_city=Space-time contact_address=Somewhere in space-time -contact_city=Ciudad Autonoma de Buenos Aires -contact_stateorprovince=Argentina contact_postalcode=0 contact_country=World, Internet -contact_phone=+54 -contact_fax=+54 +contact_phone=+00 +contact_fax=+00 contact_email=gbazan@outlook.com -contact_url=http://lycheepy.org +contact_url=https://github.com/gabrielbazan/lycheepy contact_hours=12:00-20:00UTC contact_instructions=Knock on the heavens door -contact_role=hallo +contact_role=LycheePy Development Team [server] maxsingleinputsize=10mb maxrequestsize=30mb -url=http://wps/wps -outputurl=http://wps/wps/outputs/ -outputpath=/root/wps/outputs -workdir=/root/wps/workdir +url=http://{host_name_or_ip} +outputurl=http://{host_name_or_ip}/outputs/ +outputpath=/root/outputs +workdir=/root/workdir maxprocesses=10 parallelprocesses=20 [logging] -level=INFO -file=/root/wps/logs/pywps.log -database=sqlite:////root/wps/logs/pywps-logs.sqlite3 - -[grass] -gisbase=/usr/local/grass-7.3.svn/ +level=DEBUG +file=/root/logs/pywps.log +database=sqlite:////root/logs/pywps-logs.sqlite3 diff --git a/lycheepy/wps/wps/settings.py b/lycheepy/wps/wps/settings.py index 3bd132c..46e7d6f 100644 --- a/lycheepy/wps/wps/settings.py +++ b/lycheepy/wps/wps/settings.py @@ -9,6 +9,7 @@ BROKER_PROTOCOL = 'amqp' BROKER_USERNAME = 'guest' BROKER_APPLICATION_NAME = 'lycheepy' -BROKER_TASK_NAME = 'run_process' +BROKER_PROCESS_EXECUTION_TASK_NAME = 'run_process' +BROKER_CHAIN_PROCESS_EXECUTION_TASK_NAME = 'run_chain_process' -PROCESS_EXECUTION_TIMEOUT = 30 +PROCESS_EXECUTION_TIMEOUT = 120 diff --git a/tests/cosmo/execute_cosmo.xml b/tests/cosmo/execute_cosmo.xml index b8fba23..4e0cf07 100644 --- a/tests/cosmo/execute_cosmo.xml +++ b/tests/cosmo/execute_cosmo.xml @@ -4,9 +4,12 @@ crude - - /sar/data/acquisition.sar - + + + + + + diff --git a/tests/cosmo/processes/L0.py b/tests/cosmo/processes/L0.py index 55b4dfc..2202a4c 100644 --- a/tests/cosmo/processes/L0.py +++ b/tests/cosmo/processes/L0.py @@ -1,11 +1,11 @@ -from pywps import Format, ComplexOutput, Process, LiteralInput +from pywps import Format, ComplexOutput, Process, ComplexInput from pywps.app.Common import Metadata class L0(Process): def __init__(self): inputs = [ - LiteralInput('crude', 'Crude Data', data_type='string') + ComplexInput('crude', 'Crude Data', [Format('image/tiff; subtype=geotiff')]) ] outputs = [ @@ -27,6 +27,6 @@ def __init__(self): ) def _handler(self, request, response): - response.outputs['RAW'].file = '/root/workdir/CSKS2_GEC_B_HI_16_HH_RA_SF_20130301045754_20130301045801.S01.QLK.tif' + response.outputs['RAW'].file = request.inputs['crude'][0].file response.outputs['RAW'].as_reference = True return response diff --git a/tests/cosmo/processes/L1A.py b/tests/cosmo/processes/L1A.py index 3e357b9..a7dc027 100644 --- a/tests/cosmo/processes/L1A.py +++ b/tests/cosmo/processes/L1A.py @@ -1,11 +1,11 @@ -from pywps import Format, ComplexOutput, Process, LiteralInput +from pywps import Format, ComplexOutput, Process, ComplexInput from pywps.app.Common import Metadata class L1A(Process): def __init__(self): inputs = [ - LiteralInput('RAW', 'RAW Data', data_type='string') + ComplexInput('RAW', 'RAW Data', [Format('image/tiff; subtype=geotiff')]) ] outputs = [ @@ -27,8 +27,6 @@ def __init__(self): ) def _handler(self, request, response): - - response.outputs['SCS'].file = '/root/workdir/CSKS2_GEC_B_HI_16_HH_RA_SF_20130301045754_20130301045801.S01.QLK.tif' + response.outputs['SCS'].file = request.inputs['RAW'][0].file response.outputs['SCS'].as_reference = True - return response diff --git a/tests/cosmo/processes/L1B.py b/tests/cosmo/processes/L1B.py index d60472c..833c611 100644 --- a/tests/cosmo/processes/L1B.py +++ b/tests/cosmo/processes/L1B.py @@ -1,4 +1,4 @@ -from pywps import ComplexInput, Format, ComplexOutput, Process, LiteralInput +from pywps import ComplexInput, Format, ComplexOutput, Process from pywps.app.Common import Metadata @@ -27,8 +27,6 @@ def __init__(self): ) def _handler(self, request, response): - - response.outputs['MDG'].file = '/root/workdir/CSKS2_GEC_B_HI_16_HH_RA_SF_20130301045754_20130301045801.S01.QLK.tif' + response.outputs['MDG'].file = request.inputs['SCS'][0].file response.outputs['MDG'].as_reference = True - return response diff --git a/tests/cosmo/processes/L1C.py b/tests/cosmo/processes/L1C.py index cbb3758..f807a56 100644 --- a/tests/cosmo/processes/L1C.py +++ b/tests/cosmo/processes/L1C.py @@ -1,4 +1,4 @@ -from pywps import ComplexInput, Format, ComplexOutput, Process, LiteralInput +from pywps import ComplexInput, Format, ComplexOutput, Process from pywps.app.Common import Metadata @@ -27,8 +27,6 @@ def __init__(self): ) def _handler(self, request, response): - - response.outputs['GEC'].file = '/root/workdir/CSKS2_GEC_B_HI_16_HH_RA_SF_20130301045754_20130301045801.S01.QLK.tif' + response.outputs['GEC'].file = request.inputs['MDG'][0].file response.outputs['GEC'].as_reference = True - return response diff --git a/tests/cosmo/processes/L1D.py b/tests/cosmo/processes/L1D.py index 6245cca..c8751fa 100644 --- a/tests/cosmo/processes/L1D.py +++ b/tests/cosmo/processes/L1D.py @@ -1,11 +1,11 @@ -from pywps import Format, ComplexOutput, Process, LiteralInput +from pywps import Format, ComplexOutput, Process, ComplexInput from pywps.app.Common import Metadata class L1D(Process): def __init__(self): inputs = [ - LiteralInput('MDG', 'MDG', data_type='string') + ComplexInput('MDG', 'MDG', [Format('image/tiff; subtype=geotiff')]) ] outputs = [ @@ -27,8 +27,6 @@ def __init__(self): ) def _handler(self, request, response): - - response.outputs['GTC'].file = '/root/workdir/CSKS2_GEC_B_HI_16_HH_RA_SF_20130301045754_20130301045801.S01.QLK.tif' + response.outputs['GTC'].file = request.inputs['MDG'][0].file response.outputs['GTC'].as_reference = True - return response diff --git a/tests/cosmo/requirements.txt b/tests/cosmo/requirements.txt new file mode 100644 index 0000000..f229360 --- /dev/null +++ b/tests/cosmo/requirements.txt @@ -0,0 +1 @@ +requests diff --git a/tests/cosmo/settings.py b/tests/cosmo/settings.py index 4738d40..982d414 100644 --- a/tests/cosmo/settings.py +++ b/tests/cosmo/settings.py @@ -27,7 +27,7 @@ identifier='crude', title='crude', abstract='Crude data', - dataType='string' + format='GEOTIFF' ) ], outputs=[ diff --git a/tests/cosmo/test_cosmo.py b/tests/cosmo/test_cosmo.py index 94f49ed..82bce81 100644 --- a/tests/cosmo/test_cosmo.py +++ b/tests/cosmo/test_cosmo.py @@ -5,6 +5,8 @@ from string import Template from requests import post, get from settings import * +from time import sleep +import urlparse class TestCosmoSkymed(unittest.TestCase): @@ -28,7 +30,9 @@ def create_processes(self): file=(process.get('file'), open(process.get('file'), 'rb'), 'application/octet-stream') ) ) - self.assertEqual(response.status_code, 201, 'Failed to create process {}'.format(process.get('identifier'))) + self.assertEqual( + response.status_code, 201, 'Failed to create process {}'.format(process.get('identifier')) + ) def create_chain(self): if not self.chain_exists(CHAIN.get('identifier')): @@ -41,6 +45,34 @@ def execute_chain(self): tree = ElementTree.ElementTree(ElementTree.fromstring(response.text)).getroot() status_location = tree.get('statusLocation') execution_id = re.search('(.*)/(.*).xml', status_location).group(2) + + seconds = .5 + finished = False + timeout = 20 + time_counter = 0 + + while not finished: + + self.assertFalse( + time_counter >= timeout, + 'Asynchronous chain executions take more than {} seconds'.format(timeout) + ) + + path = urlparse.urlparse(status_location).path + + status_response = get('{}/{}/{}'.format(HOST, 'wps', path)) + self.assertEqual(status_response.status_code, 200, 'Failed to retrieve chain Status Location') + status = status_response.text + status_tree = ElementTree.ElementTree(ElementTree.fromstring(status)).getroot() + + namespaces = dict(wps='http://www.opengis.net/wps/1.0.0') + + if status_tree.findall('wps:ProcessOutputs', namespaces): + finished = True + + sleep(seconds) + time_counter += seconds + return execution_id def execution_is_published(self, execution_id): @@ -48,11 +80,15 @@ def execution_is_published(self, execution_id): self.assertEqual(response.status_code, 200, 'Failed to query chain executions') results = response.json().get('results') self.assertTrue(bool(results), 'Chain execution is not published') - self.assertTrue(results[0].get('status').get('name') == 'SUCCESS', 'Chain execution is published, but is not a success') + self.assertTrue( + results[0].get('status').get('name') == 'SUCCESS', 'Chain execution is published, but is not a success' + ) def products_are_published(self, execution_id): response = post(CSW_URL, Template(CSW_GET_RECORDS).safe_substitute(execution_id=execution_id)) - self.assertEqual(response.status_code, 200, 'Failed to query execution products through CSW Get Records operation') + self.assertEqual( + response.status_code, 200, 'Failed to query execution products through CSW Get Records operation' + ) tree = ElementTree.ElementTree(ElementTree.fromstring(response.text)).getroot() search_results = tree.find('csw:SearchResults', namespaces=dict(csw='http://www.opengis.net/cat/csw/2.0.2')) number_of_records = int(search_results.get('numberOfRecordsReturned'))