Merge pull request #116 from plepe/json-parse-complete

DB/Overpass: parse complete json answer from Overpass

CHANGELOG.creole

@@ -1,5 +1,6 @@
 0.10.4 release upcoming
 * DB/Overpass: set max allowed memory by config option {{{db.overpass-memory}}}
+* DB/Overpass: load full response at once; this greatly improves performance, but needs more memory.
 * Bugfix :lefthandtraffic / :righthandtraffic (use correct SRIDs) -> need database re-initialization (-r init)
 * When converting SVGs to PNGs keep transparent background (e.g. for {{{image}}} resp. {{{repeat-image}}}); Mapnik 3.0 still does not support SVGs for LinePatterns (auto-convert SVG to PNG)
 * Icons (icon-image): maki icons may be the result of an eval expression. Other icons must have a '.' in their file name.

doc/config_options.md

@@ -30,6 +30,6 @@ Advances options:
 | db.has-hstore    | osm2pgsql only: Additional tags can be read from the 'tags' column (of type hstore). Usually autodetected. Needed when using offline mode. | |
 | db.multipolygons | osmosis only: Specify whether the multipolygons table is present and should be used. Usually autodected. Needed when using offline mode (default: false) | true/false
 | db.hstore_key_index| osm2pgsql/osmosis: Assume that there's a btree index on the specified keys (,-separated) and therefore add a tag-exists condition into the SQL query. E.g.: db.hstore_key_index=sport,amenity . | |
-| db.serial_requests | overpass only: Always finish parsing result data, before sending a parallel request (e.g. for handling relationships). Necessary on some APIs which disallow multiple requests at once. Impacts memory usage, as results need to be cached. | true/**false**
 | db.overpass-timeout | overpass only: specify timeout in seconds for queries to the Overpass API (make sure that the Overpass server accepts timeouts that long, e.g. by adding a "Timeout" parameter to Apache2). As a render job usually needs several queries, the total rendering time might be a multiple of this value. | 180
 | db.overpass-memory | overpass only: Sets the 'maxsize' parameter for an overpass query, which indicates the maximum allowed memory for the query in bytes RAM on the server. | 536870912 (512 MB)
+| db.overpass-profiler | overpass only: prints all queries, their duration (incl. parsing json) and the count of returned features. | true/**false**

doc/database.md

@@ -26,10 +26,10 @@ Behaviour can be influenced with the following config options:
 | Config option    | Description | Possible values
 |------------------|-------------|-----------------
 | db.overpass-url  | overpass only: Use this alternative Overpass API url. default: http://overpass-api.de/api | |
-| db.serial_requests | overpass only: Always finish parsing result data, before sending a parallel request (e.g. for handling relationships). Necessary on some APIs which disallow multiple requests at once. Impacts memory usage, as results need to be cached. | true/**false**
 | debug.profiler   | during execution, show some statistics about query/processing time and count of objects. | true/**false** |
 | db.overpass-timeout | specify timeout in seconds for queries to the Overpass API (make sure that the Overpass server accepts timeouts that long, e.g. by adding a "Timeout" parameter to Apache2). As a render job usually needs several queries, the total rendering time might be a multiple of this value. | 180
 | db.overpass-memory | Sets the 'maxsize' parameter for an overpass query, which indicates the maximum allowed memory for the query in bytes RAM on the server. | 536870912 (512 MB)
+| db.overpass-profiler | overpass only: prints all queries, their duration (incl. parsing json) and the count of returned features. | true/**false**
 
 Example usage:
 ```sh

pgmapcss/db/overpass/db.py

@@ -13,9 +13,6 @@ def __init__(self, conn, stat):
         if not 'db.overpass-url' in self.stat['config']:
             self.stat['config']['db.overpass-url'] = 'http://overpass-api.de/api'
 
-        if 'debug.profiler' in self.stat['config']:
-            self.stat['config']['db.serial_requests'] = True
-
         if not 'db.overpass-timeout' in self.stat['config']:
             self.stat['config']['db.overpass-timeout'] = '180'
 

pgmapcss/db/overpass/db_functions.py

@@ -5,12 +5,9 @@ def overpass_query(query):
     import urllib.parse
     import json
 
-# START db.serial_requests
-    ret = []
-# END db.serial_requests
-# START debug.profiler
+# START debug.overpass-profiler
     time_start = datetime.datetime.now()
-# END debug.profiler
+# END debug.overpass-profiler
     url = '{db.overpass-url}/interpreter'
     data = urllib.parse.urlencode({ 'data': query })
     data = data.encode('utf-8')
@@ -22,71 +19,30 @@ def overpass_query(query):
         plpy.warning('Overpass query failed:\n' + query)
         raise
 
-    block = ''
-    mode = 0
-    while True:
-        try:
-            r = f.readline().decode('utf-8')
-        except urllib.error.HTTPError as err:
-            plpy.warning('Overpass query failed (after {} features):\n'.format(count) + query)
-            raise
-
-        if r == '':
-            if mode == 2:
-                f.close()
-
-                after_elements = json.loads(block)
-
-# START debug.profiler
-                plpy.warning('%s\nquery took %.2fs for %d features' % (query, (datetime.datetime.now() - time_start).total_seconds(), len(ret)))
-# END debug.profiler
-# START db.serial_requests
-                for r in ret:
-                    yield r
-# END db.serial_requests
-
-                if 'remark' in after_elements:
-                    # ignore timeout if it happens in "print"
-                    if not re.search("Query timed out in \"print\"", after_elements['remark']):
-                      raise Exception('Error in Overpass API (after {} features): {}\nFailed query was:\n{}'.format(count, after_elements['remark'], query))
-
-                return
-
-            raise Exception('Connection closed early (after {} features) from Overpass API'.format(count))
-
-        if mode == 0:
-            if re.search('"elements":', r):
-                mode = 1
-
-            # areas not initialized -> ignore
-            if re.search('osm3s_v[0-9\.]+_areas', r):
-                f.close()
-                return
-
-        elif mode == 1:
-            if re.match('}', r):
-                block += '}'
-# START db.serial_requests
-                ret.append(json.loads(block))
-# ELSE db.serial_requests
-                yield json.loads(block)
-# END db.serial_requests
-
-                count += 1
-                block = ''
-
-            elif re.match('\s*$', block) and re.match('.*\]', r):
-                mode = 2
-                block = '{'
+    try:
+        r = f.read().decode('utf-8')
+    except urllib.error.HTTPError as err:
+        plpy.warning('Overpass query failed (after {} features):\n'.format(count) + query)
+        raise
 
-            else:
-                block += r
+    # areas not initialized -> ignore
+    if re.search('osm3s_v[0-9\.]+_areas', r):
+        f.close()
+        return
+
+    data = json.loads(r)
+
+    if 'remark' in data:
+        # ignore timeout if it happens in "print"
+        if not re.search("Query timed out in \"print\"", data['remark']):
+          raise Exception('Error in Overpass API (after {} features): {}\nFailed query was:\n{}'.format(count, data['remark'], query))
 
-        elif mode == 2:
-            block += r
+# START debug.overpass-profiler
+    plpy.warning('%s\nquery took %.2fs for %d features' % (query, (datetime.datetime.now() - time_start).total_seconds(), len(data['elements'])))
+# END debug.overpass-profiler
 
-    if mode == 0:
-        raise Exception('Could not parse Overpass API result')
+    for e in data['elements']:
+        yield e
 
 def node_geom(lat, lon):
     global geom_plan