be able to render the planet with 32gb of RAM

CMakeLists.txt

@@ -97,14 +97,18 @@ file(GLOB tilemaker_src_files
 	src/mbtiles.cpp
 	src/mmap_allocator.cpp
 	src/node_stores.cpp
+	src/options_parser.cpp
 	src/osm_lua_processing.cpp
 	src/osm_mem_tiles.cpp
 	src/osm_store.cpp
 	src/output_object.cpp
 	src/pbf_blocks.cpp
 	src/pmtiles.cpp
+	src/pooled_string.cpp
 	src/read_pbf.cpp
 	src/read_shp.cpp
+	src/sharded_node_store.cpp
+	src/sharded_way_store.cpp
 	src/shared_data.cpp
 	src/shp_mem_tiles.cpp
 	src/sorted_node_store.cpp

Makefile

@@ -106,14 +106,18 @@ tilemaker: \
 	src/mbtiles.o \
 	src/mmap_allocator.o \
 	src/node_stores.o \
+	src/options_parser.o \
 	src/osm_lua_processing.o \
 	src/osm_mem_tiles.o \
 	src/osm_store.o \
 	src/output_object.o \
 	src/pbf_blocks.o \
 	src/pmtiles.o \
+	src/pooled_string.o \
 	src/read_pbf.o \
 	src/read_shp.o \
+	src/sharded_node_store.o \
+	src/sharded_way_store.o \
 	src/shared_data.o \
 	src/shp_mem_tiles.o \
 	src/sorted_node_store.o \
@@ -125,15 +129,57 @@ tilemaker: \
 	src/write_geometry.o
 	$(CXX) $(CXXFLAGS) -o tilemaker $^ $(INC) $(LIB) $(LDFLAGS)
 
-test: test_sorted_way_store
+test: \
+	test_append_vector \
+	test_attribute_store \
+	test_deque_map \
+	test_pooled_string \
+	test_sorted_node_store \
+	test_sorted_way_store
+
+test_append_vector: \
+	src/mmap_allocator.o \
+	test/append_vector.test.o
+	$(CXX) $(CXXFLAGS) -o test.append_vector $^ $(INC) $(LIB) $(LDFLAGS) && ./test.append_vector
+
+test_attribute_store: \
+	src/mmap_allocator.o \
+	src/attribute_store.o \
+	src/pooled_string.o \
+	test/attribute_store.test.o
+	$(CXX) $(CXXFLAGS) -o test.attribute_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.attribute_store
+
+test_deque_map: \
+	test/deque_map.test.o
+	$(CXX) $(CXXFLAGS) -o test.deque_map $^ $(INC) $(LIB) $(LDFLAGS) && ./test.deque_map
+
+test_options_parser: \
+	src/options_parser.o \
+	test/options_parser.test.o
+	$(CXX) $(CXXFLAGS) -o test.options_parser $^ $(INC) $(LIB) $(LDFLAGS) && ./test.options_parser
+
+test_pooled_string: \
+	src/mmap_allocator.o \
+	src/pooled_string.o \
+	test/pooled_string.test.o
+	$(CXX) $(CXXFLAGS) -o test.pooled_string $^ $(INC) $(LIB) $(LDFLAGS) && ./test.pooled_string
+
+test_sorted_node_store: \
+	src/external/streamvbyte_decode.o \
+	src/external/streamvbyte_encode.o \
+	src/external/streamvbyte_zigzag.o \
+	src/mmap_allocator.o \
+	src/sorted_node_store.o \
+	test/sorted_node_store.test.o
+	$(CXX) $(CXXFLAGS) -o test.sorted_node_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.sorted_node_store
 
 test_sorted_way_store: \
 	src/external/streamvbyte_decode.o \
 	src/external/streamvbyte_encode.o \
 	src/external/streamvbyte_zigzag.o \
 	src/mmap_allocator.o \
 	src/sorted_way_store.o \
-	src/sorted_way_store.test.o
+	test/sorted_way_store.test.o
 	$(CXX) $(CXXFLAGS) -o test.sorted_way_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.sorted_way_store
 
 
@@ -153,6 +199,6 @@ install:
 	install docs/man/tilemaker.1 ${DESTDIR}${MANPREFIX}/man1/
 
 clean:
-	rm -f tilemaker src/*.o src/external/*.o include/*.o include/*.pb.h
+	rm -f tilemaker src/*.o src/external/*.o include/*.o include/*.pb.h test/*.o
 
 .PHONY: install

include/append_vector.h

@@ -0,0 +1,195 @@
+#ifndef _APPEND_VECTOR_H
+#define _APPEND_VECTOR_H
+
+#include "mmap_allocator.h"
+#include <vector>
+#include <queue>
+
+// Tilemaker collects OutputObjects in a list that
+// - spills to disk
+// - only gets appended to
+//
+// Vector is great for linear access, but resizes cause expensive disk I/O to
+// copy elements.
+//
+// Deque is great for growing without disk I/O, but it allocates in blocks of 512,
+// which is inefficient for linear access.
+//
+// Instead, we author a limited vector-of-vectors class that allocates in bigger chunks,
+// to get the best of both worlds.
+
+#define APPEND_VECTOR_SIZE 8192
+namespace AppendVectorNS {
+	template <class T>
+	class AppendVector {
+	public:
+		struct Iterator {
+			using iterator_category = std::random_access_iterator_tag;
+			using difference_type   = std::ptrdiff_t;
+			using value_type        = T;
+			using pointer           = T*;
+			using reference         = T&;
+
+			Iterator(AppendVector<T>& appendVector, uint16_t vec, uint16_t offset):
+				appendVector(&appendVector), vec(vec), offset(offset) {}
+
+			Iterator():
+				appendVector(nullptr), vec(0), offset(0) {}
+
+
+			bool operator<(const Iterator& other) const {
+				if (vec < other.vec)
+					return true;
+
+				if (vec > other.vec)
+					return false;
+
+				return offset < other.offset;
+			}
+
+			bool operator>=(const Iterator& other) const {
+				return !(*this < other);
+			}
+
+			Iterator operator-(int delta) const {
+				int64_t absolute = vec * APPEND_VECTOR_SIZE + offset;
+				absolute -= delta;
+				return Iterator(*appendVector, absolute / APPEND_VECTOR_SIZE, absolute % APPEND_VECTOR_SIZE);
+			}
+
+			Iterator operator+(int delta) const {
+				int64_t absolute = vec * APPEND_VECTOR_SIZE + offset;
+				absolute += delta;
+				return Iterator(*appendVector, absolute / APPEND_VECTOR_SIZE, absolute % APPEND_VECTOR_SIZE);
+			}
+
+			bool operator==(const Iterator& other) const {
+				return appendVector == other.appendVector && vec == other.vec && offset == other.offset;
+			}
+
+			bool operator!=(const Iterator& other) const {
+				return !(*this == other);
+			}
+
+			std::ptrdiff_t operator-(const Iterator& other) const {
+				int64_t absolute = vec * APPEND_VECTOR_SIZE + offset;
+				int64_t otherAbsolute = other.vec * APPEND_VECTOR_SIZE + other.offset;
+
+				return absolute - otherAbsolute;
+			}
+
+			reference operator*() const {
+				auto& vector = appendVector->vecs[vec];
+				auto& el = vector[offset];
+				return el;
+			}
+
+			pointer operator->() const {
+				auto& vector = appendVector->vecs[vec];
+				auto& el = vector[offset];
+				return &el;
+			}
+
+			Iterator& operator+= (int delta) {
+				int64_t absolute = vec * APPEND_VECTOR_SIZE + offset;
+				absolute += delta;
+
+				vec = absolute / APPEND_VECTOR_SIZE;
+				offset = absolute % APPEND_VECTOR_SIZE;
+				return *this;
+			}
+
+			Iterator& operator-= (int delta) {
+				int64_t absolute = vec * APPEND_VECTOR_SIZE + offset;
+				absolute -= delta;
+
+				vec = absolute / APPEND_VECTOR_SIZE;
+				offset = absolute % APPEND_VECTOR_SIZE;
+				return *this;
+			}
+
+			// Prefix increment
+			Iterator& operator++() {
+				offset++;
+				if (offset == APPEND_VECTOR_SIZE) {
+					offset = 0;
+					vec++;
+				}
+				return *this;
+			}  
+
+			// Postfix increment
+			Iterator operator++(int) { Iterator tmp = *this; ++(*this); return tmp; }
+
+			// Prefix decrement
+			Iterator& operator--() {
+				if (offset > 0) {
+					offset--;
+				} else {
+					vec--;
+					offset = APPEND_VECTOR_SIZE - 1;
+				}
+
+				return *this;
+			}
+
+			// Postfix decrement
+			Iterator operator--(int) { Iterator tmp = *this; --(*this); return tmp; }
+
+		private:
+			mutable AppendVector<T>* appendVector;
+			int32_t vec, offset;
+		};
+
+		AppendVector():
+			count(0),
+			vecs(1) {
+		}
+
+		void clear() {
+			count = 0;
+			vecs.clear();
+			vecs.push_back(std::vector<T, mmap_allocator<T>>());
+			vecs.back().reserve(APPEND_VECTOR_SIZE);
+		}
+
+		size_t size() const {
+			return count;
+		}
+
+		T& operator [](int idx) {
+			auto& vec = vecs[idx / APPEND_VECTOR_SIZE];
+			auto& el = vec[idx % APPEND_VECTOR_SIZE];
+			return el;
+		}
+
+		Iterator begin() {
+			return Iterator(*this, 0, 0);
+		}
+
+		Iterator end() {
+			return Iterator(*this, vecs.size() - 1, count % APPEND_VECTOR_SIZE);
+		}
+
+		void push_back(const T& el) {
+			if (vecs.back().capacity() == 0)
+				vecs.back().reserve(APPEND_VECTOR_SIZE);
+
+			vecs.back().push_back(el);
+
+			if (vecs.back().size() == vecs.back().capacity()) {
+				vecs.push_back(std::vector<T, mmap_allocator<T>>());
+				vecs.back().reserve(APPEND_VECTOR_SIZE);
+			}
+
+			count++;
+		}
+
+		size_t count;
+		std::deque<std::vector<T, mmap_allocator<T>>> vecs;
+	};
+}
+
+#undef APPEND_VECTOR_SIZE
+
+#endif